From 7d1f20a5d005cbf2a6bb16a0a3711199eedbf6ad Mon Sep 17 00:00:00 2001 From: Keith Herrington Date: Tue, 21 Apr 2026 22:36:44 -0700 Subject: [PATCH 1/4] feat: harden review flows and backfill regression coverage --- .github/workflows/ci.yml | 10 +- .github/workflows/nightly-issue-sync.yml | 2 +- .github/workflows/nightly-reaction-sync.yml | 2 +- bunfig.toml | 7 +- eslint.config.mjs | 39 + package.json | 33 +- scripts/backfill-issues.ts | 1 - scripts/backfill-review-comments.ts | 1 - scripts/check-migrations-have-downs.test.ts | 283 +++++++ scripts/check-migrations-have-downs.ts | 436 +++++++++++ scripts/check-orphaned-tests.test.ts | 185 +++++ scripts/check-orphaned-tests.ts | 425 +++++++++++ scripts/cleanup-wiki-issue.ts | 1 - scripts/publish-wiki-updates.ts | 1 - scripts/sync-triage-reactions.ts | 1 - scripts/verify-m027-s03.test.ts | 28 +- scripts/verify-m029-s04.ts | 8 +- scripts/verify-m044-s01.ts | 1 - scripts/verify-m049-s02.ts | 1 - scripts/verify-m053.test.ts | 122 +++ scripts/verify-m053.ts | 288 +++++++ scripts/verify-m054-s01.test.ts | 152 ++++ scripts/verify-m054-s01.ts | 396 ++++++++++ scripts/verify-m054-s02.test.ts | 270 +++++++ scripts/verify-m054-s02.ts | 344 +++++++++ scripts/verify-m054-s03.test.ts | 256 +++++++ scripts/verify-m054-s03.ts | 442 +++++++++++ scripts/verify-m054-s04.test.ts | 375 +++++++++ scripts/verify-m054-s04.ts | 632 ++++++++++++++++ scripts/verify-m055-s01.test.ts | 299 ++++++++ scripts/verify-m055-s01.ts | 376 ++++++++++ scripts/verify-m055-s02.test.ts | 314 ++++++++ scripts/verify-m055-s02.ts | 387 ++++++++++ scripts/verify-m055-s03.test.ts | 396 ++++++++++ scripts/verify-m055-s03.ts | 564 ++++++++++++++ scripts/verify-m056-s01.test.ts | 263 +++++++ scripts/verify-m056-s01.ts | 443 +++++++++++ scripts/verify-m056-s02.test.ts | 315 ++++++++ scripts/verify-m056-s02.ts | 551 ++++++++++++++ scripts/verify-m056-s03.test.ts | 328 ++++++++ scripts/verify-m056-s03.ts | 421 +++++++++++ scripts/verify-m057-s02.ts | 30 + scripts/verify-m057-s03.ts | 32 + scripts/verify-m057-s04.test.ts | 25 + scripts/verify-m057-s04.ts | 36 + scripts/verify-m058-s01.test.ts | 267 +++++++ scripts/verify-m058-s01.ts | 334 ++++++++ scripts/verify-m058-s02.test.ts | 317 ++++++++ scripts/verify-m058-s02.ts | 436 +++++++++++ scripts/verify-m058-s03.test.ts | 308 ++++++++ scripts/verify-m058-s03.ts | 423 +++++++++++ scripts/verify-m059-s01.test.ts | 391 ++++++++++ scripts/verify-m059-s01.ts | 710 ++++++++++++++++++ scripts/verify-m059-s02.test.ts | 287 +++++++ scripts/verify-m059-s02.ts | 548 ++++++++++++++ scripts/verify-m060-s01.test.ts | 356 +++++++++ scripts/verify-m060-s01.ts | 614 +++++++++++++++ scripts/verify-m060-s02.test.ts | 211 ++++++ scripts/verify-m060-s02.ts | 367 +++++++++ src/config.test.ts | 41 - src/config.ts | 16 - .../calibration-change-contract.test.ts | 10 +- src/contributor/calibration-evaluator.test.ts | 8 +- src/contributor/profile-store.test.ts | 29 +- src/contributor/xbmc-fixture-refresh.test.ts | 6 +- src/contributor/xbmc-fixture-refresh.ts | 1 - src/contributor/xbmc-fixture-snapshot.test.ts | 6 +- src/db/migrate.ts | 4 + .../012-wiki-staleness-run-state.down.sql | 1 + .../migrations/013-review-clusters.down.sql | 3 + .../016-issue-triage-state.down.sql | 1 + .../migrations/025-wiki-style-cache.down.sql | 1 + .../migrations/026-guardrail-audit.down.sql | 1 + src/db/migrations/030-reserved.down.sql | 7 + src/db/migrations/030-reserved.sql | 10 + .../033-canonical-code-corpus.down.sql | 2 + src/db/migrations/034-review-graph.down.sql | 4 + .../migrations/035-generated-rules.down.sql | 1 + .../036-suggestion-cluster-models.down.sql | 1 + src/execution/executor.test.ts | 264 +++++-- src/execution/executor.ts | 44 +- src/execution/review-prompt.test.ts | 13 - src/execution/review-prompt.ts | 18 +- src/execution/types.ts | 7 - src/handlers/ci-failure.test.ts | 678 +++++++++++++++++ src/handlers/mention.test.ts | 476 ++++++++++++ src/handlers/mention.ts | 11 +- src/handlers/review.test.ts | 514 +------------ src/handlers/review.ts | 393 ++++------ src/index.ts | 12 - src/jobs/fork-manager.test.ts | 314 ++++++++ src/jobs/fork-manager.ts | 1 + src/jobs/gist-publisher.test.ts | 123 +++ src/knowledge/cluster-scheduler.test.ts | 117 +++ src/knowledge/cluster-scheduler.ts | 5 + src/knowledge/isolation.test.ts | 199 +++++ src/knowledge/issue-retrieval.test.ts | 179 +++++ src/knowledge/store.test.ts | 21 +- src/knowledge/test-coverage-exemptions.ts | 26 + src/knowledge/wiki-fetch.test.ts | 61 ++ src/knowledge/wiki-linkshere-fetcher.test.ts | 255 +++++++ src/knowledge/wiki-popularity-config.test.ts | 98 +++ src/knowledge/wiki-popularity-scorer.test.ts | 299 ++++++++ src/knowledge/wiki-popularity-scorer.ts | 5 + src/knowledge/wiki-store.test.ts | 10 +- src/lib/review-utils.test.ts | 9 - src/lib/review-utils.ts | 4 +- src/routes/slack-commands.test.ts | 1 - src/routes/slack-events.test.ts | 1 - src/routes/webhooks.test.ts | 314 ++++++++ src/slack/assistant-handler.test.ts | 84 ++- src/slack/write-runner.test.ts | 512 ++++++++++++- src/telemetry/store.test.ts | 19 +- src/webhook/dedup.test.ts | 43 ++ src/webhook/filters.test.ts | 61 ++ src/webhook/router.test.ts | 217 ++++++ src/webhook/verify.test.ts | 47 ++ src/webhook/verify.ts | 5 +- 118 files changed, 19640 insertions(+), 1064 deletions(-) create mode 100644 eslint.config.mjs create mode 100644 scripts/check-migrations-have-downs.test.ts create mode 100644 scripts/check-migrations-have-downs.ts create mode 100644 scripts/check-orphaned-tests.test.ts create mode 100644 scripts/check-orphaned-tests.ts create mode 100644 scripts/verify-m053.test.ts create mode 100644 scripts/verify-m053.ts create mode 100644 scripts/verify-m054-s01.test.ts create mode 100644 scripts/verify-m054-s01.ts create mode 100644 scripts/verify-m054-s02.test.ts create mode 100644 scripts/verify-m054-s02.ts create mode 100644 scripts/verify-m054-s03.test.ts create mode 100644 scripts/verify-m054-s03.ts create mode 100644 scripts/verify-m054-s04.test.ts create mode 100644 scripts/verify-m054-s04.ts create mode 100644 scripts/verify-m055-s01.test.ts create mode 100644 scripts/verify-m055-s01.ts create mode 100644 scripts/verify-m055-s02.test.ts create mode 100644 scripts/verify-m055-s02.ts create mode 100644 scripts/verify-m055-s03.test.ts create mode 100644 scripts/verify-m055-s03.ts create mode 100644 scripts/verify-m056-s01.test.ts create mode 100644 scripts/verify-m056-s01.ts create mode 100644 scripts/verify-m056-s02.test.ts create mode 100644 scripts/verify-m056-s02.ts create mode 100644 scripts/verify-m056-s03.test.ts create mode 100644 scripts/verify-m056-s03.ts create mode 100644 scripts/verify-m057-s02.ts create mode 100644 scripts/verify-m057-s03.ts create mode 100644 scripts/verify-m057-s04.test.ts create mode 100644 scripts/verify-m057-s04.ts create mode 100644 scripts/verify-m058-s01.test.ts create mode 100644 scripts/verify-m058-s01.ts create mode 100644 scripts/verify-m058-s02.test.ts create mode 100644 scripts/verify-m058-s02.ts create mode 100644 scripts/verify-m058-s03.test.ts create mode 100644 scripts/verify-m058-s03.ts create mode 100644 scripts/verify-m059-s01.test.ts create mode 100644 scripts/verify-m059-s01.ts create mode 100644 scripts/verify-m059-s02.test.ts create mode 100644 scripts/verify-m059-s02.ts create mode 100644 scripts/verify-m060-s01.test.ts create mode 100644 scripts/verify-m060-s01.ts create mode 100644 scripts/verify-m060-s02.test.ts create mode 100644 scripts/verify-m060-s02.ts create mode 100644 src/db/migrations/012-wiki-staleness-run-state.down.sql create mode 100644 src/db/migrations/013-review-clusters.down.sql create mode 100644 src/db/migrations/016-issue-triage-state.down.sql create mode 100644 src/db/migrations/025-wiki-style-cache.down.sql create mode 100644 src/db/migrations/026-guardrail-audit.down.sql create mode 100644 src/db/migrations/030-reserved.down.sql create mode 100644 src/db/migrations/030-reserved.sql create mode 100644 src/db/migrations/033-canonical-code-corpus.down.sql create mode 100644 src/db/migrations/034-review-graph.down.sql create mode 100644 src/db/migrations/035-generated-rules.down.sql create mode 100644 src/db/migrations/036-suggestion-cluster-models.down.sql create mode 100644 src/handlers/ci-failure.test.ts create mode 100644 src/jobs/fork-manager.test.ts create mode 100644 src/jobs/gist-publisher.test.ts create mode 100644 src/knowledge/cluster-scheduler.test.ts create mode 100644 src/knowledge/isolation.test.ts create mode 100644 src/knowledge/issue-retrieval.test.ts create mode 100644 src/knowledge/test-coverage-exemptions.ts create mode 100644 src/knowledge/wiki-fetch.test.ts create mode 100644 src/knowledge/wiki-linkshere-fetcher.test.ts create mode 100644 src/knowledge/wiki-popularity-config.test.ts create mode 100644 src/knowledge/wiki-popularity-scorer.test.ts create mode 100644 src/routes/webhooks.test.ts create mode 100644 src/webhook/dedup.test.ts create mode 100644 src/webhook/filters.test.ts create mode 100644 src/webhook/router.test.ts create mode 100644 src/webhook/verify.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 316ece08..25ed6cee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,11 +29,17 @@ jobs: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v2 with: - bun-version: latest + bun-version: 1.3.8 - run: bun install + - run: bun run lint + - run: bun run verify:m056:s03 + - run: bun run verify:m059:s01 + - run: bun run verify:m059:s02 + - run: bun run check:orphaned-tests # Bun has been unstable on GitHub runners with one monolithic test process. # Keep DB-backed tests on a low concurrency cap and split the suite into # two shorter invocations to avoid cross-file schema interference and runner crashes. - - run: bun test --max-concurrency=2 scripts src/contributor src/enforcement src/execution src/feedback src/handlers src/jobs src/lib src/review-audit src/review-graph src/routes src/slack src/structural-impact src/telemetry src/triage + # The first run covers scripts plus non-knowledge src tests; src/knowledge stays isolated. + - run: bun test --max-concurrency=2 scripts src - run: bun test --max-concurrency=2 src/knowledge - run: bunx tsc --noEmit diff --git a/.github/workflows/nightly-issue-sync.yml b/.github/workflows/nightly-issue-sync.yml index 28df818f..0b9f828a 100644 --- a/.github/workflows/nightly-issue-sync.yml +++ b/.github/workflows/nightly-issue-sync.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v2 with: - bun-version: latest + bun-version: 1.3.8 - run: bun install --frozen-lockfile - name: Run issue sync run: bun scripts/backfill-issues.ts --sync diff --git a/.github/workflows/nightly-reaction-sync.yml b/.github/workflows/nightly-reaction-sync.yml index b312fe5f..38cea3ce 100644 --- a/.github/workflows/nightly-reaction-sync.yml +++ b/.github/workflows/nightly-reaction-sync.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v2 with: - bun-version: latest + bun-version: 1.3.8 - run: bun install --frozen-lockfile - name: Sync triage comment reactions run: bun scripts/sync-triage-reactions.ts diff --git a/bunfig.toml b/bunfig.toml index 15ec7860..5e3e88e4 100644 --- a/bunfig.toml +++ b/bunfig.toml @@ -1,4 +1,5 @@ [test] -# Only scan for tests in the actual codebase. -# This repo vendors reference code under tmp/ which contains upstream tests. -root = "src" +# Scan the repo root so targeted script verifiers can be run by path, but ignore +# vendored reference code under tmp/ so upstream tests are never discovered. +root = "." +pathIgnorePatterns = ["tmp/**"] diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 00000000..f112c66e --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,39 @@ +import tsParser from "@typescript-eslint/parser"; + +const typescriptFiles = ["src/**/*.ts", "scripts/**/*.ts"]; +const operatorFacingCliFiles = [ + // Migration progress and rollback status are intentionally printed for humans. + "src/db/migrate.ts", + // These are operator/entrypoint surfaces that fail loudly on startup problems. + "src/index.ts", + "src/config.ts", + "src/execution/agent-entrypoint.ts", + // Repo-owned scripts are mostly verifiers/CLI utilities, so console output is expected. + "scripts/**/*.ts", +]; + +export default [ + { + ignores: ["node_modules/**", "tmp/**", ".gsd/**"], + }, + { + files: typescriptFiles, + languageOptions: { + parser: tsParser, + ecmaVersion: "latest", + sourceType: "module", + }, + rules: { + // Start with the repo contract this slice actually needs: catch accidental + // console usage in normal source files without introducing a noisy day-one gate. + "no-console": "error", + }, + }, + { + files: operatorFacingCliFiles, + rules: { + // These surfaces intentionally communicate directly with operators. + "no-console": "off", + }, + }, +]; diff --git a/package.json b/package.json index caf309c3..759f43fe 100644 --- a/package.json +++ b/package.json @@ -3,9 +3,14 @@ "module": "src/index.ts", "type": "module", "private": true, + "packageManager": "bun@1.3.8", + "engines": { + "bun": "1.3.8" + }, "scripts": { "dev": "bun run --watch src/index.ts", "start": "bun run src/index.ts", + "lint": "eslint src scripts", "audit:embeddings": "bun scripts/embedding-audit.ts", "repair:embeddings": "bun scripts/embedding-repair.ts", "repair:wiki-embeddings": "bun scripts/wiki-embedding-repair.ts", @@ -61,12 +66,36 @@ "verify:m048:s02": "bun scripts/verify-m048-s02.ts", "verify:m048:s03": "bun scripts/verify-m048-s03.ts", "verify:m049:s02": "bun scripts/verify-m049-s02.ts", - "verify:m052": "bun scripts/verify-m052.ts" + "verify:m053": "bun scripts/verify-m053.ts", + "verify:m054:s01": "bun scripts/verify-m054-s01.ts", + "verify:m055:s01": "bun scripts/verify-m055-s01.ts", + "verify:m055:s02": "bun scripts/verify-m055-s02.ts", + "verify:m055:s03": "bun scripts/verify-m055-s03.ts", + "verify:m056:s01": "bun scripts/verify-m056-s01.ts", + "verify:m056:s02": "bun scripts/verify-m056-s02.ts", + "verify:m056:s03": "bun scripts/verify-m056-s03.ts", + "verify:m057:s02": "bun scripts/verify-m057-s02.ts", + "verify:m057:s03": "bun scripts/verify-m057-s03.ts", + "verify:m057:s04": "bun scripts/verify-m057-s04.ts", + "verify:m058:s01": "bun scripts/verify-m058-s01.ts", + "verify:m058:s02": "bun scripts/verify-m058-s02.ts", + "verify:m058:s03": "bun scripts/verify-m058-s03.ts", + "verify:m059:s01": "bun scripts/verify-m059-s01.ts", + "verify:m059:s02": "bun scripts/verify-m059-s02.ts", + "verify:m060:s01": "bun scripts/verify-m060-s01.ts", + "verify:m060:s02": "bun scripts/verify-m060-s02.ts", + "verify:m054:s02": "bun scripts/verify-m054-s02.ts", + "verify:m054:s03": "bun scripts/verify-m054-s03.ts", + "verify:m054:s04": "bun scripts/verify-m054-s04.ts", + "check:migrations-have-downs": "bun scripts/check-migrations-have-downs.ts", + "check:orphaned-tests": "bun scripts/check-orphaned-tests.ts" }, "devDependencies": { "@octokit/webhooks-types": "^7.6.1", "@types/bun": "latest", - "@types/js-yaml": "^4.0.9" + "@types/js-yaml": "^4.0.9", + "@typescript-eslint/parser": "^8.46.1", + "eslint": "^9.38.0" }, "peerDependencies": { "typescript": "^5" diff --git a/scripts/backfill-issues.ts b/scripts/backfill-issues.ts index b5d64789..8a7476df 100644 --- a/scripts/backfill-issues.ts +++ b/scripts/backfill-issues.ts @@ -137,7 +137,6 @@ async function main() { slackKodiaiChannelId: "unused", slackDefaultRepo: repo, slackAssistantModel: "unused", - slackWebhookRelaySources: [], port: 0, logLevel: "info", botAllowList: [], diff --git a/scripts/backfill-review-comments.ts b/scripts/backfill-review-comments.ts index 0c111ede..8c4f604f 100644 --- a/scripts/backfill-review-comments.ts +++ b/scripts/backfill-review-comments.ts @@ -148,7 +148,6 @@ async function main() { slackKodiaiChannelId: "unused", slackDefaultRepo: repo, slackAssistantModel: "unused", - slackWebhookRelaySources: [], port: 0, logLevel: "info", botAllowList: [], diff --git a/scripts/check-migrations-have-downs.test.ts b/scripts/check-migrations-have-downs.test.ts new file mode 100644 index 00000000..6740e444 --- /dev/null +++ b/scripts/check-migrations-have-downs.test.ts @@ -0,0 +1,283 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { CheckerReport } from "./check-migrations-have-downs.ts"; +import { + CHECK_MIGRATIONS_HAVE_DOWNS_CHECK_IDS, + buildCheckMigrationsHaveDownsHarness, + evaluateMigrationPairing, + parseCheckMigrationsHaveDownsArgs, + renderCheckMigrationsHaveDownsReport, +} from "./check-migrations-have-downs.ts"; + +const EXPECTED_CHECK_IDS = [ + "MIGRATIONS-DIR-STATE", + "MIGRATION-ALLOWLIST-STATE", + "MIGRATION-PAIRS", + "PACKAGE-WIRING", +] as const; + +const PASSING_PACKAGE_JSON = JSON.stringify( + { + name: "kodiai", + scripts: { + "check:migrations-have-downs": "bun scripts/check-migrations-have-downs.ts", + }, + }, + null, + 2, +); + +describe("check migrations have downs", () => { + test("exports stable check ids and cli parsing", () => { + expect(CHECK_MIGRATIONS_HAVE_DOWNS_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseCheckMigrationsHaveDownsArgs([])).toEqual({ json: false }); + expect(parseCheckMigrationsHaveDownsArgs(["--json"])).toEqual({ json: true }); + expect(() => parseCheckMigrationsHaveDownsArgs(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes for a fully paired migration set with an empty allowlist and canonical package wiring", async () => { + const report = await evaluateMigrationPairing({ + generatedAt: "2026-04-21T09:00:00.000Z", + readDir: async () => [ + "001-init.sql", + "001-init.down.sql", + "002-users.sql", + "002-users.down.sql", + ], + readPackageJson: async () => PASSING_PACKAGE_JSON, + readTextFile: async () => "-- ok\n", + allowlistEntries: [], + }); + + expect(report.command).toBe("check:migrations-have-downs"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "MIGRATIONS-DIR-STATE", + passed: true, + status_code: "migrations_dir_ok", + }), + expect.objectContaining({ + id: "MIGRATION-ALLOWLIST-STATE", + passed: true, + status_code: "allowlist_empty", + }), + expect.objectContaining({ + id: "MIGRATION-PAIRS", + passed: true, + status_code: "all_rollbacks_present", + }), + expect.objectContaining({ + id: "PACKAGE-WIRING", + passed: true, + status_code: "package_wiring_ok", + }), + ]); + + const rendered = renderCheckMigrationsHaveDownsReport(report); + expect(rendered).toContain("Migration rollback sibling gate: PASS"); + expect(rendered).toContain("MIGRATIONS-DIR-STATE PASS"); + expect(rendered).toContain("MIGRATION-ALLOWLIST-STATE PASS"); + expect(rendered).toContain("MIGRATION-PAIRS PASS"); + expect(rendered).toContain("PACKAGE-WIRING PASS"); + }); + + test("fails with stable status codes for missing rollback siblings, malformed allowlist entries, and package drift", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildCheckMigrationsHaveDownsHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + readDir: async () => [ + "001-init.sql", + "001-init.down.sql", + "002-users.sql", + "README.md", + ], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("001-init.down.sql")) { + return "-- ok\n"; + } + throw new Error(`ENOENT: no such file or directory, open '${filePath}'`); + }, + readPackageJson: async () => JSON.stringify({ name: "kodiai", scripts: {} }), + allowlistEntries: [ + { + migration: "002-users.sql", + rationale: "legacy exception", + }, + { + migration: "999-missing.sql", + rationale: "stale entry", + }, + ], + }); + + const report = JSON.parse(stdout.join("")) as CheckerReport; + + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "MIGRATIONS-DIR-STATE", + passed: true, + status_code: "migrations_dir_ok", + }), + expect.objectContaining({ + id: "MIGRATION-ALLOWLIST-STATE", + passed: false, + status_code: "allowlist_entry_missing_forward_migration", + }), + expect.objectContaining({ + id: "MIGRATION-PAIRS", + passed: false, + status_code: "rollback_missing", + }), + expect.objectContaining({ + id: "PACKAGE-WIRING", + passed: false, + status_code: "package_wiring_missing", + }), + ]); + expect(report.checks[1]?.detail).toContain("999-missing.sql"); + expect(report.checks[2]?.detail).toContain("002-users.sql"); + expect(report.checks[3]?.detail).toContain("check:migrations-have-downs"); + expect(stderr.join(" ")).toContain("allowlist_entry_missing_forward_migration"); + expect(stderr.join(" ")).toContain("rollback_missing"); + expect(stderr.join(" ")).toContain("package_wiring_missing"); + }); + + test("fails closed for unreadable directory state, invalid package json, duplicate allowlist entries, unreadable migration files, and malformed allowlist rationale", async () => { + const dirUnreadable = await evaluateMigrationPairing({ + readDir: async () => { + throw new Error("EACCES: src/db/migrations"); + }, + readPackageJson: async () => PASSING_PACKAGE_JSON, + allowlistEntries: [], + }); + + expect(dirUnreadable.checks[0]).toEqual( + expect.objectContaining({ + id: "MIGRATIONS-DIR-STATE", + passed: false, + status_code: "migrations_dir_unreadable", + }), + ); + expect(dirUnreadable.checks[2]).toEqual( + expect.objectContaining({ + id: "MIGRATION-PAIRS", + passed: false, + status_code: "migrations_scan_unavailable", + }), + ); + + const malformedInputs = await evaluateMigrationPairing({ + readDir: async () => [ + "001-init.sql", + "001-init.down.sql", + "002-users.sql", + "002-users.down.sql", + ], + readPackageJson: async () => "{ nope", + allowlistEntries: [ + { + migration: "002-users.sql", + rationale: "valid rationale", + }, + { + migration: "002-users.sql", + rationale: "duplicate", + }, + ], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("002-users.down.sql")) { + throw new Error("EACCES: 002-users.down.sql"); + } + return "-- ok\n"; + }, + }); + + expect(malformedInputs.checks[1]).toEqual( + expect.objectContaining({ + id: "MIGRATION-ALLOWLIST-STATE", + passed: false, + status_code: "allowlist_duplicate_migration", + }), + ); + expect(malformedInputs.checks[1]?.detail).toContain("002-users.sql"); + expect(malformedInputs.checks[2]).toEqual( + expect.objectContaining({ + id: "MIGRATION-PAIRS", + passed: false, + status_code: "rollback_file_unreadable", + }), + ); + expect(malformedInputs.checks[2]?.detail).toContain("002-users.down.sql"); + expect(malformedInputs.checks[3]).toEqual( + expect.objectContaining({ + id: "PACKAGE-WIRING", + passed: false, + status_code: "package_json_invalid", + }), + ); + + const malformedRationale = await evaluateMigrationPairing({ + readDir: async () => [ + "001-init.sql", + "001-init.down.sql", + ], + readPackageJson: async () => PASSING_PACKAGE_JSON, + readTextFile: async () => "-- ok\n", + allowlistEntries: [ + { + migration: "001-init.sql", + rationale: "", + }, + ], + }); + + expect(malformedRationale.checks[1]).toEqual( + expect.objectContaining({ + id: "MIGRATION-ALLOWLIST-STATE", + passed: false, + status_code: "allowlist_rationale_invalid", + }), + ); + }); + + test("uses the same forward-file discovery rule as src/db/migrate.ts and ignores non-sql files plus .down.sql entries", async () => { + const report = await evaluateMigrationPairing({ + readDir: async () => [ + "001-init.sql", + "001-init.down.sql", + "002-users.sql", + "002-users.down.sql", + "notes.txt", + "nested", + ], + readPackageJson: async () => PASSING_PACKAGE_JSON, + readTextFile: async () => "-- ok\n", + allowlistEntries: [], + }); + + const pairCheck = report.checks.find((check) => check.id === "MIGRATION-PAIRS"); + expect(pairCheck?.passed).toBe(true); + expect(pairCheck?.detail).toContain("001-init.sql -> 001-init.down.sql"); + expect(pairCheck?.detail).toContain("002-users.sql -> 002-users.down.sql"); + expect(pairCheck?.detail).not.toContain("001-init.down.sql ->"); + expect(pairCheck?.detail).not.toContain("notes.txt"); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["check:migrations-have-downs"]).toBe( + "bun scripts/check-migrations-have-downs.ts", + ); + }); +}); diff --git a/scripts/check-migrations-have-downs.ts b/scripts/check-migrations-have-downs.ts new file mode 100644 index 00000000..0ca84e71 --- /dev/null +++ b/scripts/check-migrations-have-downs.ts @@ -0,0 +1,436 @@ +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; + +const COMMAND_NAME = "check:migrations-have-downs" as const; +const REPO_ROOT = path.resolve(import.meta.dir, ".."); +const MIGRATIONS_DIR = path.resolve(REPO_ROOT, "src/db/migrations"); +const PACKAGE_JSON_PATH = path.resolve(REPO_ROOT, "package.json"); +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/check-migrations-have-downs.ts"; + +export const CHECK_MIGRATIONS_HAVE_DOWNS_CHECK_IDS = [ + "MIGRATIONS-DIR-STATE", + "MIGRATION-ALLOWLIST-STATE", + "MIGRATION-PAIRS", + "PACKAGE-WIRING", +] as const; + +export type CheckMigrationsHaveDownsCheckId = + (typeof CHECK_MIGRATIONS_HAVE_DOWNS_CHECK_IDS)[number]; + +export type MigrationAllowlistEntry = { + migration: string; + rationale: string; +}; + +export type CheckerCheck = { + id: CheckMigrationsHaveDownsCheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type CheckerReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly CheckMigrationsHaveDownsCheckId[]; + overallPassed: boolean; + checks: CheckerCheck[]; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EvaluateOptions = { + generatedAt?: string; + allowlistEntries?: readonly MigrationAllowlistEntry[]; + readDir?: (dirPath: string) => Promise; + readTextFile?: (filePath: string) => Promise; + readPackageJson?: () => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +type FileSetResult = + | { ok: true; entries: string[] } + | { ok: false; error: unknown }; + +export const MIGRATION_PAIR_ALLOWLIST: readonly MigrationAllowlistEntry[] = []; + +export async function evaluateMigrationPairing( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const allowlistEntries = [...(options.allowlistEntries ?? MIGRATION_PAIR_ALLOWLIST)]; + const readDirImpl = options.readDir ?? defaultReadDir; + const readTextFile = options.readTextFile ?? defaultReadTextFile; + const readPackageJson = options.readPackageJson ?? defaultReadPackageJson; + + const dirState = await readMigrationDirState(readDirImpl); + const migrationFiles = dirState.ok ? [...dirState.entries].sort() : []; + const forwardMigrations = migrationFiles + .filter((fileName) => fileName.endsWith(".sql") && !fileName.endsWith(".down.sql")) + .sort(); + const forwardMigrationSet = new Set(forwardMigrations); + + const dirCheck = buildDirCheck(dirState, migrationFiles, forwardMigrations); + const allowlistCheck = buildAllowlistCheck(allowlistEntries, forwardMigrationSet); + const pairsCheck = await buildPairsCheck({ + dirState, + migrationFiles, + forwardMigrations, + allowlistEntries, + allowlistUsable: allowlistCheck.passed, + readTextFile, + }); + const packageCheck = await buildPackageCheck(readPackageJson); + + const checks = [dirCheck, allowlistCheck, pairsCheck, packageCheck]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: CHECK_MIGRATIONS_HAVE_DOWNS_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderCheckMigrationsHaveDownsReport(report: CheckerReport): string { + const lines = [ + "Migration rollback sibling gate", + `Generated at: ${report.generatedAt}`, + `Migration rollback sibling gate: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildCheckMigrationsHaveDownsHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: CheckerReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateMigrationPairing(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderCheckMigrationsHaveDownsReport(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`${COMMAND_NAME} failed: ${failingCodes}\n`); + } + + return { exitCode: report.overallPassed ? 0 : 1, report }; +} + +export function parseCheckMigrationsHaveDownsArgs( + args: readonly string[], +): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +function buildDirCheck( + dirState: FileSetResult, + migrationFiles: string[], + forwardMigrations: string[], +): CheckerCheck { + if (!dirState.ok) { + return failCheck( + "MIGRATIONS-DIR-STATE", + "migrations_dir_unreadable", + normalizeDetail(dirState.error), + ); + } + + return passCheck( + "MIGRATIONS-DIR-STATE", + "migrations_dir_ok", + `Scanned ${migrationFiles.length} entries and discovered ${forwardMigrations.length} forward migrations in ${normalizeRepoRelativePath(MIGRATIONS_DIR)}.`, + ); +} + +function buildAllowlistCheck( + allowlistEntries: readonly MigrationAllowlistEntry[], + forwardMigrationSet: ReadonlySet, +): CheckerCheck { + if (allowlistEntries.length === 0) { + return passCheck( + "MIGRATION-ALLOWLIST-STATE", + "allowlist_empty", + "No migration pairing exceptions are allowlisted.", + ); + } + + const seen = new Set(); + for (const entry of allowlistEntries) { + if (typeof entry.migration !== "string" || !entry.migration.endsWith(".sql")) { + return failCheck( + "MIGRATION-ALLOWLIST-STATE", + "allowlist_entry_invalid_migration", + `Allowlist migration entries must be forward *.sql filenames: ${JSON.stringify(entry)}`, + ); + } + + if (entry.migration.endsWith(".down.sql")) { + return failCheck( + "MIGRATION-ALLOWLIST-STATE", + "allowlist_entry_invalid_migration", + `Allowlist cannot target rollback files: ${entry.migration}`, + ); + } + + if (typeof entry.rationale !== "string" || entry.rationale.trim().length === 0) { + return failCheck( + "MIGRATION-ALLOWLIST-STATE", + "allowlist_rationale_invalid", + `Allowlist entry for ${entry.migration} must include a non-empty rationale.`, + ); + } + + if (seen.has(entry.migration)) { + return failCheck( + "MIGRATION-ALLOWLIST-STATE", + "allowlist_duplicate_migration", + `Allowlist must not repeat forward migrations: ${entry.migration}`, + ); + } + seen.add(entry.migration); + + if (!forwardMigrationSet.has(entry.migration)) { + return failCheck( + "MIGRATION-ALLOWLIST-STATE", + "allowlist_entry_missing_forward_migration", + `Allowlist references a nonexistent forward migration: ${entry.migration}`, + ); + } + } + + return passCheck( + "MIGRATION-ALLOWLIST-STATE", + "allowlist_entries_ok", + `Allowlisted migrations: ${allowlistEntries.map((entry) => `${entry.migration} (${entry.rationale.trim()})`).join(", ")}`, + ); +} + +async function buildPairsCheck({ + dirState, + migrationFiles, + forwardMigrations, + allowlistEntries, + allowlistUsable, + readTextFile, +}: { + dirState: FileSetResult; + migrationFiles: string[]; + forwardMigrations: string[]; + allowlistEntries: readonly MigrationAllowlistEntry[]; + allowlistUsable: boolean; + readTextFile: (filePath: string) => Promise; +}): Promise { + if (!dirState.ok) { + return failCheck( + "MIGRATION-PAIRS", + "migrations_scan_unavailable", + `Cannot evaluate rollback siblings because ${normalizeRepoRelativePath(MIGRATIONS_DIR)} could not be read: ${normalizeDetail(dirState.error)}`, + ); + } + + const allowlistSet = allowlistUsable + ? new Set(allowlistEntries.map((entry) => entry.migration)) + : new Set(); + const downFiles = new Set(migrationFiles.filter((fileName) => fileName.endsWith(".down.sql"))); + const resolvedPairs: string[] = []; + + for (const migration of forwardMigrations) { + const rollback = migration.replace(/\.sql$/, ".down.sql"); + + if (!downFiles.has(rollback)) { + if (allowlistSet.has(migration)) { + resolvedPairs.push(`${migration} -> allowlisted`); + continue; + } + + return failCheck( + "MIGRATION-PAIRS", + "rollback_missing", + `Missing rollback sibling for ${migration}: expected ${rollback}`, + ); + } + + try { + await readTextFile(path.resolve(MIGRATIONS_DIR, rollback)); + } catch (error) { + return failCheck( + "MIGRATION-PAIRS", + "rollback_file_unreadable", + `${normalizeRepoRelativePath(path.resolve(MIGRATIONS_DIR, rollback))} (${normalizeDetail(error)})`, + ); + } + + resolvedPairs.push(`${migration} -> ${rollback}`); + } + + return passCheck( + "MIGRATION-PAIRS", + "all_rollbacks_present", + resolvedPairs.length === 0 + ? "No forward migrations found." + : `Verified rollback siblings: ${resolvedPairs.join(", ")}`, + ); +} + +async function buildPackageCheck( + readPackageJson: () => Promise, +): Promise { + let packageJsonContent: string; + try { + packageJsonContent = await readPackageJson(); + } catch (error) { + return failCheck( + "PACKAGE-WIRING", + "package_file_unreadable", + normalizeDetail(error), + ); + } + + let packageJson: { scripts?: Record }; + try { + packageJson = JSON.parse(packageJsonContent) as { scripts?: Record }; + } catch (error) { + return failCheck("PACKAGE-WIRING", "package_json_invalid", error); + } + + const actualScript = packageJson.scripts?.[COMMAND_NAME]; + if (actualScript == null) { + return failCheck( + "PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT}`, + ); + } + + if (actualScript !== EXPECTED_PACKAGE_SCRIPT) { + return failCheck( + "PACKAGE-WIRING", + "package_wiring_incorrect", + `Expected scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT} but found ${actualScript}`, + ); + } + + return passCheck( + "PACKAGE-WIRING", + "package_wiring_ok", + `package.json wires ${COMMAND_NAME} to ${EXPECTED_PACKAGE_SCRIPT}`, + ); +} + +function passCheck( + id: CheckMigrationsHaveDownsCheckId, + status_code: string, + detail?: unknown, +): CheckerCheck { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck( + id: CheckMigrationsHaveDownsCheckId, + status_code: string, + detail?: unknown, +): CheckerCheck { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +function normalizeRepoRelativePath(filePath: string): string { + const relativePath = path.isAbsolute(filePath) + ? path.relative(REPO_ROOT, filePath) + : filePath; + return relativePath.split(path.sep).join("/"); +} + +async function readMigrationDirState( + readDirImpl: (dirPath: string) => Promise, +): Promise { + try { + const entries = await readDirImpl(MIGRATIONS_DIR); + return { ok: true, entries }; + } catch (error) { + return { ok: false, error }; + } +} + +async function defaultReadDir(dirPath: string): Promise { + return readdir(dirPath); +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +async function defaultReadPackageJson(): Promise { + return readFile(PACKAGE_JSON_PATH, "utf8"); +} + +if (import.meta.main) { + try { + const args = parseCheckMigrationsHaveDownsArgs(process.argv.slice(2)); + const { exitCode } = await buildCheckMigrationsHaveDownsHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${COMMAND_NAME} failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/check-orphaned-tests.test.ts b/scripts/check-orphaned-tests.test.ts new file mode 100644 index 00000000..be82b97c --- /dev/null +++ b/scripts/check-orphaned-tests.test.ts @@ -0,0 +1,185 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { OrphanedTestCheckerReport } from "./check-orphaned-tests.ts"; +import { + CHECK_ORPHANED_TESTS_CHECK_IDS, + EXPLICIT_TEST_TARGET_MAP, + buildCheckOrphanedTestsHarness, + evaluateOrphanedTests, + parseCheckOrphanedTestsArgs, + renderCheckOrphanedTestsReport, +} from "./check-orphaned-tests.ts"; + +const EXPECTED_CHECK_IDS = [ + "TRACKED-FILE-DISCOVERY", + "TARGET-MAP-STATE", + "ORPHANED-TESTS", + "PACKAGE-WIRING", +] as const; + +const PASSING_PACKAGE_JSON = JSON.stringify( + { + name: "kodiai", + scripts: { + "check:orphaned-tests": "bun scripts/check-orphaned-tests.ts", + }, + }, + null, + 2, +); + +describe("check orphaned tests", () => { + test("exports stable check ids, explicit mappings, and strict cli parsing", () => { + expect(CHECK_ORPHANED_TESTS_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(EXPLICIT_TEST_TARGET_MAP).toEqual({ + "scripts/deploy.test.ts": "deploy.sh", + "scripts/deploy-timeout-alignment.test.ts": "deploy.sh", + "src/slack/v1-safety-contract.test.ts": "src/slack/safety-rails.ts", + }); + expect(parseCheckOrphanedTestsArgs([])).toEqual({ json: false }); + expect(parseCheckOrphanedTestsArgs(["--json"])).toEqual({ json: true }); + expect(() => parseCheckOrphanedTestsArgs(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes when sibling ownership and explicit mappings resolve every tracked src/scripts test and package wiring is canonical", async () => { + const report = await evaluateOrphanedTests({ + generatedAt: "2026-04-21T11:00:00.000Z", + listTrackedFiles: async () => [ + "deploy.sh", + "scripts/check-orphaned-tests.ts", + "scripts/deploy.test.ts", + "scripts/deploy-timeout-alignment.test.ts", + "src/execution/executor.test.ts", + "src/execution/executor.ts", + "src/knowledge/retrieval.e2e.test.ts", + "src/knowledge/retrieval.ts", + "src/slack/safety-rails.ts", + "src/slack/v1-safety-contract.test.ts", + ], + readPackageJson: async () => PASSING_PACKAGE_JSON, + }); + + expect(report.command).toBe("check:orphaned-tests"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "TRACKED-FILE-DISCOVERY", + passed: true, + status_code: "tracked_files_ok", + }), + expect.objectContaining({ + id: "TARGET-MAP-STATE", + passed: true, + status_code: "target_map_ok", + }), + expect.objectContaining({ + id: "ORPHANED-TESTS", + passed: true, + status_code: "all_tests_resolved", + }), + expect.objectContaining({ + id: "PACKAGE-WIRING", + passed: true, + status_code: "package_wiring_ok", + }), + ]); + + const rendered = renderCheckOrphanedTestsReport(report); + expect(rendered).toContain("Orphaned test ownership gate: PASS"); + expect(rendered).toContain("TRACKED-FILE-DISCOVERY PASS"); + expect(rendered).toContain("TARGET-MAP-STATE PASS"); + expect(rendered).toContain("ORPHANED-TESTS PASS"); + expect(rendered).toContain("PACKAGE-WIRING PASS"); + }); + + test("fails with stable orphan and package status codes when a tracked test has no sibling and package wiring drifts", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildCheckOrphanedTestsHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + listTrackedFiles: async () => [ + "deploy.sh", + "scripts/check-orphaned-tests.ts", + "scripts/deploy.test.ts", + "scripts/deploy-timeout-alignment.test.ts", + "src/execution/missing-owner.test.ts", + "src/slack/safety-rails.ts", + "src/slack/v1-safety-contract.test.ts", + ], + readPackageJson: async () => JSON.stringify({ name: "kodiai", scripts: {} }), + }); + + const report = JSON.parse(stdout.join("")) as OrphanedTestCheckerReport; + + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "TRACKED-FILE-DISCOVERY", + passed: true, + status_code: "tracked_files_ok", + }), + expect.objectContaining({ + id: "TARGET-MAP-STATE", + passed: true, + status_code: "target_map_ok", + }), + expect.objectContaining({ + id: "ORPHANED-TESTS", + passed: false, + status_code: "orphaned_tests_found", + }), + expect.objectContaining({ + id: "PACKAGE-WIRING", + passed: false, + status_code: "package_wiring_missing", + }), + ]); + expect(report.checks[2]?.detail).toContain("src/execution/missing-owner.test.ts"); + expect(stderr.join(" ")).toContain("orphaned_tests_found"); + expect(stderr.join(" ")).toContain("package_wiring_missing"); + }); + + test("fails closed when explicit mappings drift away from tracked targets", async () => { + const report = await evaluateOrphanedTests({ + listTrackedFiles: async () => [ + "scripts/check-orphaned-tests.ts", + "scripts/deploy.test.ts", + "src/slack/safety-rails.ts", + ], + readPackageJson: async () => PASSING_PACKAGE_JSON, + }); + + expect(report.overallPassed).toBe(false); + expect(report.checks[1]).toEqual( + expect.objectContaining({ + id: "TARGET-MAP-STATE", + passed: false, + status_code: "mapped_target_missing", + }), + ); + expect(report.checks[1]?.detail).toContain("scripts/deploy-timeout-alignment.test.ts -> deploy.sh"); + expect(report.checks[1]?.detail).toContain("src/slack/v1-safety-contract.test.ts -> src/slack/safety-rails.ts"); + expect(report.checks[2]).toEqual( + expect.objectContaining({ + id: "ORPHANED-TESTS", + passed: false, + status_code: "ownership_map_unusable", + }), + ); + }); + + test("wires the canonical package script in the real package.json", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["check:orphaned-tests"]).toBe( + "bun scripts/check-orphaned-tests.ts", + ); + }); +}); diff --git a/scripts/check-orphaned-tests.ts b/scripts/check-orphaned-tests.ts new file mode 100644 index 00000000..35e11afd --- /dev/null +++ b/scripts/check-orphaned-tests.ts @@ -0,0 +1,425 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +const COMMAND_NAME = "check:orphaned-tests" as const; +const REPO_ROOT = path.resolve(import.meta.dir, ".."); +const PACKAGE_JSON_PATH = path.resolve(REPO_ROOT, "package.json"); +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/check-orphaned-tests.ts"; +const TRACKED_SCAN_PREFIXES = ["src/", "scripts/"] as const; + +export const CHECK_ORPHANED_TESTS_CHECK_IDS = [ + "TRACKED-FILE-DISCOVERY", + "TARGET-MAP-STATE", + "ORPHANED-TESTS", + "PACKAGE-WIRING", +] as const; + +export type OrphanedTestsCheckId = + (typeof CHECK_ORPHANED_TESTS_CHECK_IDS)[number]; + +export type OrphanedTestCheckerCheck = { + id: OrphanedTestsCheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type OrphanedTestCheckerReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly OrphanedTestsCheckId[]; + overallPassed: boolean; + checks: OrphanedTestCheckerCheck[]; +}; + +export const EXPLICIT_TEST_TARGET_MAP = { + "scripts/deploy.test.ts": "deploy.sh", + "scripts/deploy-timeout-alignment.test.ts": "deploy.sh", + "src/slack/v1-safety-contract.test.ts": "src/slack/safety-rails.ts", +} as const satisfies Record; + +const EXPLICIT_TARGET_PATHS = [...new Set(Object.values(EXPLICIT_TEST_TARGET_MAP))].sort(); +const EXPLICIT_TARGET_LOOKUP_PATHS = EXPLICIT_TARGET_PATHS.filter( + (targetPath) => !TRACKED_SCAN_PREFIXES.some((prefix) => targetPath.startsWith(prefix)), +); + +const EXPLICIT_TEST_TARGET_MAP_ENTRIES = Object.entries(EXPLICIT_TEST_TARGET_MAP).sort( + ([left], [right]) => left.localeCompare(right), +); + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EvaluateOptions = { + generatedAt?: string; + listTrackedFiles?: () => Promise; + readPackageJson?: () => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +type FileSetResult = + | { ok: true; entries: string[] } + | { ok: false; error: unknown }; + +export async function evaluateOrphanedTests( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const listTrackedFiles = options.listTrackedFiles ?? defaultListTrackedFiles; + const readPackageJson = options.readPackageJson ?? defaultReadPackageJson; + + const trackedFileState = await readTrackedFileState(listTrackedFiles); + const trackedFiles = trackedFileState.ok ? [...trackedFileState.entries].sort() : []; + const trackedFileSet = new Set(trackedFiles); + const trackedTestFiles = trackedFiles.filter(isTrackedTestPath).sort(); + + const discoveryCheck = buildTrackedFileDiscoveryCheck(trackedFileState, trackedFiles, trackedTestFiles); + const targetMapCheck = buildTargetMapCheck(trackedFileSet); + const orphanCheck = buildOrphanedTestsCheck({ + trackedFileState, + trackedFileSet, + trackedTestFiles, + targetMapUsable: targetMapCheck.passed, + }); + const packageCheck = await buildPackageCheck(readPackageJson); + + const checks = [discoveryCheck, targetMapCheck, orphanCheck, packageCheck]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: CHECK_ORPHANED_TESTS_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderCheckOrphanedTestsReport( + report: OrphanedTestCheckerReport, +): string { + const lines = [ + "Orphaned test ownership gate", + `Generated at: ${report.generatedAt}`, + `Orphaned test ownership gate: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildCheckOrphanedTestsHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: OrphanedTestCheckerReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateOrphanedTests(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderCheckOrphanedTestsReport(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`${COMMAND_NAME} failed: ${failingCodes}\n`); + } + + return { exitCode: report.overallPassed ? 0 : 1, report }; +} + +export function parseCheckOrphanedTestsArgs( + args: readonly string[], +): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +function buildTrackedFileDiscoveryCheck( + trackedFileState: FileSetResult, + trackedFiles: string[], + trackedTestFiles: string[], +): OrphanedTestCheckerCheck { + if (!trackedFileState.ok) { + return failCheck( + "TRACKED-FILE-DISCOVERY", + "tracked_files_unreadable", + normalizeDetail(trackedFileState.error), + ); + } + + return passCheck( + "TRACKED-FILE-DISCOVERY", + "tracked_files_ok", + `Scanned ${trackedFiles.length} tracked files under ${TRACKED_SCAN_PREFIXES.join(", ")} and discovered ${trackedTestFiles.length} tracked test suites.`, + ); +} + +function buildTargetMapCheck( + trackedFileSet: ReadonlySet, +): OrphanedTestCheckerCheck { + const missingMappings: string[] = []; + + for (const [testPath, targetPath] of EXPLICIT_TEST_TARGET_MAP_ENTRIES) { + if (!trackedFileSet.has(testPath) || !trackedFileSet.has(targetPath)) { + missingMappings.push(`${testPath} -> ${targetPath}`); + } + } + + if (missingMappings.length > 0) { + return failCheck( + "TARGET-MAP-STATE", + "mapped_target_missing", + `Explicit ownership mappings must resolve tracked tests and targets: ${missingMappings.join(", ")}`, + ); + } + + return passCheck( + "TARGET-MAP-STATE", + "target_map_ok", + `Verified explicit ownership mappings: ${EXPLICIT_TEST_TARGET_MAP_ENTRIES.map(([testPath, targetPath]) => `${testPath} -> ${targetPath}`).join(", ")}`, + ); +} + +function buildOrphanedTestsCheck({ + trackedFileState, + trackedFileSet, + trackedTestFiles, + targetMapUsable, +}: { + trackedFileState: FileSetResult; + trackedFileSet: ReadonlySet; + trackedTestFiles: string[]; + targetMapUsable: boolean; +}): OrphanedTestCheckerCheck { + if (!trackedFileState.ok) { + return failCheck( + "ORPHANED-TESTS", + "test_scan_unavailable", + `Cannot evaluate orphaned tests because tracked-file discovery failed: ${normalizeDetail(trackedFileState.error)}`, + ); + } + + if (!targetMapUsable) { + return failCheck( + "ORPHANED-TESTS", + "ownership_map_unusable", + "Cannot evaluate orphaned tests until explicit target mappings resolve tracked tests and targets.", + ); + } + + const resolvedPairs: string[] = []; + const orphanedTests: string[] = []; + + for (const testPath of trackedTestFiles) { + const targetPath = resolveOwnedTarget(testPath); + if (!trackedFileSet.has(targetPath)) { + orphanedTests.push(`${testPath} -> ${targetPath}`); + continue; + } + resolvedPairs.push(`${testPath} -> ${targetPath}`); + } + + if (orphanedTests.length > 0) { + return failCheck( + "ORPHANED-TESTS", + "orphaned_tests_found", + `Tracked tests without an owning target: ${orphanedTests.join(", ")}`, + ); + } + + return passCheck( + "ORPHANED-TESTS", + "all_tests_resolved", + resolvedPairs.length === 0 + ? "No tracked src/scripts test suites found." + : `Resolved tracked test ownership: ${resolvedPairs.join(", ")}`, + ); +} + +async function buildPackageCheck( + readPackageJson: () => Promise, +): Promise { + let packageJsonContent: string; + try { + packageJsonContent = await readPackageJson(); + } catch (error) { + return failCheck( + "PACKAGE-WIRING", + "package_file_unreadable", + normalizeDetail(error), + ); + } + + let packageJson: { scripts?: Record }; + try { + packageJson = JSON.parse(packageJsonContent) as { scripts?: Record }; + } catch (error) { + return failCheck("PACKAGE-WIRING", "package_json_invalid", error); + } + + const actualScript = packageJson.scripts?.[COMMAND_NAME]; + if (actualScript == null) { + return failCheck( + "PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT}`, + ); + } + + if (actualScript !== EXPECTED_PACKAGE_SCRIPT) { + return failCheck( + "PACKAGE-WIRING", + "package_wiring_incorrect", + `Expected scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT} but found ${actualScript}`, + ); + } + + return passCheck( + "PACKAGE-WIRING", + "package_wiring_ok", + `package.json wires ${COMMAND_NAME} to ${EXPECTED_PACKAGE_SCRIPT}`, + ); +} + +function resolveOwnedTarget(testPath: string): string { + const explicitTarget = EXPLICIT_TEST_TARGET_MAP[testPath as keyof typeof EXPLICIT_TEST_TARGET_MAP]; + if (explicitTarget) { + return explicitTarget; + } + + if (testPath.endsWith(".e2e.test.ts")) { + return testPath.replace(/\.e2e\.test\.ts$/, ".ts"); + } + + return testPath.replace(/\.test\.ts$/, ".ts"); +} + +function isTrackedTestPath(filePath: string): boolean { + return ( + TRACKED_SCAN_PREFIXES.some((prefix) => filePath.startsWith(prefix)) && + (filePath.endsWith(".test.ts") || filePath.endsWith(".e2e.test.ts")) + ); +} + +function passCheck( + id: OrphanedTestsCheckId, + status_code: string, + detail?: unknown, +): OrphanedTestCheckerCheck { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck( + id: OrphanedTestsCheckId, + status_code: string, + detail?: unknown, +): OrphanedTestCheckerCheck { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +async function readTrackedFileState( + listTrackedFiles: () => Promise, +): Promise { + try { + const entries = await listTrackedFiles(); + return { ok: true, entries }; + } catch (error) { + return { ok: false, error }; + } +} + +async function defaultListTrackedFiles(): Promise { + const process = Bun.spawn([ + "git", + "ls-files", + "--", + ...TRACKED_SCAN_PREFIXES, + ...EXPLICIT_TARGET_LOOKUP_PATHS, + ], { + cwd: REPO_ROOT, + stdout: "pipe", + stderr: "pipe", + }); + + const [stdoutText, stderrText, exitCode] = await Promise.all([ + new Response(process.stdout).text(), + new Response(process.stderr).text(), + process.exited, + ]); + + if (exitCode !== 0) { + throw new Error(stderrText.trim() || `git ls-files exited with code ${exitCode}`); + } + + return stdoutText + .split(/\r?\n/) + .map((entry) => entry.trim()) + .filter(Boolean) + .sort(); +} + +async function defaultReadPackageJson(): Promise { + return readFile(PACKAGE_JSON_PATH, "utf8"); +} + +if (import.meta.main) { + try { + const args = parseCheckOrphanedTestsArgs(process.argv.slice(2)); + const { exitCode } = await buildCheckOrphanedTestsHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${COMMAND_NAME} failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/cleanup-wiki-issue.ts b/scripts/cleanup-wiki-issue.ts index 1d185964..27fa457d 100644 --- a/scripts/cleanup-wiki-issue.ts +++ b/scripts/cleanup-wiki-issue.ts @@ -164,7 +164,6 @@ async function main() { slackKodiaiChannelId: "unused", slackDefaultRepo: "unused", slackAssistantModel: "unused", - slackWebhookRelaySources: [], port: 3000, logLevel: "info", botAllowList: [], diff --git a/scripts/publish-wiki-updates.ts b/scripts/publish-wiki-updates.ts index 5d4db029..a525cfdc 100644 --- a/scripts/publish-wiki-updates.ts +++ b/scripts/publish-wiki-updates.ts @@ -154,7 +154,6 @@ async function main() { slackKodiaiChannelId: "unused", slackDefaultRepo: "xbmc/xbmc", slackAssistantModel: "unused", - slackWebhookRelaySources: [], port: 0, logLevel: process.env.LOG_LEVEL ?? "info", botAllowList: [], diff --git a/scripts/sync-triage-reactions.ts b/scripts/sync-triage-reactions.ts index cb5da178..f6c56eb4 100644 --- a/scripts/sync-triage-reactions.ts +++ b/scripts/sync-triage-reactions.ts @@ -135,7 +135,6 @@ async function main(): Promise { slackKodiaiChannelId: "unused", slackDefaultRepo: "xbmc/xbmc", slackAssistantModel: "unused", - slackWebhookRelaySources: [], port: 0, logLevel: "info", botAllowList: [], diff --git a/scripts/verify-m027-s03.test.ts b/scripts/verify-m027-s03.test.ts index 8446724f..499599de 100644 --- a/scripts/verify-m027-s03.test.ts +++ b/scripts/verify-m027-s03.test.ts @@ -117,7 +117,7 @@ function makeRepairReport(overrides: Partial = {}): RepairCliRe success: true, status_code: "repair_completed", corpus: "review_comments", - target_model: "voyage-code-3", + target_model: "voyage-4", resumed: false, dry_run: false, run: { @@ -149,7 +149,7 @@ function makeNoopProbeReport(overrides: Partial = {}): RepairCl success: true, status_code: "repair_not_needed", corpus: "issues", - target_model: "voyage-code-3", + target_model: "voyage-4", resumed: false, dry_run: true, run: { @@ -184,8 +184,8 @@ function makeAuditReport(overrides: Partial = {}): AuditReport { corpus: "review_comments", status: "pass", severity: "info", - expected_model: "voyage-code-3", - actual_models: ["voyage-code-3"], + expected_model: "voyage-4", + actual_models: ["voyage-4"], model_mismatch: 0, missing_or_null: 0, }, @@ -193,8 +193,8 @@ function makeAuditReport(overrides: Partial = {}): AuditReport { corpus: "issues", status: "pass", severity: "info", - expected_model: "voyage-code-3", - actual_models: ["voyage-code-3"], + expected_model: "voyage-4", + actual_models: ["voyage-4"], model_mismatch: 0, missing_or_null: 0, }, @@ -314,8 +314,8 @@ describe("slice proof harness contract for scripts/verify-m027-s03.ts", () => { corpus: "review_comments", status: "fail", severity: "critical", - expected_model: "voyage-code-3", - actual_models: ["voyage-code-3"], + expected_model: "voyage-4", + actual_models: ["voyage-4"], model_mismatch: 0, missing_or_null: 3, }, @@ -323,8 +323,8 @@ describe("slice proof harness contract for scripts/verify-m027-s03.ts", () => { corpus: "issues", status: "pass", severity: "info", - expected_model: "voyage-code-3", - actual_models: ["voyage-code-3"], + expected_model: "voyage-4", + actual_models: ["voyage-4"], model_mismatch: 0, missing_or_null: 0, }, @@ -440,8 +440,8 @@ describe("slice proof harness contract for scripts/verify-m027-s03.ts", () => { corpus: "review_comments", status: "fail", severity: "critical", - expected_model: "voyage-code-3", - actual_models: ["voyage-code-3"], + expected_model: "voyage-4", + actual_models: ["voyage-4"], model_mismatch: 0, missing_or_null: 1, }, @@ -449,8 +449,8 @@ describe("slice proof harness contract for scripts/verify-m027-s03.ts", () => { corpus: "issues", status: "pass", severity: "info", - expected_model: "voyage-code-3", - actual_models: ["voyage-code-3"], + expected_model: "voyage-4", + actual_models: ["voyage-4"], model_mismatch: 0, missing_or_null: 0, }, diff --git a/scripts/verify-m029-s04.ts b/scripts/verify-m029-s04.ts index d31d6a87..07e4aee6 100644 --- a/scripts/verify-m029-s04.ts +++ b/scripts/verify-m029-s04.ts @@ -430,10 +430,13 @@ export async function buildM029S04ProofHarness(opts?: { const stdout = opts?.stdout ?? process.stdout; const stderr = opts?.stderr ?? process.stderr; const useJson = opts?.json ?? false; + const hasInjectedSql = Boolean(opts && Object.prototype.hasOwnProperty.call(opts, "sql")); + const hasInjectedOctokit = Boolean(opts && Object.prototype.hasOwnProperty.call(opts, "octokit")); + const shouldAutoDiscover = !opts || Object.keys(opts).every((key) => key === "json"); // Try DB connection if not injected let sql: unknown = opts?.sql; - if (sql === undefined) { + if (shouldAutoDiscover && !hasInjectedSql && sql === undefined) { try { const dbUrl = process.env.DATABASE_URL; if (dbUrl) { @@ -453,7 +456,7 @@ export async function buildM029S04ProofHarness(opts?: { // Try GitHub auth if not injected and env vars are present let octokit: unknown = opts?.octokit; - if (octokit === undefined) { + if (shouldAutoDiscover && !hasInjectedOctokit && octokit === undefined) { try { const appId = process.env.GITHUB_APP_ID; const privateKeyEnv = process.env.GITHUB_PRIVATE_KEY ?? process.env.GITHUB_PRIVATE_KEY_BASE64; @@ -482,7 +485,6 @@ export async function buildM029S04ProofHarness(opts?: { slackKodiaiChannelId: "unused", slackDefaultRepo: "unused", slackAssistantModel: "unused", - slackWebhookRelaySources: [], port: 3000, logLevel: "silent", botAllowList: [], diff --git a/scripts/verify-m044-s01.ts b/scripts/verify-m044-s01.ts index 5d0c4f52..475852dc 100644 --- a/scripts/verify-m044-s01.ts +++ b/scripts/verify-m044-s01.ts @@ -148,7 +148,6 @@ function buildGitHubAppConfig(repo: string, githubPrivateKey: string) { slackKodiaiChannelId: "unused", slackDefaultRepo: repo, slackAssistantModel: "unused", - slackWebhookRelaySources: [], port: 0, logLevel: "info", botAllowList: [], diff --git a/scripts/verify-m049-s02.ts b/scripts/verify-m049-s02.ts index b8ae7c36..fa3d891f 100644 --- a/scripts/verify-m049-s02.ts +++ b/scripts/verify-m049-s02.ts @@ -235,7 +235,6 @@ function buildGitHubAppConfig(repo: string, githubPrivateKey: string) { slackKodiaiChannelId: "unused", slackDefaultRepo: repo, slackAssistantModel: "unused", - slackWebhookRelaySources: [], port: 0, logLevel: "info", botAllowList: [], diff --git a/scripts/verify-m053.test.ts b/scripts/verify-m053.test.ts new file mode 100644 index 00000000..69253699 --- /dev/null +++ b/scripts/verify-m053.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport } from "./verify-m053.ts"; +import { + M053_CHECK_IDS, + buildM053ProofHarness, + evaluateM053, + parseM053Args, + renderM053Report, +} from "./verify-m053.ts"; + +const EXPECTED_CHECK_IDS = [ + "M053-HELPER-REMOVED", + "M053-SRC-NEW-FUNCTION-CLEAN", + "M053-DECISION-RECORDED", +] as const; + +describe("verify m053 invariant proof harness", () => { + test("exports stable check ids and cli parsing", () => { + expect(M053_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM053Args([])).toEqual({ json: false }); + expect(parseM053Args(["--json"])).toEqual({ json: true }); + expect(() => parseM053Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes when helper is absent, src tree is clean, and the decision record is present", async () => { + const report = await evaluateM053({ + generatedAt: "2026-04-21T04:30:00.000Z", + pathExists: async () => false, + walkFiles: async () => [ + "/repo/src/a.ts", + "/repo/src/contributor/example.test.ts", + ], + readTextFile: async (filePath: string) => { + if (filePath.endsWith(".gsd/DECISIONS.md")) { + return "| D999 | M053/S01/T02 | security | M053 src-tree no-dynamic-evaluator invariant | choice | rationale | Yes | agent |"; + } + return "export const ok = true;"; + }, + }); + + expect(report.command).toBe("verify:m053"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M053-HELPER-REMOVED", + passed: true, + status_code: "removed_helper_absent", + }), + expect.objectContaining({ + id: "M053-SRC-NEW-FUNCTION-CLEAN", + passed: true, + status_code: "src_tree_no_new_function", + }), + expect.objectContaining({ + id: "M053-DECISION-RECORDED", + passed: true, + status_code: "decision_record_present", + }), + ]); + + const rendered = renderM053Report(report); + expect(rendered).toContain("Proof surface: PASS"); + expect(rendered).toContain("M053-HELPER-REMOVED PASS"); + expect(rendered).toContain("M053-SRC-NEW-FUNCTION-CLEAN PASS"); + expect(rendered).toContain("M053-DECISION-RECORDED PASS"); + }); + + test("fails with named status codes when the helper reappears, src tree regresses, or the decision is missing", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM053ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + pathExists: async () => true, + walkFiles: async () => ["/repo/src/bad.ts"], + readTextFile: async (filePath: string) => { + if (filePath.endsWith(".gsd/DECISIONS.md")) { + return "# Decisions Register\n"; + } + return 'const importModule = new Function("specifier", "return import(specifier)");'; + }, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M053-HELPER-REMOVED", + passed: false, + status_code: "removed_helper_present", + }), + expect.objectContaining({ + id: "M053-SRC-NEW-FUNCTION-CLEAN", + passed: false, + status_code: "src_tree_contains_new_function", + }), + expect.objectContaining({ + id: "M053-DECISION-RECORDED", + passed: false, + status_code: "decision_record_missing", + }), + ]); + expect(report.checks[1]?.detail).toContain("src/bad.ts"); + expect(stderr.join(" ")).toContain("removed_helper_present"); + expect(stderr.join(" ")).toContain("src_tree_contains_new_function"); + expect(stderr.join(" ")).toContain("decision_record_missing"); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m053"]).toBe("bun scripts/verify-m053.ts"); + }); +}); diff --git a/scripts/verify-m053.ts b/scripts/verify-m053.ts new file mode 100644 index 00000000..bbacd93d --- /dev/null +++ b/scripts/verify-m053.ts @@ -0,0 +1,288 @@ +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; + +const COMMAND_NAME = "verify:m053" as const; +const SRC_ROOT = path.resolve(import.meta.dir, "../src"); +const REMOVED_HELPER_PATH = "src/phase28-inline-minconfidence-live-check.ts" as const; +const DECISIONS_PATH = path.resolve(import.meta.dir, "../.gsd/DECISIONS.md"); +const DECISION_MARKER = "M053/S01/T02" as const; +const DECISION_TEXT_MARKER = "M053 src-tree no-dynamic-evaluator invariant" as const; + +export const M053_CHECK_IDS = [ + "M053-HELPER-REMOVED", + "M053-SRC-NEW-FUNCTION-CLEAN", + "M053-DECISION-RECORDED", +] as const; + +export type M053CheckId = (typeof M053_CHECK_IDS)[number]; + +export type Check = { + id: M053CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M053CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EvaluateOptions = { + generatedAt?: string; + readTextFile?: (filePath: string) => Promise; + pathExists?: (filePath: string) => Promise; + walkFiles?: (dirPath: string) => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +export async function evaluateM053( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const readTextFile = options.readTextFile ?? defaultReadTextFile; + const pathExists = options.pathExists ?? defaultPathExists; + const walkFiles = options.walkFiles ?? walkDirectoryFiles; + + const helperCheck = await buildHelperRemovedCheck(pathExists); + const srcTreeCheck = await buildSrcTreeCheck({ walkFiles, readTextFile }); + const decisionCheck = await buildDecisionRecordedCheck(readTextFile); + + const checks = [helperCheck, srcTreeCheck, decisionCheck]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M053_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM053Report(report: EvaluationReport): string { + const lines = [ + "M053 invariant proof harness: no src-side dynamic evaluators", + `Generated at: ${report.generatedAt}`, + `Proof surface: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM053ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM053(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM053Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`verify:m053 failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM053Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +async function buildHelperRemovedCheck( + pathExists: (filePath: string) => Promise, +): Promise { + const helperExists = await pathExists(path.resolve(import.meta.dir, `../${REMOVED_HELPER_PATH}`)); + + return helperExists + ? failCheck( + "M053-HELPER-REMOVED", + "removed_helper_present", + `${REMOVED_HELPER_PATH} still exists.`, + ) + : passCheck( + "M053-HELPER-REMOVED", + "removed_helper_absent", + `${REMOVED_HELPER_PATH} is absent.`, + ); +} + +async function buildSrcTreeCheck(params: { + walkFiles: (dirPath: string) => Promise; + readTextFile: (filePath: string) => Promise; +}): Promise { + try { + const files = await params.walkFiles(SRC_ROOT); + const offenders: string[] = []; + + for (const filePath of files) { + const content = await params.readTextFile(filePath); + if (content.includes("new Function(")) { + offenders.push(path.relative(path.resolve(import.meta.dir, ".."), filePath)); + } + } + + return offenders.length === 0 + ? passCheck( + "M053-SRC-NEW-FUNCTION-CLEAN", + "src_tree_no_new_function", + `scanned ${files.length} src files`, + ) + : failCheck( + "M053-SRC-NEW-FUNCTION-CLEAN", + "src_tree_contains_new_function", + `new Function() found in: ${offenders.join(", ")}`, + ); + } catch (error) { + return failCheck( + "M053-SRC-NEW-FUNCTION-CLEAN", + "src_tree_scan_failed", + normalizeDetail(error), + ); + } +} + +async function buildDecisionRecordedCheck( + readTextFile: (filePath: string) => Promise, +): Promise { + try { + const content = await readTextFile(DECISIONS_PATH); + const hasDecision = + content.includes(DECISION_MARKER) && content.includes(DECISION_TEXT_MARKER); + + return hasDecision + ? passCheck( + "M053-DECISION-RECORDED", + "decision_record_present", + `Found ${DECISION_MARKER} decision marker in .gsd/DECISIONS.md.`, + ) + : failCheck( + "M053-DECISION-RECORDED", + "decision_record_missing", + `Missing ${DECISION_MARKER} / ${DECISION_TEXT_MARKER} entry in .gsd/DECISIONS.md.`, + ); + } catch (error) { + return failCheck( + "M053-DECISION-RECORDED", + "decision_record_unreadable", + normalizeDetail(error), + ); + } +} + +function passCheck(id: M053CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M053CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +async function defaultPathExists(filePath: string): Promise { + try { + await readFile(filePath); + return true; + } catch { + return false; + } +} + +async function walkDirectoryFiles(dirPath: string): Promise { + const entries = await readdir(dirPath, { withFileTypes: true }); + const files: string[] = []; + + for (const entry of entries) { + const entryPath = path.join(dirPath, entry.name); + if (entry.isDirectory()) { + files.push(...(await walkDirectoryFiles(entryPath))); + continue; + } + if (entry.isFile()) { + files.push(entryPath); + } + } + + files.sort((left, right) => left.localeCompare(right)); + return files; +} + +if (import.meta.main) { + try { + const args = parseM053Args(process.argv.slice(2)); + const { exitCode } = await buildM053ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`verify:m053 failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m054-s01.test.ts b/scripts/verify-m054-s01.test.ts new file mode 100644 index 00000000..d1747a06 --- /dev/null +++ b/scripts/verify-m054-s01.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport } from "./verify-m054-s01.ts"; +import { + M054_S01_CHECK_IDS, + buildM054S01ProofHarness, + evaluateM054S01QueueTruth, + parseM054S01Args, + renderM054S01Report, +} from "./verify-m054-s01.ts"; + +const EXPECTED_CHECK_IDS = [ + "M054-S01-PENDING-QUEUE-MEMBERSHIP", + "M054-S01-NOT-PENDING-REDIRECT", + "M054-S01-PROJECT-SHIPPED-MILESTONE-ALIGNMENT", +] as const; + +describe("verify m054 s01 queue truth harness", () => { + test("exports stable check ids and cli parsing", () => { + expect(M054_S01_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM054S01Args([])).toEqual({ json: false }); + expect(parseM054S01Args(["--json"])).toEqual({ json: true }); + expect(() => parseM054S01Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes for the current compact queue format and shipped milestone redirect contract", async () => { + const report = await evaluateM054S01QueueTruth({ + generatedAt: "2026-04-21T04:40:00.000Z", + readTextFile: async (filePath: string) => { + if (filePath.endsWith("QUEUE.md")) { + return `# Queue\n\n## Pending Milestones\n\n### M027 — Embedding Integrity\n- GitHub backlog: \`#91\`\n\n### M028 — Wiki Modification\n- GitHub backlog: \`#91\`\n\n### M029 — Wiki Generation\n- GitHub backlog: \`#91\`\n\n### M031 — Security Hardening\n- GitHub backlog: \`#91\`\n\n### M032 — Agent Process Isolation\n- GitHub backlog: \`#91\`\n\n### M053 — Unsafe new Function Removal\n- GitHub backlog: \`#92\`\n\n### M054 — Queue Repair\n- GitHub backlog: \`#93\`\n\n### M055 — Docs Pass\n- GitHub backlog: \`#94\`\n\n### M056 — Rollback Completeness\n- GitHub backlog: \`#95\`\n\n### M057 — Test Backfill\n- GitHub backlog: \`#96\`\n\n### M058 — CI Hardening\n- GitHub backlog: \`#97\`\n\n### M059 — Script Registry\n- GitHub backlog: \`#98\`\n\n### M060 — Knowledge Tests\n- GitHub backlog: \`#99\`\n\n## Not Pending\n\nCompleted milestones are tracked in \`.gsd/PROJECT.md\`. M044 through M052 are complete and are intentionally omitted from the pending queue.\n`; + } + + return `# Kodiai\n\nMilestones M043, M044, M045, M046, M047, M051, M052, and **M053** are complete.\n`; + }, + }); + + expect(report.command).toBe("verify:m054:s01"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M054-S01-PENDING-QUEUE-MEMBERSHIP", + passed: true, + status_code: "pending_queue_membership_ok", + }), + expect.objectContaining({ + id: "M054-S01-NOT-PENDING-REDIRECT", + passed: true, + status_code: "not_pending_redirect_present", + }), + expect.objectContaining({ + id: "M054-S01-PROJECT-SHIPPED-MILESTONE-ALIGNMENT", + passed: true, + status_code: "project_shipped_alignment_ok", + }), + ]); + + const rendered = renderM054S01Report(report); + expect(rendered).toContain("Queue truth proof surface: PASS"); + expect(rendered).toContain("M054-S01-PENDING-QUEUE-MEMBERSHIP PASS"); + expect(rendered).toContain("M054-S01-NOT-PENDING-REDIRECT PASS"); + expect(rendered).toContain("M054-S01-PROJECT-SHIPPED-MILESTONE-ALIGNMENT PASS"); + }); + + test("fails with named status codes for missing queue entries, stale pending milestones, and missing project redirect guidance", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM054S01ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("QUEUE.md")) { + return `# Queue\n\n## Pending Milestones\n\n### M027 — Embedding Integrity\n### M028 — Wiki Modification\n### M029 — Wiki Generation\n### M031 — Security Hardening\n### M032 — Agent Process Isolation\n### M053 — Unsafe new Function Removal\n### M054 — Queue Repair\n### M055 — Docs Pass\n### M056 — Rollback Completeness\n### M057 — Test Backfill\n### M058 — CI Hardening\n### M059 — Script Registry\n### M044 — Already Complete\n\n## Not Pending\n\nCompleted milestones live elsewhere.\n`; + } + + return `# Kodiai\n\nMilestones M043, M044, M045, M046, M047, M051, M052, and **M053** are complete.\n`; + }, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M054-S01-PENDING-QUEUE-MEMBERSHIP", + passed: false, + status_code: "pending_queue_membership_mismatch", + }), + expect.objectContaining({ + id: "M054-S01-NOT-PENDING-REDIRECT", + passed: false, + status_code: "not_pending_redirect_missing_project_reference", + }), + expect.objectContaining({ + id: "M054-S01-PROJECT-SHIPPED-MILESTONE-ALIGNMENT", + passed: false, + status_code: "project_shipped_milestone_still_pending", + }), + ]); + expect(report.checks[0]?.detail).toContain("missing: M060"); + expect(report.checks[0]?.detail).toContain("unexpected: M044"); + expect(report.checks[2]?.detail).toContain("M044"); + expect(stderr.join(" ")).toContain("pending_queue_membership_mismatch"); + expect(stderr.join(" ")).toContain("not_pending_redirect_missing_project_reference"); + expect(stderr.join(" ")).toContain("project_shipped_milestone_still_pending"); + }); + + test("surfaces stable malformed-input and unreadable-file failures", async () => { + const missingSection = await evaluateM054S01QueueTruth({ + readTextFile: async () => "# Queue\n\nNo pending milestones section here.\n", + }); + expect(missingSection.checks[0]).toEqual( + expect.objectContaining({ + id: "M054-S01-PENDING-QUEUE-MEMBERSHIP", + passed: false, + status_code: "pending_section_missing", + }), + ); + + const unreadableProject = await evaluateM054S01QueueTruth({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("QUEUE.md")) { + return `# Queue\n\n## Pending Milestones\n\n### M027 — Embedding Integrity\n### M028 — Wiki Modification\n### M029 — Wiki Generation\n### M031 — Security Hardening\n### M032 — Agent Process Isolation\n### M053 — Unsafe new Function Removal\n### M054 — Queue Repair\n### M055 — Docs Pass\n### M056 — Rollback Completeness\n### M057 — Test Backfill\n### M058 — CI Hardening\n### M059 — Script Registry\n### M060 — Knowledge Tests\n\n## Not Pending\n\nCompleted milestones are tracked in \`.gsd/PROJECT.md\`.\n`; + } + + throw new Error("EACCES: PROJECT.md"); + }, + }); + + expect(unreadableProject.checks[2]).toEqual( + expect.objectContaining({ + id: "M054-S01-PROJECT-SHIPPED-MILESTONE-ALIGNMENT", + passed: false, + status_code: "project_file_unreadable", + }), + ); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m054:s01"]).toBe( + "bun scripts/verify-m054-s01.ts", + ); + }); +}); diff --git a/scripts/verify-m054-s01.ts b/scripts/verify-m054-s01.ts new file mode 100644 index 00000000..1461eaa3 --- /dev/null +++ b/scripts/verify-m054-s01.ts @@ -0,0 +1,396 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +const COMMAND_NAME = "verify:m054:s01" as const; +const QUEUE_PATH = path.resolve(import.meta.dir, "../.gsd/QUEUE.md"); +const PROJECT_PATH = path.resolve(import.meta.dir, "../.gsd/PROJECT.md"); +const EXPECTED_PENDING_MILESTONES = [ + "M027", + "M028", + "M029", + "M031", + "M032", + "M053", + "M054", + "M055", + "M056", + "M057", + "M058", + "M059", + "M060", +] as const; +const CLEARLY_SHIPPED_MILESTONES = [ + "M044", + "M045", + "M046", + "M047", + "M048", + "M049", + "M050", + "M051", + "M052", +] as const; + +export const M054_S01_CHECK_IDS = [ + "M054-S01-PENDING-QUEUE-MEMBERSHIP", + "M054-S01-NOT-PENDING-REDIRECT", + "M054-S01-PROJECT-SHIPPED-MILESTONE-ALIGNMENT", +] as const; + +export type M054S01CheckId = (typeof M054_S01_CHECK_IDS)[number]; + +export type Check = { + id: M054S01CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M054S01CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EvaluateOptions = { + generatedAt?: string; + readTextFile?: (filePath: string) => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +export async function evaluateM054S01QueueTruth( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const readTextFile = options.readTextFile ?? defaultReadTextFile; + + let queueContent: string | null = null; + let queueReadError: unknown = null; + + try { + queueContent = await readTextFile(QUEUE_PATH); + } catch (error) { + queueReadError = error; + } + + const queueMembershipCheck = + queueContent == null + ? failCheck( + "M054-S01-PENDING-QUEUE-MEMBERSHIP", + "queue_file_unreadable", + queueReadError, + ) + : buildPendingQueueMembershipCheck(queueContent); + + const redirectCheck = + queueContent == null + ? failCheck( + "M054-S01-NOT-PENDING-REDIRECT", + "queue_file_unreadable", + queueReadError, + ) + : buildNotPendingRedirectCheck(queueContent); + + const projectAlignmentCheck = + queueContent == null + ? failCheck( + "M054-S01-PROJECT-SHIPPED-MILESTONE-ALIGNMENT", + "queue_file_unreadable", + queueReadError, + ) + : await buildProjectAlignmentCheck(queueContent, readTextFile); + + const checks = [queueMembershipCheck, redirectCheck, projectAlignmentCheck]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M054_S01_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM054S01Report(report: EvaluationReport): string { + const lines = [ + "M054 S01 queue truth verifier", + `Generated at: ${report.generatedAt}`, + `Queue truth proof surface: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM054S01ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM054S01QueueTruth(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM054S01Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`verify:m054:s01 failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM054S01Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +function buildPendingQueueMembershipCheck(queueContent: string): Check { + const pendingSection = extractSection(queueContent, "Pending Milestones", "Not Pending"); + + if (pendingSection == null) { + return failCheck( + "M054-S01-PENDING-QUEUE-MEMBERSHIP", + "pending_section_missing", + "QUEUE.md is missing the '## Pending Milestones' section.", + ); + } + + const actualMilestones = extractMilestoneIds(pendingSection); + const expectedMilestones = [...EXPECTED_PENDING_MILESTONES]; + const missing = expectedMilestones.filter((milestone) => !actualMilestones.includes(milestone)); + const unexpected = actualMilestones.filter( + (milestone) => !expectedMilestones.includes(milestone as (typeof EXPECTED_PENDING_MILESTONES)[number]), + ); + + if (missing.length > 0 || unexpected.length > 0) { + return failCheck( + "M054-S01-PENDING-QUEUE-MEMBERSHIP", + "pending_queue_membership_mismatch", + [ + missing.length > 0 ? `missing: ${missing.join(", ")}` : null, + unexpected.length > 0 ? `unexpected: ${unexpected.join(", ")}` : null, + ] + .filter(Boolean) + .join("; "), + ); + } + + return passCheck( + "M054-S01-PENDING-QUEUE-MEMBERSHIP", + "pending_queue_membership_ok", + `Pending milestones match expected set: ${expectedMilestones.join(", ")}`, + ); +} + +function buildNotPendingRedirectCheck(queueContent: string): Check { + const notPendingSection = extractSection(queueContent, "Not Pending"); + + if (notPendingSection == null) { + return failCheck( + "M054-S01-NOT-PENDING-REDIRECT", + "not_pending_section_missing", + "QUEUE.md is missing the '## Not Pending' section.", + ); + } + + if (!notPendingSection.includes(".gsd/PROJECT.md")) { + return failCheck( + "M054-S01-NOT-PENDING-REDIRECT", + "not_pending_redirect_missing_project_reference", + "Not Pending section must redirect completed-history lookups to .gsd/PROJECT.md.", + ); + } + + return passCheck( + "M054-S01-NOT-PENDING-REDIRECT", + "not_pending_redirect_present", + "Not Pending section redirects completed-history lookups to .gsd/PROJECT.md.", + ); +} + +async function buildProjectAlignmentCheck( + queueContent: string, + readTextFile: (filePath: string) => Promise, +): Promise { + const pendingSection = extractSection(queueContent, "Pending Milestones", "Not Pending"); + if (pendingSection == null) { + return failCheck( + "M054-S01-PROJECT-SHIPPED-MILESTONE-ALIGNMENT", + "pending_section_missing", + "QUEUE.md is missing the '## Pending Milestones' section.", + ); + } + + let projectContent: string; + try { + projectContent = await readTextFile(PROJECT_PATH); + } catch (error) { + return failCheck( + "M054-S01-PROJECT-SHIPPED-MILESTONE-ALIGNMENT", + "project_file_unreadable", + error, + ); + } + + const pendingMilestones = extractMilestoneIds(pendingSection); + const shippedMilestones = extractProjectCompleteMilestoneIds(projectContent).filter((milestone) => + CLEARLY_SHIPPED_MILESTONES.includes( + milestone as (typeof CLEARLY_SHIPPED_MILESTONES)[number], + ), + ); + const stalePending = pendingMilestones.filter((milestone) => shippedMilestones.includes(milestone)); + + if (stalePending.length > 0) { + return failCheck( + "M054-S01-PROJECT-SHIPPED-MILESTONE-ALIGNMENT", + "project_shipped_milestone_still_pending", + `Pending section still lists shipped milestones: ${stalePending.join(", ")}`, + ); + } + + return passCheck( + "M054-S01-PROJECT-SHIPPED-MILESTONE-ALIGNMENT", + "project_shipped_alignment_ok", + `No clearly shipped milestones from .gsd/PROJECT.md remain in the pending queue (${shippedMilestones.length} milestones checked: ${CLEARLY_SHIPPED_MILESTONES.join(", ")}).`, + ); +} + +function extractSection(content: string, heading: string, untilHeading?: string): string | null { + const lines = content.split(/\r?\n/); + const startIndex = lines.findIndex((line) => line.trim() === `## ${heading}`); + + if (startIndex === -1) { + return null; + } + + const collected: string[] = []; + for (let index = startIndex + 1; index < lines.length; index += 1) { + const line = lines[index] ?? ""; + if (untilHeading != null && line.trim() === `## ${untilHeading}`) { + break; + } + collected.push(line); + } + + return collected.join("\n").trim(); +} + +function extractMilestoneIds(content: string): string[] { + return Array.from(content.matchAll(/^###\s+(M\d{3})\b/gm), (match) => match[1]).filter( + (milestone): milestone is string => milestone != null, + ); +} + +function extractProjectCompleteMilestoneIds(projectContent: string): string[] { + const shipped = new Set(); + const milestoneSequenceSection = extractSection(projectContent, "Milestone Sequence", "Queued GitHub Backlog"); + + if (milestoneSequenceSection != null) { + for (const match of milestoneSequenceSection.matchAll(/^- \[x\]\s+(M\d{3})\b/gm)) { + const milestoneId = match[1]; + if (milestoneId != null) { + shipped.add(milestoneId); + } + } + } + + for (const line of projectContent.split(/\r?\n/)) { + if (!/complete|shipped|merged and deployed/i.test(line)) { + continue; + } + + for (const match of line.matchAll(/M\d{3}/g)) { + const milestoneId = match[0]; + if (milestoneId != null) { + shipped.add(milestoneId); + } + } + } + + return [...shipped].sort((left, right) => left.localeCompare(right)); +} + +function passCheck(id: M054S01CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M054S01CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +if (import.meta.main) { + try { + const args = parseM054S01Args(process.argv.slice(2)); + const { exitCode } = await buildM054S01ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`verify:m054:s01 failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m054-s02.test.ts b/scripts/verify-m054-s02.test.ts new file mode 100644 index 00000000..c52b29f2 --- /dev/null +++ b/scripts/verify-m054-s02.test.ts @@ -0,0 +1,270 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport } from "./verify-m054-s02.ts"; +import { + M054_S02_CHECK_IDS, + buildM054S02ProofHarness, + evaluateM054S02HistoricalFolders, + parseM054S02Args, + renderM054S02Report, +} from "./verify-m054-s02.ts"; + +function requireLastPathSegment(filePath: string): string { + const segment = filePath.split("/").at(-1); + if (segment == null) { + throw new Error(`Missing trailing path segment for: ${filePath}`); + } + return segment; +} + +const EXPECTED_CHECK_IDS = [ + "M054-S02-HISTORICAL-INVENTORY-M035-M042", + "M054-S02-HISTORICAL-INVENTORY-M043", + "M054-S02-PACKAGE-SCRIPT-WIRING", +] as const; + +describe("verify m054 s02 historical folder harness", () => { + test("exports stable check ids and cli parsing", () => { + expect(M054_S02_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM054S02Args([])).toEqual({ json: false }); + expect(parseM054S02Args(["--json"])).toEqual({ json: true }); + expect(() => parseM054S02Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes for the repaired M035-M043 historical folder contract", async () => { + const report = await evaluateM054S02HistoricalFolders({ + generatedAt: "2026-04-21T06:10:00.000Z", + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return JSON.stringify({ + scripts: { + "verify:m054:s02": "bun scripts/verify-m054-s02.ts", + }, + }); + } + + const normalized = filePath.replace(/\\/g, "/"); + const match = normalized.match(/\/(M0\d{2})\/\1-(.+)$/); + if (!match) { + throw new Error(`Unexpected file path: ${filePath}`); + } + + const milestoneId = match[1]; + const suffix = match[2]; + if (milestoneId == null || suffix == null) { + throw new Error(`Unexpected file path groups: ${filePath}`); + } + + if (milestoneId === "M043") { + if (["CONTEXT.md", "ROADMAP.md", "SUMMARY.md", "VALIDATION.md"].includes(suffix)) { + return `${milestoneId} ${suffix} content`; + } + throw new Error(`Unexpected M043 file: ${suffix}`); + } + + if (["CONTEXT-DRAFT.md", "CONTEXT.md", "SUMMARY.md"].includes(suffix)) { + return `${milestoneId} ${suffix} content`; + } + + throw new Error(`Unexpected repaired milestone file: ${suffix}`); + }, + listTopLevelFiles: async (milestoneDir: string) => { + const milestoneId = requireLastPathSegment(milestoneDir); + if (milestoneId === "M043") { + return [ + "M043-CONTEXT.md", + "M043-ROADMAP.md", + "M043-SUMMARY.md", + "M043-VALIDATION.md", + ]; + } + + return [ + `${milestoneId}-CONTEXT-DRAFT.md`, + `${milestoneId}-CONTEXT.md`, + `${milestoneId}-SUMMARY.md`, + ]; + }, + }); + + expect(report.command).toBe("verify:m054:s02"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M054-S02-HISTORICAL-INVENTORY-M035-M042", + passed: true, + status_code: "historical_inventory_ok", + }), + expect.objectContaining({ + id: "M054-S02-HISTORICAL-INVENTORY-M043", + passed: true, + status_code: "historical_inventory_ok", + }), + expect.objectContaining({ + id: "M054-S02-PACKAGE-SCRIPT-WIRING", + passed: true, + status_code: "package_script_wiring_ok", + }), + ]); + + const rendered = renderM054S02Report(report); + expect(rendered).toContain("Historical folder proof surface: PASS"); + expect(rendered).toContain("M054-S02-HISTORICAL-INVENTORY-M035-M042 PASS"); + expect(rendered).toContain("M054-S02-HISTORICAL-INVENTORY-M043 PASS"); + expect(rendered).toContain("M054-S02-PACKAGE-SCRIPT-WIRING PASS"); + }); + + test("fails with named status codes for inventory drift, unreadable files, and script mismatch", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM054S02ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return JSON.stringify({ + scripts: { + "verify:m054:s02": "bun scripts/not-the-canonical-path.ts", + }, + }); + } + + const normalized = filePath.replace(/\\/g, "/"); + if (normalized.endsWith("/M035/M035-CONTEXT.md")) { + throw new Error("EACCES: M035-CONTEXT.md"); + } + + return `${normalized} content`; + }, + listTopLevelFiles: async (milestoneDir: string) => { + const milestoneId = requireLastPathSegment(milestoneDir); + if (milestoneId === "M043") { + return [ + "M043-CONTEXT.md", + "M043-SUMMARY.md", + "M043-VALIDATION.md", + ]; + } + + return [ + `${milestoneId}-CONTEXT.md`, + `${milestoneId}-SUMMARY.md`, + `${milestoneId}-UNEXPECTED.md`, + ]; + }, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M054-S02-HISTORICAL-INVENTORY-M035-M042", + passed: false, + status_code: "historical_inventory_drift", + }), + expect.objectContaining({ + id: "M054-S02-HISTORICAL-INVENTORY-M043", + passed: false, + status_code: "historical_inventory_drift", + }), + expect.objectContaining({ + id: "M054-S02-PACKAGE-SCRIPT-WIRING", + passed: false, + status_code: "package_script_wiring_mismatch", + }), + ]); + expect(report.checks[0]?.detail).toContain("M035"); + expect(report.checks[0]?.detail).toContain("M035-UNEXPECTED.md"); + expect(report.checks[0]?.detail).toContain("EACCES: M035-CONTEXT.md"); + expect(report.checks[1]?.detail).toContain("M043"); + expect(report.checks[1]?.detail).toContain("missing: M043-ROADMAP.md"); + expect(stderr.join(" ")).toContain("historical_inventory_drift"); + expect(stderr.join(" ")).toContain("package_script_wiring_mismatch"); + }); + + test("surfaces stable malformed-input and unreadable-package failures", async () => { + const malformedPackage = await evaluateM054S02HistoricalFolders({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return "{ not valid json"; + } + + return `${filePath} content`; + }, + listTopLevelFiles: async (milestoneDir: string) => { + const milestoneId = requireLastPathSegment(milestoneDir); + if (milestoneId === "M043") { + return [ + "M043-CONTEXT.md", + "M043-ROADMAP.md", + "M043-SUMMARY.md", + "M043-VALIDATION.md", + ]; + } + + return [ + `${milestoneId}-CONTEXT-DRAFT.md`, + `${milestoneId}-CONTEXT.md`, + `${milestoneId}-SUMMARY.md`, + ]; + }, + }); + + expect(malformedPackage.checks[2]).toEqual( + expect.objectContaining({ + id: "M054-S02-PACKAGE-SCRIPT-WIRING", + passed: false, + status_code: "package_json_malformed", + }), + ); + + const unreadableInventory = await evaluateM054S02HistoricalFolders({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return JSON.stringify({ + scripts: { + "verify:m054:s02": "bun scripts/verify-m054-s02.ts", + }, + }); + } + + return `${filePath} content`; + }, + listTopLevelFiles: async (milestoneDir: string) => { + if (milestoneDir.endsWith("/M043")) { + throw new Error("EACCES: M043 directory"); + } + + const milestoneId = requireLastPathSegment(milestoneDir); + return [ + `${milestoneId}-CONTEXT-DRAFT.md`, + `${milestoneId}-CONTEXT.md`, + `${milestoneId}-SUMMARY.md`, + ]; + }, + }); + + expect(unreadableInventory.checks[1]).toEqual( + expect.objectContaining({ + id: "M054-S02-HISTORICAL-INVENTORY-M043", + passed: false, + status_code: "historical_inventory_unreadable", + }), + ); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m054:s02"]).toBe( + "bun scripts/verify-m054-s02.ts", + ); + }); +}); diff --git a/scripts/verify-m054-s02.ts b/scripts/verify-m054-s02.ts new file mode 100644 index 00000000..64dd6952 --- /dev/null +++ b/scripts/verify-m054-s02.ts @@ -0,0 +1,344 @@ +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; + +const COMMAND_NAME = "verify:m054:s02" as const; +const PACKAGE_JSON_PATH = path.resolve(import.meta.dir, "../package.json"); +const HISTORICAL_MILESTONES = [ + "M035", + "M036", + "M037", + "M038", + "M039", + "M040", + "M041", + "M042", +] as const; +const BROADER_MILESTONE = "M043" as const; +const CANONICAL_SCRIPT_COMMAND = "bun scripts/verify-m054-s02.ts" as const; + +const EXPECTED_HISTORICAL_FILES = [ + "CONTEXT-DRAFT.md", + "CONTEXT.md", + "SUMMARY.md", +] as const; +const EXPECTED_M043_FILES = [ + "CONTEXT.md", + "ROADMAP.md", + "SUMMARY.md", + "VALIDATION.md", +] as const; + +export const M054_S02_CHECK_IDS = [ + "M054-S02-HISTORICAL-INVENTORY-M035-M042", + "M054-S02-HISTORICAL-INVENTORY-M043", + "M054-S02-PACKAGE-SCRIPT-WIRING", +] as const; + +export type M054S02CheckId = (typeof M054_S02_CHECK_IDS)[number]; + +export type Check = { + id: M054S02CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M054S02CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EvaluateOptions = { + generatedAt?: string; + readTextFile?: (filePath: string) => Promise; + listTopLevelFiles?: (dirPath: string) => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +export async function evaluateM054S02HistoricalFolders( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const readTextFile = options.readTextFile ?? defaultReadTextFile; + const listTopLevelFiles = options.listTopLevelFiles ?? defaultListTopLevelFiles; + + const historicalCheck = await buildHistoricalInventoryCheck({ + milestoneIds: HISTORICAL_MILESTONES, + checkId: "M054-S02-HISTORICAL-INVENTORY-M035-M042", + expectedSuffixes: EXPECTED_HISTORICAL_FILES, + readTextFile, + listTopLevelFiles, + }); + const m043Check = await buildHistoricalInventoryCheck({ + milestoneIds: [BROADER_MILESTONE], + checkId: "M054-S02-HISTORICAL-INVENTORY-M043", + expectedSuffixes: EXPECTED_M043_FILES, + readTextFile, + listTopLevelFiles, + }); + const packageScriptCheck = await buildPackageScriptCheck(readTextFile); + + const checks = [historicalCheck, m043Check, packageScriptCheck]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M054_S02_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM054S02Report(report: EvaluationReport): string { + const lines = [ + "M054 S02 historical folder verifier", + `Generated at: ${report.generatedAt}`, + `Historical folder proof surface: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM054S02ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM054S02HistoricalFolders(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM054S02Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`verify:m054:s02 failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM054S02Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +type InventoryCheckOptions = { + milestoneIds: readonly string[]; + checkId: M054S02CheckId; + expectedSuffixes: readonly string[]; + readTextFile: (filePath: string) => Promise; + listTopLevelFiles: (dirPath: string) => Promise; +}; + +async function buildHistoricalInventoryCheck( + options: InventoryCheckOptions, +): Promise { + const driftMessages: string[] = []; + const unreadableMessages: string[] = []; + + for (const milestoneId of options.milestoneIds) { + const milestoneDir = path.resolve(import.meta.dir, `../.gsd/milestones/${milestoneId}`); + + let actualFiles: string[]; + try { + actualFiles = (await options.listTopLevelFiles(milestoneDir)).sort((left, right) => + left.localeCompare(right), + ); + } catch (error) { + unreadableMessages.push(`${milestoneId}: ${normalizeDetail(error)}`); + continue; + } + + const expectedFiles = options.expectedSuffixes + .map((suffix) => `${milestoneId}-${suffix}`) + .sort((left, right) => left.localeCompare(right)); + const missing = expectedFiles.filter((fileName) => !actualFiles.includes(fileName)); + const unexpected = actualFiles.filter((fileName) => !expectedFiles.includes(fileName)); + + if (missing.length > 0 || unexpected.length > 0) { + driftMessages.push( + [ + `${milestoneId}`, + missing.length > 0 ? `missing: ${missing.join(", ")}` : null, + unexpected.length > 0 ? `unexpected: ${unexpected.join(", ")}` : null, + ] + .filter(Boolean) + .join("; "), + ); + } + + for (const fileName of expectedFiles) { + const filePath = path.join(milestoneDir, fileName); + try { + const content = await options.readTextFile(filePath); + if (content.trim().length === 0) { + driftMessages.push(`${milestoneId}; empty: ${fileName}`); + } + } catch (error) { + driftMessages.push(`${milestoneId}; unreadable: ${fileName}; ${normalizeDetail(error)}`); + } + } + } + + if (unreadableMessages.length > 0) { + return failCheck( + options.checkId, + "historical_inventory_unreadable", + unreadableMessages.join(" | "), + ); + } + + if (driftMessages.length > 0) { + return failCheck( + options.checkId, + "historical_inventory_drift", + driftMessages.join(" | "), + ); + } + + return passCheck( + options.checkId, + "historical_inventory_ok", + `Verified ${options.milestoneIds.join(", ")} top-level inventory.`, + ); +} + +async function buildPackageScriptCheck( + readTextFile: (filePath: string) => Promise, +): Promise { + let packageJsonText: string; + try { + packageJsonText = await readTextFile(PACKAGE_JSON_PATH); + } catch (error) { + return failCheck( + "M054-S02-PACKAGE-SCRIPT-WIRING", + "package_json_unreadable", + error, + ); + } + + let packageJson: { scripts?: Record }; + try { + packageJson = JSON.parse(packageJsonText) as { scripts?: Record }; + } catch (error) { + return failCheck( + "M054-S02-PACKAGE-SCRIPT-WIRING", + "package_json_malformed", + error, + ); + } + + const actualCommand = packageJson.scripts?.[COMMAND_NAME]; + if (actualCommand == null) { + return failCheck( + "M054-S02-PACKAGE-SCRIPT-WIRING", + "package_script_wiring_missing", + `package.json is missing scripts.${COMMAND_NAME}`, + ); + } + + if (actualCommand !== CANONICAL_SCRIPT_COMMAND) { + return failCheck( + "M054-S02-PACKAGE-SCRIPT-WIRING", + "package_script_wiring_mismatch", + `Expected ${CANONICAL_SCRIPT_COMMAND} but found ${actualCommand}`, + ); + } + + return passCheck( + "M054-S02-PACKAGE-SCRIPT-WIRING", + "package_script_wiring_ok", + `package.json scripts.${COMMAND_NAME} matches the canonical command.`, + ); +} + +function passCheck(id: M054S02CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M054S02CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +async function defaultListTopLevelFiles(dirPath: string): Promise { + const entries = await readdir(dirPath, { withFileTypes: true }); + return entries.filter((entry) => entry.isFile()).map((entry) => entry.name); +} + +if (import.meta.main) { + try { + const args = parseM054S02Args(process.argv.slice(2)); + const { exitCode } = await buildM054S02ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`verify:m054:s02 failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m054-s03.test.ts b/scripts/verify-m054-s03.test.ts new file mode 100644 index 00000000..0fb89697 --- /dev/null +++ b/scripts/verify-m054-s03.test.ts @@ -0,0 +1,256 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport } from "./verify-m054-s03.ts"; +import { + M054_S03_CHECK_IDS, + buildM054S03ProofHarness, + evaluateM054S03RecentHistory, + parseM054S03Args, + renderM054S03Report, +} from "./verify-m054-s03.ts"; + +function requireLastPathSegment(filePath: string): string { + const segment = filePath.replace(/\\/g, "/").split("/").at(-1); + if (segment == null) { + throw new Error(`Missing trailing path segment for: ${filePath}`); + } + return segment; +} + +const EXPECTED_CHECK_IDS = [ + "M054-S03-RECENT-INVENTORY-M048-M050", + "M054-S03-RECENT-SUMMARIES-M051-M052", + "M054-S03-M052-SLICE-TASK-SUMMARIES", + "M054-S03-PACKAGE-SCRIPT-WIRING", +] as const; + +describe("verify m054 s03 recent-history harness", () => { + test("exports stable check ids and cli parsing", () => { + expect(M054_S03_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM054S03Args([])).toEqual({ json: false }); + expect(parseM054S03Args(["--json"])).toEqual({ json: true }); + expect(() => parseM054S03Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes for the repaired recent-history contract", async () => { + const report = await evaluateM054S03RecentHistory({ + generatedAt: "2026-04-21T07:05:00.000Z", + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return JSON.stringify({ + scripts: { + "verify:m054:s03": "bun scripts/verify-m054-s03.ts", + }, + }); + } + + return `${filePath} content`; + }, + listTopLevelFiles: async (dirPath: string) => { + const milestoneId = requireLastPathSegment(dirPath); + if (["M048", "M049", "M050"].includes(milestoneId)) { + return [ + `${milestoneId}-CONTEXT.md`, + `${milestoneId}-SUMMARY.md`, + ]; + } + + throw new Error(`Unexpected top-level listing request: ${dirPath}`); + }, + listTaskSummaryFiles: async () => { + return ["T01-SUMMARY.md", "T02-SUMMARY.md"]; + }, + }); + + expect(report.command).toBe("verify:m054:s03"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M054-S03-RECENT-INVENTORY-M048-M050", + passed: true, + status_code: "recent_inventory_ok", + }), + expect.objectContaining({ + id: "M054-S03-RECENT-SUMMARIES-M051-M052", + passed: true, + status_code: "recent_milestone_summaries_ok", + }), + expect.objectContaining({ + id: "M054-S03-M052-SLICE-TASK-SUMMARIES", + passed: true, + status_code: "m052_slice_task_summaries_ok", + }), + expect.objectContaining({ + id: "M054-S03-PACKAGE-SCRIPT-WIRING", + passed: true, + status_code: "package_script_wiring_ok", + }), + ]); + + const rendered = renderM054S03Report(report); + expect(rendered).toContain("Recent-history proof surface: PASS"); + expect(rendered).toContain("M054-S03-RECENT-INVENTORY-M048-M050 PASS"); + expect(rendered).toContain("M054-S03-RECENT-SUMMARIES-M051-M052 PASS"); + expect(rendered).toContain("M054-S03-M052-SLICE-TASK-SUMMARIES PASS"); + expect(rendered).toContain("M054-S03-PACKAGE-SCRIPT-WIRING PASS"); + }); + + test("fails with named status codes for strict inventory drift, missing M052 summary surfaces, and script mismatch", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM054S03ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return JSON.stringify({ + scripts: { + "verify:m054:s03": "bun scripts/not-the-canonical-path.ts", + }, + }); + } + + const normalized = filePath.replace(/\\/g, "/"); + if (normalized.endsWith("/M052/slices/S03/tasks/T03-SUMMARY.md")) { + throw new Error("EACCES: T03-SUMMARY.md"); + } + + return `${normalized} content`; + }, + listTopLevelFiles: async (dirPath: string) => { + const milestoneId = requireLastPathSegment(dirPath); + if (milestoneId === "M048") { + return ["M048-CONTEXT.md", "M048-SUMMARY.md", "M048-CONTEXT-DRAFT.md"]; + } + if (milestoneId === "M049") { + return ["M049-CONTEXT.md", "M049-SUMMARY.md"]; + } + if (milestoneId === "M050") { + return ["M050-SUMMARY.md"]; + } + + throw new Error(`Unexpected top-level listing request: ${dirPath}`); + }, + listTaskSummaryFiles: async (sliceTaskDir: string) => { + const normalized = sliceTaskDir.replace(/\\/g, "/"); + if (normalized.endsWith("/S01/tasks")) { + return ["T01-SUMMARY.md"]; + } + if (normalized.endsWith("/S02/tasks")) { + return []; + } + if (normalized.endsWith("/S03/tasks")) { + return ["T03-SUMMARY.md"]; + } + + throw new Error(`Unexpected task listing request: ${sliceTaskDir}`); + }, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M054-S03-RECENT-INVENTORY-M048-M050", + passed: false, + status_code: "recent_inventory_drift", + }), + expect.objectContaining({ + id: "M054-S03-RECENT-SUMMARIES-M051-M052", + passed: true, + status_code: "recent_milestone_summaries_ok", + }), + expect.objectContaining({ + id: "M054-S03-M052-SLICE-TASK-SUMMARIES", + passed: false, + status_code: "m052_slice_task_summaries_drift", + }), + expect.objectContaining({ + id: "M054-S03-PACKAGE-SCRIPT-WIRING", + passed: false, + status_code: "package_script_wiring_mismatch", + }), + ]); + expect(report.checks[0]?.detail).toContain("unexpected: M048-CONTEXT-DRAFT.md"); + expect(report.checks[0]?.detail).toContain("M050"); + expect(report.checks[0]?.detail).toContain("missing: M050-CONTEXT.md"); + expect(report.checks[2]?.detail).toContain("S02"); + expect(report.checks[2]?.detail).toContain("no task summary files found"); + expect(report.checks[2]?.detail).toContain("EACCES: T03-SUMMARY.md"); + expect(stderr.join(" ")).toContain("recent_inventory_drift"); + expect(stderr.join(" ")).toContain("m052_slice_task_summaries_drift"); + expect(stderr.join(" ")).toContain("package_script_wiring_mismatch"); + }); + + test("surfaces stable malformed-input failures for unreadable summaries and malformed package json", async () => { + const malformedPackage = await evaluateM054S03RecentHistory({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return "{ not valid json"; + } + + return `${filePath} content`; + }, + listTopLevelFiles: async (dirPath: string) => { + const milestoneId = requireLastPathSegment(dirPath); + return [`${milestoneId}-CONTEXT.md`, `${milestoneId}-SUMMARY.md`]; + }, + listTaskSummaryFiles: async () => ["T01-SUMMARY.md"], + }); + + expect(malformedPackage.checks[3]).toEqual( + expect.objectContaining({ + id: "M054-S03-PACKAGE-SCRIPT-WIRING", + passed: false, + status_code: "package_json_malformed", + }), + ); + + const unreadableSummary = await evaluateM054S03RecentHistory({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return JSON.stringify({ + scripts: { + "verify:m054:s03": "bun scripts/verify-m054-s03.ts", + }, + }); + } + + const normalized = filePath.replace(/\\/g, "/"); + if (normalized.endsWith("/M051/M051-SUMMARY.md")) { + throw new Error("EACCES: M051-SUMMARY.md"); + } + + return `${normalized} content`; + }, + listTopLevelFiles: async (dirPath: string) => { + const milestoneId = requireLastPathSegment(dirPath); + return [`${milestoneId}-CONTEXT.md`, `${milestoneId}-SUMMARY.md`]; + }, + listTaskSummaryFiles: async () => ["T01-SUMMARY.md"], + }); + + expect(unreadableSummary.checks[1]).toEqual( + expect.objectContaining({ + id: "M054-S03-RECENT-SUMMARIES-M051-M052", + passed: false, + status_code: "recent_milestone_summaries_unreadable", + }), + ); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m054:s03"]).toBe( + "bun scripts/verify-m054-s03.ts", + ); + }); +}); diff --git a/scripts/verify-m054-s03.ts b/scripts/verify-m054-s03.ts new file mode 100644 index 00000000..5d099068 --- /dev/null +++ b/scripts/verify-m054-s03.ts @@ -0,0 +1,442 @@ +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; + +const COMMAND_NAME = "verify:m054:s03" as const; +const PACKAGE_JSON_PATH = path.resolve(import.meta.dir, "../package.json"); +const RECENT_TOP_LEVEL_MILESTONES = ["M048", "M049", "M050"] as const; +const RECENT_SUMMARY_MILESTONES = ["M051", "M052"] as const; +const M052_SLICE_IDS = ["S01", "S02", "S03"] as const; +const CANONICAL_SCRIPT_COMMAND = "bun scripts/verify-m054-s03.ts" as const; +const EXPECTED_RECENT_TOP_LEVEL_FILES = ["CONTEXT.md", "SUMMARY.md"] as const; + +export const M054_S03_CHECK_IDS = [ + "M054-S03-RECENT-INVENTORY-M048-M050", + "M054-S03-RECENT-SUMMARIES-M051-M052", + "M054-S03-M052-SLICE-TASK-SUMMARIES", + "M054-S03-PACKAGE-SCRIPT-WIRING", +] as const; + +export type M054S03CheckId = (typeof M054_S03_CHECK_IDS)[number]; + +export type Check = { + id: M054S03CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M054S03CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EvaluateOptions = { + generatedAt?: string; + readTextFile?: (filePath: string) => Promise; + listTopLevelFiles?: (dirPath: string) => Promise; + listTaskSummaryFiles?: (dirPath: string) => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +export async function evaluateM054S03RecentHistory( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const readTextFile = options.readTextFile ?? defaultReadTextFile; + const listTopLevelFiles = options.listTopLevelFiles ?? defaultListTopLevelFiles; + const listTaskSummaryFiles = options.listTaskSummaryFiles ?? defaultListTaskSummaryFiles; + + const recentInventoryCheck = await buildRecentInventoryCheck({ + readTextFile, + listTopLevelFiles, + }); + const recentMilestoneSummariesCheck = await buildRecentMilestoneSummariesCheck(readTextFile); + const m052SliceTaskSummariesCheck = await buildM052SliceTaskSummariesCheck({ + readTextFile, + listTaskSummaryFiles, + }); + const packageScriptCheck = await buildPackageScriptCheck(readTextFile); + + const checks = [ + recentInventoryCheck, + recentMilestoneSummariesCheck, + m052SliceTaskSummariesCheck, + packageScriptCheck, + ]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M054_S03_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM054S03Report(report: EvaluationReport): string { + const lines = [ + "M054 S03 recent-history verifier", + `Generated at: ${report.generatedAt}`, + `Recent-history proof surface: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM054S03ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM054S03RecentHistory(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM054S03Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`verify:m054:s03 failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM054S03Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +type RecentInventoryCheckOptions = { + readTextFile: (filePath: string) => Promise; + listTopLevelFiles: (dirPath: string) => Promise; +}; + +async function buildRecentInventoryCheck( + options: RecentInventoryCheckOptions, +): Promise { + const driftMessages: string[] = []; + const unreadableMessages: string[] = []; + + for (const milestoneId of RECENT_TOP_LEVEL_MILESTONES) { + const milestoneDir = path.resolve(import.meta.dir, `../.gsd/milestones/${milestoneId}`); + + let actualFiles: string[]; + try { + actualFiles = (await options.listTopLevelFiles(milestoneDir)).sort((left, right) => + left.localeCompare(right), + ); + } catch (error) { + unreadableMessages.push(`${milestoneId}: ${normalizeDetail(error)}`); + continue; + } + + const expectedFiles = EXPECTED_RECENT_TOP_LEVEL_FILES.map( + (suffix) => `${milestoneId}-${suffix}`, + ).sort((left, right) => left.localeCompare(right)); + const missing = expectedFiles.filter((fileName) => !actualFiles.includes(fileName)); + const unexpected = actualFiles.filter((fileName) => !expectedFiles.includes(fileName)); + + if (missing.length > 0 || unexpected.length > 0) { + driftMessages.push( + [ + milestoneId, + missing.length > 0 ? `missing: ${missing.join(", ")}` : null, + unexpected.length > 0 ? `unexpected: ${unexpected.join(", ")}` : null, + ] + .filter(Boolean) + .join("; "), + ); + } + + for (const fileName of expectedFiles) { + const filePath = path.join(milestoneDir, fileName); + try { + const content = await options.readTextFile(filePath); + if (content.trim().length === 0) { + driftMessages.push(`${milestoneId}; empty: ${fileName}`); + } + } catch (error) { + driftMessages.push(`${milestoneId}; unreadable: ${fileName}; ${normalizeDetail(error)}`); + } + } + } + + if (unreadableMessages.length > 0) { + return failCheck( + "M054-S03-RECENT-INVENTORY-M048-M050", + "recent_inventory_unreadable", + unreadableMessages.join(" | "), + ); + } + + if (driftMessages.length > 0) { + return failCheck( + "M054-S03-RECENT-INVENTORY-M048-M050", + "recent_inventory_drift", + driftMessages.join(" | "), + ); + } + + return passCheck( + "M054-S03-RECENT-INVENTORY-M048-M050", + "recent_inventory_ok", + `Verified strict top-level inventory for ${RECENT_TOP_LEVEL_MILESTONES.join(", ")}.`, + ); +} + +async function buildRecentMilestoneSummariesCheck( + readTextFile: (filePath: string) => Promise, +): Promise { + const unreadableMessages: string[] = []; + const driftMessages: string[] = []; + + for (const milestoneId of RECENT_SUMMARY_MILESTONES) { + const summaryPath = path.resolve(import.meta.dir, `../.gsd/milestones/${milestoneId}/${milestoneId}-SUMMARY.md`); + + try { + const content = await readTextFile(summaryPath); + if (content.trim().length === 0) { + driftMessages.push(`${milestoneId}; empty summary: ${path.basename(summaryPath)}`); + } + } catch (error) { + unreadableMessages.push(`${milestoneId}: ${normalizeDetail(error)}`); + } + } + + if (unreadableMessages.length > 0) { + return failCheck( + "M054-S03-RECENT-SUMMARIES-M051-M052", + "recent_milestone_summaries_unreadable", + unreadableMessages.join(" | "), + ); + } + + if (driftMessages.length > 0) { + return failCheck( + "M054-S03-RECENT-SUMMARIES-M051-M052", + "recent_milestone_summaries_drift", + driftMessages.join(" | "), + ); + } + + return passCheck( + "M054-S03-RECENT-SUMMARIES-M051-M052", + "recent_milestone_summaries_ok", + `Verified milestone summaries for ${RECENT_SUMMARY_MILESTONES.join(" and ")}.`, + ); +} + +type M052SliceTaskCheckOptions = { + readTextFile: (filePath: string) => Promise; + listTaskSummaryFiles: (dirPath: string) => Promise; +}; + +async function buildM052SliceTaskSummariesCheck( + options: M052SliceTaskCheckOptions, +): Promise { + const driftMessages: string[] = []; + + for (const sliceId of M052_SLICE_IDS) { + const sliceDir = path.resolve(import.meta.dir, `../.gsd/milestones/M052/slices/${sliceId}`); + const sliceSummaryPath = path.join(sliceDir, `${sliceId}-SUMMARY.md`); + + try { + const content = await options.readTextFile(sliceSummaryPath); + if (content.trim().length === 0) { + driftMessages.push(`${sliceId}; empty slice summary: ${sliceId}-SUMMARY.md`); + } + } catch (error) { + driftMessages.push(`${sliceId}; unreadable slice summary: ${normalizeDetail(error)}`); + continue; + } + + const tasksDir = path.join(sliceDir, "tasks"); + let taskSummaryFiles: string[]; + try { + taskSummaryFiles = (await options.listTaskSummaryFiles(tasksDir)).sort((left, right) => + left.localeCompare(right), + ); + } catch (error) { + driftMessages.push(`${sliceId}; unreadable tasks directory: ${normalizeDetail(error)}`); + continue; + } + + if (taskSummaryFiles.length === 0) { + driftMessages.push(`${sliceId}; no task summary files found`); + continue; + } + + for (const fileName of taskSummaryFiles) { + const taskSummaryPath = path.join(tasksDir, fileName); + try { + const content = await options.readTextFile(taskSummaryPath); + if (content.trim().length === 0) { + driftMessages.push(`${sliceId}; empty task summary: ${fileName}`); + } + } catch (error) { + driftMessages.push(`${sliceId}; unreadable task summary: ${fileName}; ${normalizeDetail(error)}`); + } + } + } + + if (driftMessages.length > 0) { + return failCheck( + "M054-S03-M052-SLICE-TASK-SUMMARIES", + "m052_slice_task_summaries_drift", + driftMessages.join(" | "), + ); + } + + return passCheck( + "M054-S03-M052-SLICE-TASK-SUMMARIES", + "m052_slice_task_summaries_ok", + `Verified M052 slice summaries and task summaries for ${M052_SLICE_IDS.join(", ")}.`, + ); +} + +async function buildPackageScriptCheck( + readTextFile: (filePath: string) => Promise, +): Promise { + let packageJsonText: string; + try { + packageJsonText = await readTextFile(PACKAGE_JSON_PATH); + } catch (error) { + return failCheck( + "M054-S03-PACKAGE-SCRIPT-WIRING", + "package_json_unreadable", + error, + ); + } + + let packageJson: { scripts?: Record }; + try { + packageJson = JSON.parse(packageJsonText) as { scripts?: Record }; + } catch (error) { + return failCheck( + "M054-S03-PACKAGE-SCRIPT-WIRING", + "package_json_malformed", + error, + ); + } + + const actualCommand = packageJson.scripts?.[COMMAND_NAME]; + if (actualCommand == null) { + return failCheck( + "M054-S03-PACKAGE-SCRIPT-WIRING", + "package_script_wiring_missing", + `package.json is missing scripts.${COMMAND_NAME}`, + ); + } + + if (actualCommand !== CANONICAL_SCRIPT_COMMAND) { + return failCheck( + "M054-S03-PACKAGE-SCRIPT-WIRING", + "package_script_wiring_mismatch", + `Expected ${CANONICAL_SCRIPT_COMMAND} but found ${actualCommand}`, + ); + } + + return passCheck( + "M054-S03-PACKAGE-SCRIPT-WIRING", + "package_script_wiring_ok", + `package.json scripts.${COMMAND_NAME} matches the canonical command.`, + ); +} + +function passCheck(id: M054S03CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M054S03CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +async function defaultListTopLevelFiles(dirPath: string): Promise { + const entries = await readdir(dirPath, { withFileTypes: true }); + return entries.filter((entry) => entry.isFile()).map((entry) => entry.name); +} + +async function defaultListTaskSummaryFiles(dirPath: string): Promise { + const entries = await readdir(dirPath, { withFileTypes: true }); + return entries + .filter((entry) => entry.isFile() && /^T\d+-SUMMARY\.md$/u.test(entry.name)) + .map((entry) => entry.name); +} + +if (import.meta.main) { + try { + const args = parseM054S03Args(process.argv.slice(2)); + const { exitCode } = await buildM054S03ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`verify:m054:s03 failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m054-s04.test.ts b/scripts/verify-m054-s04.test.ts new file mode 100644 index 00000000..3cb36ad3 --- /dev/null +++ b/scripts/verify-m054-s04.test.ts @@ -0,0 +1,375 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport } from "./verify-m054-s04.ts"; +import { + M054_S04_CHECK_IDS, + buildM054S04ProofHarness, + evaluateM054S04VerifierCoverage, + parseM054S04Args, + renderM054S04Report, +} from "./verify-m054-s04.ts"; + +const EXPECTED_CHECK_IDS = [ + "M054-S04-COMPLETED-MILESTONE-COVERAGE", + "M054-S04-PACKAGE-SCRIPT-WIRING", +] as const; + +const COVERED_MILESTONES = [ + "M035", + "M036", + "M037", + "M038", + "M039", + "M040", + "M041", + "M042", + "M043", + "M044", + "M045", + "M046", + "M047", + "M048", + "M049", + "M050", + "M051", + "M052", +] as const; + +function buildPackageJson(overrides: Record = {}): string { + return JSON.stringify( + { + scripts: { + "verify:m036:s01": "bun scripts/verify-m036-s01.ts", + "verify:m036:s02": "bun scripts/verify-m036-s02.ts", + "verify:m036:s03": "bun scripts/verify-m036-s03.ts", + "verify:m037:s01": "bun scripts/verify-m037-s01.ts", + "verify:m037:s02": "bun scripts/verify-m037-s02.ts", + "verify:m037:s03": "bun scripts/verify-m037-s03.ts", + "verify:m038:s02": "bun scripts/verify-m038-s02.ts", + "verify:m038:s03": "bun scripts/verify-m038-s03.ts", + "verify:m040:s02": "bun scripts/verify-m040-s02.ts", + "verify:m040:s03": "bun scripts/verify-m040-s03.ts", + "verify:m041:s02": "bun scripts/verify-m041-s02.ts", + "verify:m041:s03": "bun scripts/verify-m041-s03.ts", + "verify:m042:s01": "bun scripts/verify-m042-s01.ts", + "verify:m042:s02": "bun scripts/verify-m042-s02.ts", + "verify:m042:s03": "bun scripts/verify-m042-s03.ts", + "verify:m044": "bun scripts/verify-m044-s01.ts", + "verify:m044:s01": "bun scripts/verify-m044-s01.ts", + "verify:m045:s01": "bun scripts/verify-m045-s01.ts", + "verify:m045:s03": "bun scripts/verify-m045-s03.ts", + "verify:m046": "bun scripts/verify-m046.ts", + "verify:m046:s01": "bun scripts/verify-m046-s01.ts", + "verify:m046:s02": "bun scripts/verify-m046-s02.ts", + "verify:m047": "bun scripts/verify-m047.ts", + "verify:m047:s01": "bun scripts/verify-m047-s01.ts", + "verify:m047:s02": "bun scripts/verify-m047-s02.ts", + "verify:m048:s01": "bun scripts/verify-m048-s01.ts", + "verify:m048:s02": "bun scripts/verify-m048-s02.ts", + "verify:m048:s03": "bun scripts/verify-m048-s03.ts", + "verify:m049:s02": "bun scripts/verify-m049-s02.ts", + "verify:m053": "bun scripts/verify-m053.ts", + "verify:m054:s02": "bun scripts/verify-m054-s02.ts", + "verify:m054:s03": "bun scripts/verify-m054-s03.ts", + "verify:m054:s04": "bun scripts/verify-m054-s04.ts", + ...overrides, + }, + }, + null, + 2, + ); +} + +function makeArtifactReader(records: Record) { + return async (filePath: string): Promise => { + const normalized = filePath.replaceAll("\\", "/"); + const record = Object.entries(records).find(([suffix]) => normalized.endsWith(suffix))?.[1]; + if (record == null) { + throw new Error(`ENOENT: ${normalized}`); + } + if (record instanceof Error) { + throw record; + } + return record; + }; +} + +describe("verify m054 s04 verifier/rationale audit", () => { + test("exports stable check ids and cli parsing", () => { + expect(M054_S04_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM054S04Args([])).toEqual({ json: false }); + expect(parseM054S04Args(["--json"])).toEqual({ json: true }); + expect(() => parseM054S04Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes with real verifier coverage, explicit rationale coverage, and package wiring", async () => { + const report = await evaluateM054S04VerifierCoverage({ + generatedAt: "2026-04-21T05:20:00.000Z", + readTextFile: makeArtifactReader({ + "package.json": buildPackageJson(), + ".gsd/milestones/M039/M039-SUMMARY.md": "No committed `verify-m039-*` harness survives in the current tree.", + ".gsd/milestones/M043/M043-CONTEXT.md": "No `verify:m043:*` package scripts survive in the current repo.", + ".gsd/milestones/M050/M050-CONTEXT.md": "M050 intentionally reused `verify:m048:s01`, `verify:m048:s02`, and `verify:m048:s03` instead of introducing `verify:m050:*`.", + ".gsd/milestones/M050/M050-SUMMARY.md": "The milestone intentionally reused `verify:m048:s01`, `verify:m048:s02`, and `verify:m048:s03`.", + ".gsd/milestones/M051/M051-SUMMARY.md": "M051 summary still exists.", + ".gsd/milestones/M052/M052-SUMMARY.md": "Milestone prose with no verifier claims for this fixture.", + ".gsd/milestones/M052/M052-VALIDATION.md": "Validation prose with no verifier claims for this fixture.", + }), + fileExists: (filePath: string) => { + const normalized = filePath.replaceAll("\\", "/"); + return ( + normalized.endsWith("scripts/verify-m036-s01.ts") || + normalized.endsWith("scripts/verify-m036-s02.ts") || + normalized.endsWith("scripts/verify-m036-s03.ts") || + normalized.endsWith("scripts/verify-m037-s01.ts") || + normalized.endsWith("scripts/verify-m037-s02.ts") || + normalized.endsWith("scripts/verify-m037-s03.ts") || + normalized.endsWith("scripts/verify-m038-s02.ts") || + normalized.endsWith("scripts/verify-m038-s03.ts") || + normalized.endsWith("scripts/verify-m040-s02.ts") || + normalized.endsWith("scripts/verify-m040-s03.ts") || + normalized.endsWith("scripts/verify-m041-s02.ts") || + normalized.endsWith("scripts/verify-m041-s03.ts") || + normalized.endsWith("scripts/verify-m042-s01.ts") || + normalized.endsWith("scripts/verify-m042-s02.ts") || + normalized.endsWith("scripts/verify-m042-s03.ts") || + normalized.endsWith("scripts/verify-m044-s01.ts") || + normalized.endsWith("scripts/verify-m045-s01.ts") || + normalized.endsWith("scripts/verify-m045-s03.ts") || + normalized.endsWith("scripts/verify-m046.ts") || + normalized.endsWith("scripts/verify-m046-s01.ts") || + normalized.endsWith("scripts/verify-m046-s02.ts") || + normalized.endsWith("scripts/verify-m047.ts") || + normalized.endsWith("scripts/verify-m047-s01.ts") || + normalized.endsWith("scripts/verify-m047-s02.ts") || + normalized.endsWith("scripts/verify-m048-s01.ts") || + normalized.endsWith("scripts/verify-m048-s02.ts") || + normalized.endsWith("scripts/verify-m048-s03.ts") || + normalized.endsWith("scripts/verify-m049-s02.ts") || + normalized.endsWith("scripts/verify-m053.ts") || + normalized.endsWith("scripts/verify-m054-s02.ts") || + normalized.endsWith("scripts/verify-m054-s03.ts") || + normalized.endsWith("scripts/verify-m054-s04.ts") + ); + }, + }); + + expect(report.command).toBe("verify:m054:s04"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M054-S04-COMPLETED-MILESTONE-COVERAGE", + passed: true, + status_code: "completed_milestone_coverage_ok", + }), + expect.objectContaining({ + id: "M054-S04-PACKAGE-SCRIPT-WIRING", + passed: true, + status_code: "package_script_wiring_ok", + }), + ]); + + const milestoneResults = report.checks[0]?.milestones ?? []; + expect(milestoneResults).toHaveLength(COVERED_MILESTONES.length); + expect(milestoneResults.find((entry) => entry.milestoneId === "M039")).toEqual( + expect.objectContaining({ + coverageType: "rationale", + passed: true, + status_code: "explicit_rationale_present", + }), + ); + expect(milestoneResults.find((entry) => entry.milestoneId === "M043")).toEqual( + expect.objectContaining({ + coverageType: "rationale", + passed: true, + status_code: "explicit_rationale_present", + }), + ); + expect(milestoneResults.find((entry) => entry.milestoneId === "M050")).toEqual( + expect.objectContaining({ + coverageType: "rationale", + passed: true, + status_code: "explicit_rationale_present", + }), + ); + expect(milestoneResults.find((entry) => entry.milestoneId === "M044")).toEqual( + expect.objectContaining({ + coverageType: "verifier", + passed: true, + status_code: "repo_verifier_coverage_present", + }), + ); + + const rendered = renderM054S04Report(report); + expect(rendered).toContain("Verifier/rationale audit: PASS"); + expect(rendered).toContain("M052"); + expect(rendered).toContain("coverage=verifier"); + }); + + test("fails stale overclaims modeled on m052 while nearby milestones still pass", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM054S04ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + readTextFile: makeArtifactReader({ + "package.json": buildPackageJson(), + ".gsd/milestones/M039/M039-SUMMARY.md": "No committed `verify-m039-*` harness survives in the current tree.", + ".gsd/milestones/M043/M043-CONTEXT.md": "No `verify:m043:*` package scripts survive in the current repo.", + ".gsd/milestones/M050/M050-CONTEXT.md": "M050 intentionally reused `verify:m048:s01`, `verify:m048:s02`, and `verify:m048:s03` instead of introducing `verify:m050:*`.", + ".gsd/milestones/M050/M050-SUMMARY.md": "The milestone intentionally reused `verify:m048:s01`, `verify:m048:s02`, and `verify:m048:s03`.", + ".gsd/milestones/M051/M051-SUMMARY.md": "M051 summary still exists.", + ".gsd/milestones/M052/M052-SUMMARY.md": "Claims `scripts/verify-m052-s01.ts`, `scripts/verify-m052-s02.ts`, `scripts/verify-m052.ts`, and `bun run verify:m052` exist.", + ".gsd/milestones/M052/M052-VALIDATION.md": "Also cites `verify:m052:s01`, `verify:m052:s02`, and the same missing script files.", + }), + fileExists: (filePath: string) => { + const normalized = filePath.replaceAll("\\", "/"); + return ( + normalized.endsWith("scripts/verify-m036-s01.ts") || + normalized.endsWith("scripts/verify-m036-s02.ts") || + normalized.endsWith("scripts/verify-m036-s03.ts") || + normalized.endsWith("scripts/verify-m037-s01.ts") || + normalized.endsWith("scripts/verify-m037-s02.ts") || + normalized.endsWith("scripts/verify-m037-s03.ts") || + normalized.endsWith("scripts/verify-m038-s02.ts") || + normalized.endsWith("scripts/verify-m038-s03.ts") || + normalized.endsWith("scripts/verify-m040-s02.ts") || + normalized.endsWith("scripts/verify-m040-s03.ts") || + normalized.endsWith("scripts/verify-m041-s02.ts") || + normalized.endsWith("scripts/verify-m041-s03.ts") || + normalized.endsWith("scripts/verify-m042-s01.ts") || + normalized.endsWith("scripts/verify-m042-s02.ts") || + normalized.endsWith("scripts/verify-m042-s03.ts") || + normalized.endsWith("scripts/verify-m044-s01.ts") || + normalized.endsWith("scripts/verify-m045-s01.ts") || + normalized.endsWith("scripts/verify-m045-s03.ts") || + normalized.endsWith("scripts/verify-m046.ts") || + normalized.endsWith("scripts/verify-m046-s01.ts") || + normalized.endsWith("scripts/verify-m046-s02.ts") || + normalized.endsWith("scripts/verify-m047.ts") || + normalized.endsWith("scripts/verify-m047-s01.ts") || + normalized.endsWith("scripts/verify-m047-s02.ts") || + normalized.endsWith("scripts/verify-m048-s01.ts") || + normalized.endsWith("scripts/verify-m048-s02.ts") || + normalized.endsWith("scripts/verify-m048-s03.ts") || + normalized.endsWith("scripts/verify-m049-s02.ts") || + normalized.endsWith("scripts/verify-m053.ts") || + normalized.endsWith("scripts/verify-m054-s02.ts") || + normalized.endsWith("scripts/verify-m054-s03.ts") || + normalized.endsWith("scripts/verify-m054-s04.ts") + ); + }, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks[0]).toEqual( + expect.objectContaining({ + id: "M054-S04-COMPLETED-MILESTONE-COVERAGE", + passed: false, + status_code: "completed_milestone_coverage_drift", + }), + ); + const m052 = report.checks[0]?.milestones?.find((entry) => entry.milestoneId === "M052"); + expect(m052).toEqual( + expect.objectContaining({ + coverageType: "overclaim", + passed: false, + status_code: "claimed_verifier_missing", + }), + ); + expect(m052?.detail).toContain("verify:m052"); + expect(stderr.join(" ")).toContain("claimed_verifier_missing"); + }); + + test("fails with stable unreadable-artifact codes instead of aborting the whole audit", async () => { + const report = await evaluateM054S04VerifierCoverage({ + readTextFile: makeArtifactReader({ + "package.json": buildPackageJson(), + ".gsd/milestones/M039/M039-SUMMARY.md": new Error("EACCES: M039-SUMMARY.md"), + ".gsd/milestones/M043/M043-CONTEXT.md": "No `verify:m043:*` package scripts survive in the current repo.", + ".gsd/milestones/M050/M050-CONTEXT.md": "M050 intentionally reused `verify:m048:s01`, `verify:m048:s02`, and `verify:m048:s03`.", + ".gsd/milestones/M050/M050-SUMMARY.md": "The milestone intentionally reused `verify:m048:s01`, `verify:m048:s02`, and `verify:m048:s03`.", + ".gsd/milestones/M051/M051-SUMMARY.md": "M051 summary still exists.", + ".gsd/milestones/M052/M052-SUMMARY.md": "No verifier claims in this fixture.", + ".gsd/milestones/M052/M052-VALIDATION.md": "No verifier claims in this fixture.", + }), + fileExists: () => true, + }); + + expect(report.overallPassed).toBe(false); + expect(report.checks[0]).toEqual( + expect.objectContaining({ + id: "M054-S04-COMPLETED-MILESTONE-COVERAGE", + passed: false, + status_code: "completed_milestone_coverage_drift", + }), + ); + const m039 = report.checks[0]?.milestones?.find((entry) => entry.milestoneId === "M039"); + expect(m039).toEqual( + expect.objectContaining({ + coverageType: "error", + passed: false, + status_code: "artifact_unreadable", + }), + ); + }); + + test("fails package wiring when the canonical script alias is missing or wrong", async () => { + const missing = await evaluateM054S04VerifierCoverage({ + readTextFile: makeArtifactReader({ + "package.json": JSON.stringify({ scripts: {} }), + ".gsd/milestones/M039/M039-SUMMARY.md": "No committed `verify-m039-*` harness survives in the current tree.", + ".gsd/milestones/M043/M043-CONTEXT.md": "No `verify:m043:*` package scripts survive in the current repo.", + ".gsd/milestones/M050/M050-CONTEXT.md": "M050 intentionally reused `verify:m048:s01`, `verify:m048:s02`, and `verify:m048:s03`.", + ".gsd/milestones/M050/M050-SUMMARY.md": "The milestone intentionally reused `verify:m048:s01`, `verify:m048:s02`, and `verify:m048:s03`.", + ".gsd/milestones/M051/M051-SUMMARY.md": "M051 summary still exists.", + ".gsd/milestones/M052/M052-SUMMARY.md": "No verifier claims in this fixture.", + ".gsd/milestones/M052/M052-VALIDATION.md": "No verifier claims in this fixture.", + }), + fileExists: () => true, + }); + expect(missing.checks[1]).toEqual( + expect.objectContaining({ + id: "M054-S04-PACKAGE-SCRIPT-WIRING", + passed: false, + status_code: "package_script_wiring_missing", + }), + ); + + const mismatched = await evaluateM054S04VerifierCoverage({ + readTextFile: makeArtifactReader({ + "package.json": buildPackageJson({ "verify:m054:s04": "bun ./scripts/verify-m054-s04.ts" }), + ".gsd/milestones/M039/M039-SUMMARY.md": "No committed `verify-m039-*` harness survives in the current tree.", + ".gsd/milestones/M043/M043-CONTEXT.md": "No `verify:m043:*` package scripts survive in the current repo.", + ".gsd/milestones/M050/M050-CONTEXT.md": "M050 intentionally reused `verify:m048:s01`, `verify:m048:s02`, and `verify:m048:s03`.", + ".gsd/milestones/M050/M050-SUMMARY.md": "The milestone intentionally reused `verify:m048:s01`, `verify:m048:s02`, and `verify:m048:s03`.", + ".gsd/milestones/M051/M051-SUMMARY.md": "M051 summary still exists.", + ".gsd/milestones/M052/M052-SUMMARY.md": "No verifier claims in this fixture.", + ".gsd/milestones/M052/M052-VALIDATION.md": "No verifier claims in this fixture.", + }), + fileExists: () => true, + }); + expect(mismatched.checks[1]).toEqual( + expect.objectContaining({ + id: "M054-S04-PACKAGE-SCRIPT-WIRING", + passed: false, + status_code: "package_script_wiring_mismatch", + }), + ); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m054:s04"]).toBe( + "bun scripts/verify-m054-s04.ts", + ); + }); +}); diff --git a/scripts/verify-m054-s04.ts b/scripts/verify-m054-s04.ts new file mode 100644 index 00000000..4cfcf964 --- /dev/null +++ b/scripts/verify-m054-s04.ts @@ -0,0 +1,632 @@ +import { access, readFile } from "node:fs/promises"; +import path from "node:path"; + +const COMMAND_NAME = "verify:m054:s04" as const; +const PACKAGE_JSON_PATH = path.resolve(import.meta.dir, "../package.json"); +const CANONICAL_SCRIPT_COMMAND = "bun scripts/verify-m054-s04.ts" as const; + +const AUDITED_MILESTONES = [ + "M035", + "M036", + "M037", + "M038", + "M039", + "M040", + "M041", + "M042", + "M043", + "M044", + "M045", + "M046", + "M047", + "M048", + "M049", + "M050", + "M051", + "M052", +] as const; + +export const M054_S04_CHECK_IDS = [ + "M054-S04-COMPLETED-MILESTONE-COVERAGE", + "M054-S04-PACKAGE-SCRIPT-WIRING", +] as const; + +export type M054S04CheckId = (typeof M054_S04_CHECK_IDS)[number]; +export type CoverageType = "verifier" | "rationale" | "overclaim" | "missing" | "error"; + +export type MilestoneAuditResult = { + milestoneId: (typeof AUDITED_MILESTONES)[number]; + passed: boolean; + coverageType: CoverageType; + status_code: string; + detail?: string; + evidence?: string[]; +}; + +export type Check = { + id: M054S04CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; + milestones?: MilestoneAuditResult[]; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M054S04CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EvaluateOptions = { + generatedAt?: string; + readTextFile?: (filePath: string) => Promise; + fileExists?: (filePath: string) => Promise | boolean; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +type PackageJson = { scripts?: Record }; + +type MilestoneRule = { + rationaleFiles: string[]; + rationaleMatcher?: (artifactTexts: string[]) => { matched: boolean; evidence?: string[] }; + coverageCommands: string[]; +}; + +const MILESTONE_RULES: Record<(typeof AUDITED_MILESTONES)[number], MilestoneRule> = { + M035: { + rationaleFiles: [], + coverageCommands: ["verify:m054:s02"], + }, + M036: { + rationaleFiles: [], + coverageCommands: ["verify:m054:s02", "verify:m036:s01", "verify:m036:s02", "verify:m036:s03"], + }, + M037: { + rationaleFiles: [], + coverageCommands: ["verify:m054:s02", "verify:m037:s01", "verify:m037:s02", "verify:m037:s03"], + }, + M038: { + rationaleFiles: [], + coverageCommands: ["verify:m054:s02", "verify:m038:s02", "verify:m038:s03"], + }, + M039: { + rationaleFiles: [".gsd/milestones/M039/M039-SUMMARY.md"], + rationaleMatcher: (artifactTexts) => matchAllPhrases(artifactTexts, ["no committed `verify-m039-*` harness survives"]), + coverageCommands: ["verify:m054:s02"], + }, + M040: { + rationaleFiles: [], + coverageCommands: ["verify:m054:s02", "verify:m040:s02", "verify:m040:s03"], + }, + M041: { + rationaleFiles: [], + coverageCommands: ["verify:m054:s02", "verify:m041:s02", "verify:m041:s03"], + }, + M042: { + rationaleFiles: [], + coverageCommands: ["verify:m054:s02", "verify:m042:s01", "verify:m042:s02", "verify:m042:s03"], + }, + M043: { + rationaleFiles: [".gsd/milestones/M043/M043-CONTEXT.md"], + rationaleMatcher: (artifactTexts) => matchAllPhrases(artifactTexts, ["no `verify:m043:*` package scripts survive"]), + coverageCommands: ["verify:m054:s02"], + }, + M044: { + rationaleFiles: [], + coverageCommands: ["verify:m044", "verify:m044:s01"], + }, + M045: { + rationaleFiles: [], + coverageCommands: ["verify:m045:s01", "verify:m045:s03"], + }, + M046: { + rationaleFiles: [], + coverageCommands: ["verify:m046", "verify:m046:s01", "verify:m046:s02"], + }, + M047: { + rationaleFiles: [], + coverageCommands: ["verify:m047", "verify:m047:s01", "verify:m047:s02"], + }, + M048: { + rationaleFiles: [], + coverageCommands: ["verify:m054:s03", "verify:m048:s01", "verify:m048:s02", "verify:m048:s03"], + }, + M049: { + rationaleFiles: [], + coverageCommands: ["verify:m054:s03", "verify:m049:s02"], + }, + M050: { + rationaleFiles: [".gsd/milestones/M050/M050-CONTEXT.md", ".gsd/milestones/M050/M050-SUMMARY.md"], + rationaleMatcher: (artifactTexts) => + matchAllPhrases(artifactTexts, [ + "intentionally reused `verify:m048:s01`", + "instead of introducing `verify:m050:*`", + ]), + coverageCommands: ["verify:m054:s03"], + }, + M051: { + rationaleFiles: [".gsd/milestones/M051/M051-SUMMARY.md"], + rationaleMatcher: (artifactTexts) => + matchAnyPhrase(artifactTexts, [ + "closed the remaining m048 operator/verifier truthfulness debt", + "bun run verify:m048:s01", + "bun run verify:m048:s03", + ]), + coverageCommands: ["verify:m054:s03"], + }, + M052: { + rationaleFiles: [".gsd/milestones/M052/M052-SUMMARY.md", ".gsd/milestones/M052/M052-VALIDATION.md"], + coverageCommands: ["verify:m054:s03"], + }, +}; + +export async function evaluateM054S04VerifierCoverage( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const readTextFile = options.readTextFile ?? defaultReadTextFile; + const fileExists = options.fileExists ?? defaultFileExists; + + const packageScriptCheck = await buildPackageScriptCheck(readTextFile); + const packageJson = await readPackageJson(readTextFile); + const milestoneCoverageCheck = await buildMilestoneCoverageCheck({ + readTextFile, + fileExists, + packageJson, + }); + + const checks = [milestoneCoverageCheck, packageScriptCheck]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M054_S04_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM054S04Report(report: EvaluationReport): string { + const lines = [ + "M054 S04 completed-milestone verifier/rationale audit", + `Generated at: ${report.generatedAt}`, + `Verifier/rationale audit: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + + if (check.milestones != null) { + for (const milestone of check.milestones) { + const milestoneVerdict = milestone.passed ? "PASS" : "FAIL"; + lines.push( + ` - ${milestone.milestoneId} ${milestoneVerdict} coverage=${milestone.coverageType} status_code=${milestone.status_code}${milestone.detail ? ` ${milestone.detail}` : ""}`, + ); + } + } + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM054S04ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM054S04VerifierCoverage(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM054S04Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .flatMap((check) => { + const topLevel = `${check.id}:${check.status_code}`; + const milestoneCodes = + check.milestones + ?.filter((milestone) => !milestone.passed) + .map((milestone) => `${milestone.milestoneId}:${milestone.status_code}`) ?? []; + return [topLevel, ...milestoneCodes]; + }) + .join(", "); + stderr.write(`verify:m054:s04 failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM054S04Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +type MilestoneCoverageCheckOptions = { + readTextFile: (filePath: string) => Promise; + fileExists: (filePath: string) => Promise | boolean; + packageJson: PackageJson; +}; + +async function buildMilestoneCoverageCheck( + options: MilestoneCoverageCheckOptions, +): Promise { + const results: MilestoneAuditResult[] = []; + + for (const milestoneId of AUDITED_MILESTONES) { + results.push(await evaluateMilestoneCoverage(milestoneId, options)); + } + + const failures = results.filter((result) => !result.passed); + if (failures.length > 0) { + return { + id: "M054-S04-COMPLETED-MILESTONE-COVERAGE", + passed: false, + skipped: false, + status_code: "completed_milestone_coverage_drift", + detail: failures + .map((failure) => `${failure.milestoneId}:${failure.status_code}`) + .join(", "), + milestones: results, + }; + } + + return { + id: "M054-S04-COMPLETED-MILESTONE-COVERAGE", + passed: true, + skipped: false, + status_code: "completed_milestone_coverage_ok", + detail: `Verified ${AUDITED_MILESTONES.join(", ")} for verifier or rationale coverage without stale overclaims.`, + milestones: results, + }; +} + +async function evaluateMilestoneCoverage( + milestoneId: (typeof AUDITED_MILESTONES)[number], + options: MilestoneCoverageCheckOptions, +): Promise { + const rule = MILESTONE_RULES[milestoneId]; + const artifactTexts: string[] = []; + + for (const artifactPath of rule.rationaleFiles) { + try { + artifactTexts.push(await options.readTextFile(path.resolve(import.meta.dir, `../${artifactPath}`))); + } catch (error) { + return { + milestoneId, + passed: false, + coverageType: "error", + status_code: "artifact_unreadable", + detail: `${artifactPath}: ${normalizeDetail(error)}`, + }; + } + } + + if (rule.rationaleMatcher != null) { + const rationale = rule.rationaleMatcher(artifactTexts); + if (rationale.matched) { + return { + milestoneId, + passed: true, + coverageType: "rationale", + status_code: "explicit_rationale_present", + detail: `Committed artifacts explicitly explain verifier absence or reuse for ${milestoneId}.`, + evidence: rationale.evidence, + }; + } + } + + const claimedSurfaces = extractClaimedVerifierSurfaces(artifactTexts, milestoneId); + if (claimedSurfaces.length > 0) { + const missingClaims = await findMissingClaims(claimedSurfaces, options.packageJson, options.fileExists); + if (missingClaims.length > 0) { + return { + milestoneId, + passed: false, + coverageType: "overclaim", + status_code: "claimed_verifier_missing", + detail: `Missing claimed verifier surfaces: ${missingClaims.join(", ")}`, + evidence: claimedSurfaces, + }; + } + } + + const coverage = await findPresentCoverage(rule.coverageCommands, options.packageJson, options.fileExists); + if (coverage.length > 0) { + return { + milestoneId, + passed: true, + coverageType: "verifier", + status_code: "repo_verifier_coverage_present", + detail: `Repo exposes verifier coverage via ${coverage.join(", ")}.`, + evidence: coverage, + }; + } + + return { + milestoneId, + passed: false, + coverageType: "missing", + status_code: "verifier_or_rationale_missing", + detail: `No explicit rationale and no repo verifier coverage found for ${milestoneId}.`, + }; +} + +async function findMissingClaims( + claims: string[], + packageJson: PackageJson, + fileExists: (filePath: string) => Promise | boolean, +): Promise { + const missing: string[] = []; + + for (const claim of claims) { + if (claim.startsWith("verify:")) { + const command = packageJson.scripts?.[claim]; + if (command == null) { + missing.push(claim); + continue; + } + const scriptPath = getScriptPathFromCommand(command); + if (scriptPath == null || !(await fileExists(path.resolve(import.meta.dir, `../${scriptPath}`)))) { + missing.push(`${claim} -> ${command}`); + } + continue; + } + + if (!(await fileExists(path.resolve(import.meta.dir, `../${claim}`)))) { + missing.push(claim); + } + } + + return missing; +} + +async function findPresentCoverage( + commands: string[], + packageJson: PackageJson, + fileExists: (filePath: string) => Promise | boolean, +): Promise { + const coverage: string[] = []; + + for (const commandName of commands) { + const command = packageJson.scripts?.[commandName]; + if (command == null) { + continue; + } + + const scriptPath = getScriptPathFromCommand(command); + if (scriptPath == null) { + continue; + } + + if (await fileExists(path.resolve(import.meta.dir, `../${scriptPath}`))) { + coverage.push(commandName); + } + } + + return coverage; +} + +function extractClaimedVerifierSurfaces(artifactTexts: string[], milestoneId: string): string[] { + const claims = new Set(); + const commandRegex = /verify:[a-z0-9]+(?::[a-z0-9]+)*/giu; + const scriptRegex = /scripts\/verify-[a-z0-9-]+\.ts/giu; + const milestonePrefix = `verify:${milestoneId.toLowerCase()}`; + + for (const text of artifactTexts) { + for (const match of text.matchAll(commandRegex)) { + const value = match[0].toLowerCase(); + const start = match.index ?? 0; + const trailingText = text.slice(start + value.length, start + value.length + 20).toLowerCase(); + if ( + trailingText.startsWith(":*") || + trailingText.startsWith("*") || + trailingText.startsWith(" family") || + trailingText.startsWith("` family") || + trailingText.startsWith(" script family") + ) { + continue; + } + if (value.startsWith(milestonePrefix)) { + claims.add(value); + } + } + + for (const match of text.matchAll(scriptRegex)) { + const value = match[0].toLowerCase(); + if (value.includes(`verify-${milestoneId.toLowerCase()}`)) { + claims.add(value); + } + } + } + + return [...claims].sort((left, right) => left.localeCompare(right)); +} + +function matchAllPhrases( + artifactTexts: string[], + phrases: string[], +): { matched: boolean; evidence?: string[] } { + const loweredTexts = artifactTexts.map((text) => text.toLowerCase()); + const matched = phrases.every((phrase) => + loweredTexts.some((text) => text.includes(phrase.toLowerCase())), + ); + return { + matched, + evidence: matched ? phrases : undefined, + }; +} + +function matchAnyPhrase( + artifactTexts: string[], + phrases: string[], +): { matched: boolean; evidence?: string[] } { + const loweredTexts = artifactTexts.map((text) => text.toLowerCase()); + const evidence = phrases.filter((phrase) => + loweredTexts.some((text) => text.includes(phrase.toLowerCase())), + ); + return { + matched: evidence.length > 0, + evidence: evidence.length > 0 ? evidence : undefined, + }; +} + +async function buildPackageScriptCheck( + readTextFile: (filePath: string) => Promise, +): Promise { + let packageJsonText: string; + try { + packageJsonText = await readTextFile(PACKAGE_JSON_PATH); + } catch (error) { + return failCheck( + "M054-S04-PACKAGE-SCRIPT-WIRING", + "package_json_unreadable", + error, + ); + } + + let packageJson: PackageJson; + try { + packageJson = JSON.parse(packageJsonText) as PackageJson; + } catch (error) { + return failCheck( + "M054-S04-PACKAGE-SCRIPT-WIRING", + "package_json_malformed", + error, + ); + } + + const actualCommand = packageJson.scripts?.[COMMAND_NAME]; + if (actualCommand == null) { + return failCheck( + "M054-S04-PACKAGE-SCRIPT-WIRING", + "package_script_wiring_missing", + `package.json is missing scripts.${COMMAND_NAME}`, + ); + } + + if (actualCommand !== CANONICAL_SCRIPT_COMMAND) { + return failCheck( + "M054-S04-PACKAGE-SCRIPT-WIRING", + "package_script_wiring_mismatch", + `Expected ${CANONICAL_SCRIPT_COMMAND} but found ${actualCommand}`, + ); + } + + return passCheck( + "M054-S04-PACKAGE-SCRIPT-WIRING", + "package_script_wiring_ok", + `package.json scripts.${COMMAND_NAME} matches the canonical command.`, + ); +} + +async function readPackageJson( + readTextFile: (filePath: string) => Promise, +): Promise { + try { + return JSON.parse(await readTextFile(PACKAGE_JSON_PATH)) as PackageJson; + } catch { + return {}; + } +} + +function getScriptPathFromCommand(command: string): string | null { + const match = command.match(/^bun\s+(.+)$/u); + if (match == null) { + return null; + } + + const candidate = match[1]?.trim(); + if (candidate == null || !candidate.startsWith("scripts/")) { + return null; + } + + return candidate; +} + +function passCheck(id: M054S04CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M054S04CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +async function defaultFileExists(filePath: string): Promise { + try { + await access(filePath); + return true; + } catch { + return false; + } +} + +if (import.meta.main) { + try { + const args = parseM054S04Args(process.argv.slice(2)); + const { exitCode } = await buildM054S04ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`verify:m054:s04 failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m055-s01.test.ts b/scripts/verify-m055-s01.test.ts new file mode 100644 index 00000000..2dd04480 --- /dev/null +++ b/scripts/verify-m055-s01.test.ts @@ -0,0 +1,299 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport } from "./verify-m055-s01.ts"; +import { + M055_S01_CHECK_IDS, + buildM055S01ProofHarness, + evaluateM055S01DocsTruth, + parseM055S01Args, + renderM055S01Report, +} from "./verify-m055-s01.ts"; + +const EXPECTED_CHECK_IDS = [ + "M055-S01-README-SHIPPED-COUNT", + "M055-S01-README-RECENT-FEATURES", + "M055-S01-README-NIGHTLY-WORKFLOWS", + "M055-S01-CHANGELOG-RECENT-RELEASES", + "M055-S01-PACKAGE-WIRING", +] as const; + +const PASSING_README = `# Kodiai + +Kodiai is an installable GitHub App that delivers AI-powered code review, conversational assistance, issue intelligence, and Slack integration. One installation replaces per-repo workflow YAML — configure behavior with an optional .kodiai.yml file. + +31 milestones shipped (v0.1 through v0.31). See [CHANGELOG.md](CHANGELOG.md) for release history. + +## Recent Shipped Milestones + +- **M051 — manual rereview truthfulness:** @kodiai review is the only supported manual rereview trigger. +- **M052 — Slack webhook relay:** Kodiai can accept authenticated inbound relay payloads. +- **M053 — no dynamic evaluators in src/:** shipped code removes the committed helper that used new Function() and carries bun run verify:m053 as the durable proof command for the invariant. +- **M054 — planning-artifact truth repair:** milestone-specific verifier commands verify:m054:s01 through verify:m054:s04 are shipped. + +## Nightly Workflows + +Two nightly GitHub Actions keep the issue-intelligence surfaces current: + +- nightly-issue-sync runs bun scripts/backfill-issues.ts --sync on a daily cron. +- nightly-reaction-sync runs bun scripts/sync-triage-reactions.ts shortly after issue sync. + +Both workflows also support manual workflow_dispatch runs for testing, and any failures surface through the normal GitHub Actions workflow run status for the repository. +`; + +const PASSING_CHANGELOG = `# Changelog + +## v0.31 (2026-04-21) + +Planning Artifact Repair. + +### Added + +- Milestone-specific verifier family verify:m054:s01 through verify:m054:s04 covering pending-queue truth. + +## v0.30 (2026-04-21) + +No-Dynamic-Evaluator Guardrail. + +### Added + +- bun run verify:m053 as a dedicated proof command enforcing the no-new-Function invariant under importable src code. + +## v0.29 (2026-04-15) + +Explicit Review Lane Hardening. +`; + +const PASSING_PACKAGE_JSON = JSON.stringify( + { + name: "kodiai", + scripts: { + "verify:m055:s01": "bun scripts/verify-m055-s01.ts", + }, + }, + null, + 2, +); + +describe("verify m055 s01 docs truth harness", () => { + test("exports stable check ids and cli parsing", () => { + expect(M055_S01_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM055S01Args([])).toEqual({ json: false }); + expect(parseM055S01Args(["--json"])).toEqual({ json: true }); + expect(() => parseM055S01Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes for the current docs truth contract", async () => { + const report = await evaluateM055S01DocsTruth({ + generatedAt: "2026-04-21T06:30:00.000Z", + readTextFile: async (filePath: string) => { + if (filePath.endsWith("README.md")) return PASSING_README; + if (filePath.endsWith("CHANGELOG.md")) return PASSING_CHANGELOG; + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.command).toBe("verify:m055:s01"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M055-S01-README-SHIPPED-COUNT", + passed: true, + status_code: "readme_shipped_count_ok", + }), + expect.objectContaining({ + id: "M055-S01-README-RECENT-FEATURES", + passed: true, + status_code: "readme_recent_features_ok", + }), + expect.objectContaining({ + id: "M055-S01-README-NIGHTLY-WORKFLOWS", + passed: true, + status_code: "readme_nightly_workflows_ok", + }), + expect.objectContaining({ + id: "M055-S01-CHANGELOG-RECENT-RELEASES", + passed: true, + status_code: "changelog_recent_releases_ok", + }), + expect.objectContaining({ + id: "M055-S01-PACKAGE-WIRING", + passed: true, + status_code: "package_wiring_ok", + }), + ]); + + const rendered = renderM055S01Report(report); + expect(rendered).toContain("Docs truth proof surface: PASS"); + expect(rendered).toContain("M055-S01-README-SHIPPED-COUNT PASS"); + expect(rendered).toContain("M055-S01-README-RECENT-FEATURES PASS"); + expect(rendered).toContain("M055-S01-README-NIGHTLY-WORKFLOWS PASS"); + expect(rendered).toContain("M055-S01-CHANGELOG-RECENT-RELEASES PASS"); + expect(rendered).toContain("M055-S01-PACKAGE-WIRING PASS"); + }); + + test("fails with named status codes for stale doc truths and missing package wiring", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM055S01ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("README.md")) { + return PASSING_README + .replace("31 milestones shipped (v0.1 through v0.31).", "30 milestones shipped (v0.1 through v0.30).") + .replace("Slack webhook relay", "Slack integration") + .replace("nightly-reaction-sync runs bun scripts/sync-triage-reactions.ts shortly after issue sync.", "Nightly workflows keep things current."); + } + if (filePath.endsWith("CHANGELOG.md")) { + return `# Changelog\n\n## v0.29 (2026-04-15)\n\nExplicit Review Lane Hardening.\n`; + } + if (filePath.endsWith("package.json")) { + return JSON.stringify({ name: "kodiai", scripts: {} }); + } + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M055-S01-README-SHIPPED-COUNT", + passed: false, + status_code: "readme_shipped_count_stale", + }), + expect.objectContaining({ + id: "M055-S01-README-RECENT-FEATURES", + passed: false, + status_code: "readme_recent_features_missing", + }), + expect.objectContaining({ + id: "M055-S01-README-NIGHTLY-WORKFLOWS", + passed: false, + status_code: "readme_nightly_workflows_missing", + }), + expect.objectContaining({ + id: "M055-S01-CHANGELOG-RECENT-RELEASES", + passed: false, + status_code: "changelog_recent_releases_missing", + }), + expect.objectContaining({ + id: "M055-S01-PACKAGE-WIRING", + passed: false, + status_code: "package_wiring_missing", + }), + ]); + expect(report.checks[0]?.detail).toContain("31 milestones shipped"); + expect(report.checks[1]?.detail).toContain("Slack webhook relay"); + expect(report.checks[2]?.detail).toContain("nightly-reaction-sync"); + expect(report.checks[3]?.detail).toContain("v0.30"); + expect(report.checks[3]?.detail).toContain("v0.31"); + expect(report.checks[4]?.detail).toContain("verify:m055:s01"); + expect(stderr.join(" ")).toContain("readme_shipped_count_stale"); + expect(stderr.join(" ")).toContain("readme_recent_features_missing"); + expect(stderr.join(" ")).toContain("readme_nightly_workflows_missing"); + expect(stderr.join(" ")).toContain("changelog_recent_releases_missing"); + expect(stderr.join(" ")).toContain("package_wiring_missing"); + }); + + test("surfaces stable malformed-input and unreadable-file failures", async () => { + const missingSections = await evaluateM055S01DocsTruth({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("README.md")) return "# Kodiai\n\nNo shipped count or workflow section here.\n"; + if (filePath.endsWith("CHANGELOG.md")) return "# Changelog\n\nNo recent releases here.\n"; + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(missingSections.checks[0]).toEqual( + expect.objectContaining({ + id: "M055-S01-README-SHIPPED-COUNT", + passed: false, + status_code: "readme_shipped_count_missing", + }), + ); + expect(missingSections.checks[2]).toEqual( + expect.objectContaining({ + id: "M055-S01-README-NIGHTLY-WORKFLOWS", + passed: false, + status_code: "readme_nightly_workflows_missing", + }), + ); + expect(missingSections.checks[3]).toEqual( + expect.objectContaining({ + id: "M055-S01-CHANGELOG-RECENT-RELEASES", + passed: false, + status_code: "changelog_recent_releases_missing", + }), + ); + + const unreadableReadme = await evaluateM055S01DocsTruth({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("README.md")) { + throw new Error("EACCES: README.md"); + } + if (filePath.endsWith("CHANGELOG.md")) return PASSING_CHANGELOG; + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(unreadableReadme.checks[0]).toEqual( + expect.objectContaining({ + id: "M055-S01-README-SHIPPED-COUNT", + passed: false, + status_code: "readme_file_unreadable", + }), + ); + expect(unreadableReadme.checks[1]).toEqual( + expect.objectContaining({ + id: "M055-S01-README-RECENT-FEATURES", + passed: false, + status_code: "readme_file_unreadable", + }), + ); + expect(unreadableReadme.checks[2]).toEqual( + expect.objectContaining({ + id: "M055-S01-README-NIGHTLY-WORKFLOWS", + passed: false, + status_code: "readme_file_unreadable", + }), + ); + + const unreadablePackage = await evaluateM055S01DocsTruth({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("README.md")) return PASSING_README; + if (filePath.endsWith("CHANGELOG.md")) return PASSING_CHANGELOG; + if (filePath.endsWith("package.json")) { + throw new Error("EACCES: package.json"); + } + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(unreadablePackage.checks[4]).toEqual( + expect.objectContaining({ + id: "M055-S01-PACKAGE-WIRING", + passed: false, + status_code: "package_file_unreadable", + }), + ); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m055:s01"]).toBe( + "bun scripts/verify-m055-s01.ts", + ); + }); +}); diff --git a/scripts/verify-m055-s01.ts b/scripts/verify-m055-s01.ts new file mode 100644 index 00000000..c0ebc4d1 --- /dev/null +++ b/scripts/verify-m055-s01.ts @@ -0,0 +1,376 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +const COMMAND_NAME = "verify:m055:s01" as const; +const README_PATH = path.resolve(import.meta.dir, "../README.md"); +const CHANGELOG_PATH = path.resolve(import.meta.dir, "../CHANGELOG.md"); +const PACKAGE_JSON_PATH = path.resolve(import.meta.dir, "../package.json"); +const EXPECTED_SHIPPED_COUNT_LINE = "31 milestones shipped (v0.1 through v0.31)."; +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/verify-m055-s01.ts"; +const REQUIRED_RECENT_FEATURE_MARKERS = [ + "M051", + "@kodiai review", + "M052", + "Slack webhook relay", + "M053", + "new Function()", + "verify:m053", + "M054", + "verify:m054:s01", + "verify:m054:s04", +] as const; +const REQUIRED_NIGHTLY_WORKFLOW_MARKERS = [ + "nightly-issue-sync", + "bun scripts/backfill-issues.ts --sync", + "nightly-reaction-sync", + "bun scripts/sync-triage-reactions.ts", + "workflow_dispatch", + "GitHub Actions workflow run status", +] as const; +const REQUIRED_CHANGELOG_RELEASE_MARKERS = ["## v0.31", "## v0.30", "## v0.29"] as const; + +export const M055_S01_CHECK_IDS = [ + "M055-S01-README-SHIPPED-COUNT", + "M055-S01-README-RECENT-FEATURES", + "M055-S01-README-NIGHTLY-WORKFLOWS", + "M055-S01-CHANGELOG-RECENT-RELEASES", + "M055-S01-PACKAGE-WIRING", +] as const; + +export type M055S01CheckId = (typeof M055_S01_CHECK_IDS)[number]; + +export type Check = { + id: M055S01CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M055S01CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EvaluateOptions = { + generatedAt?: string; + readTextFile?: (filePath: string) => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +export async function evaluateM055S01DocsTruth( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const readTextFile = options.readTextFile ?? defaultReadTextFile; + + let readmeContent: string | null = null; + let readmeReadError: unknown = null; + try { + readmeContent = await readTextFile(README_PATH); + } catch (error) { + readmeReadError = error; + } + + let changelogContent: string | null = null; + let changelogReadError: unknown = null; + try { + changelogContent = await readTextFile(CHANGELOG_PATH); + } catch (error) { + changelogReadError = error; + } + + let packageJsonContent: string | null = null; + let packageJsonReadError: unknown = null; + try { + packageJsonContent = await readTextFile(PACKAGE_JSON_PATH); + } catch (error) { + packageJsonReadError = error; + } + + const checks: Check[] = [ + readmeContent == null + ? failCheck( + "M055-S01-README-SHIPPED-COUNT", + "readme_file_unreadable", + readmeReadError, + ) + : buildReadmeShippedCountCheck(readmeContent), + readmeContent == null + ? failCheck( + "M055-S01-README-RECENT-FEATURES", + "readme_file_unreadable", + readmeReadError, + ) + : buildReadmeRecentFeaturesCheck(readmeContent), + readmeContent == null + ? failCheck( + "M055-S01-README-NIGHTLY-WORKFLOWS", + "readme_file_unreadable", + readmeReadError, + ) + : buildReadmeNightlyWorkflowCheck(readmeContent), + changelogContent == null + ? failCheck( + "M055-S01-CHANGELOG-RECENT-RELEASES", + "changelog_file_unreadable", + changelogReadError, + ) + : buildChangelogRecentReleasesCheck(changelogContent), + packageJsonContent == null + ? failCheck( + "M055-S01-PACKAGE-WIRING", + "package_file_unreadable", + packageJsonReadError, + ) + : buildPackageWiringCheck(packageJsonContent), + ]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M055_S01_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM055S01Report(report: EvaluationReport): string { + const lines = [ + "M055 S01 docs truth verifier", + `Generated at: ${report.generatedAt}`, + `Docs truth proof surface: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM055S01ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM055S01DocsTruth(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM055S01Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`verify:m055:s01 failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM055S01Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +function buildReadmeShippedCountCheck(readmeContent: string): Check { + const actualLine = readmeContent.match(/\b\d+ milestones shipped \(v0\.1 through v0\.\d+\)\./)?.[0]; + + if (actualLine == null) { + return failCheck( + "M055-S01-README-SHIPPED-COUNT", + "readme_shipped_count_missing", + `README.md must include the shipped-count line: ${EXPECTED_SHIPPED_COUNT_LINE}`, + ); + } + + if (actualLine !== EXPECTED_SHIPPED_COUNT_LINE) { + return failCheck( + "M055-S01-README-SHIPPED-COUNT", + "readme_shipped_count_stale", + `Expected '${EXPECTED_SHIPPED_COUNT_LINE}' but found '${actualLine}'.`, + ); + } + + return passCheck( + "M055-S01-README-SHIPPED-COUNT", + "readme_shipped_count_ok", + EXPECTED_SHIPPED_COUNT_LINE, + ); +} + +function buildReadmeRecentFeaturesCheck(readmeContent: string): Check { + const missingMarkers = REQUIRED_RECENT_FEATURE_MARKERS.filter( + (marker) => !readmeContent.includes(marker), + ); + + if (missingMarkers.length > 0) { + return failCheck( + "M055-S01-README-RECENT-FEATURES", + "readme_recent_features_missing", + `README.md is missing recent shipped feature markers: ${missingMarkers.join(", ")}`, + ); + } + + return passCheck( + "M055-S01-README-RECENT-FEATURES", + "readme_recent_features_ok", + `README.md covers recent milestones M051-M054 with the expected feature markers.`, + ); +} + +function buildReadmeNightlyWorkflowCheck(readmeContent: string): Check { + const missingMarkers = REQUIRED_NIGHTLY_WORKFLOW_MARKERS.filter( + (marker) => !readmeContent.includes(marker), + ); + + if (missingMarkers.length > 0) { + return failCheck( + "M055-S01-README-NIGHTLY-WORKFLOWS", + "readme_nightly_workflows_missing", + `README.md is missing nightly workflow markers: ${missingMarkers.join(", ")}`, + ); + } + + return passCheck( + "M055-S01-README-NIGHTLY-WORKFLOWS", + "readme_nightly_workflows_ok", + "README.md documents both nightly workflows, their commands, workflow_dispatch support, and Actions status visibility.", + ); +} + +function buildChangelogRecentReleasesCheck(changelogContent: string): Check { + const missingMarkers = REQUIRED_CHANGELOG_RELEASE_MARKERS.filter( + (marker) => !changelogContent.includes(marker), + ); + + if (missingMarkers.length > 0) { + return failCheck( + "M055-S01-CHANGELOG-RECENT-RELEASES", + "changelog_recent_releases_missing", + `CHANGELOG.md must retain post-v0.29 release entries: ${missingMarkers.join(", ")}`, + ); + } + + return passCheck( + "M055-S01-CHANGELOG-RECENT-RELEASES", + "changelog_recent_releases_ok", + "CHANGELOG.md includes v0.29, v0.30, and v0.31 release entries.", + ); +} + +function buildPackageWiringCheck(packageJsonContent: string): Check { + let packageJson: { scripts?: Record }; + try { + packageJson = JSON.parse(packageJsonContent) as { scripts?: Record }; + } catch (error) { + return failCheck( + "M055-S01-PACKAGE-WIRING", + "package_json_invalid", + error, + ); + } + + const actualScript = packageJson.scripts?.[COMMAND_NAME]; + if (actualScript == null) { + return failCheck( + "M055-S01-PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT}`, + ); + } + + if (actualScript !== EXPECTED_PACKAGE_SCRIPT) { + return failCheck( + "M055-S01-PACKAGE-WIRING", + "package_wiring_incorrect", + `Expected scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT} but found ${actualScript}`, + ); + } + + return passCheck( + "M055-S01-PACKAGE-WIRING", + "package_wiring_ok", + `package.json wires ${COMMAND_NAME} to ${EXPECTED_PACKAGE_SCRIPT}`, + ); +} + +function passCheck(id: M055S01CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M055S01CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +if (import.meta.main) { + try { + const args = parseM055S01Args(process.argv.slice(2)); + const { exitCode } = await buildM055S01ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`verify:m055:s01 failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m055-s02.test.ts b/scripts/verify-m055-s02.test.ts new file mode 100644 index 00000000..121afa4b --- /dev/null +++ b/scripts/verify-m055-s02.test.ts @@ -0,0 +1,314 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport } from "./verify-m055-s02.ts"; +import { + M055_S02_CHECK_IDS, + buildM055S02ProofHarness, + evaluateM055S02DocsTruth, + parseM055S02Args, + renderM055S02Report, +} from "./verify-m055-s02.ts"; + +const EXPECTED_CHECK_IDS = [ + "M055-S02-LICENSE-CONTRACT", + "M055-S02-CONTRIBUTING-PLANNING", + "M055-S02-CONTRIBUTING-MIGRATIONS", + "M055-S02-CONTRIBUTING-VERIFICATION", + "M055-S02-PACKAGE-WIRING", +] as const; + +const PASSING_LICENSE = `KodiAI +Copyright (c) 2026 KodiAI contributors +Proprietary. +All rights reserved. + +This repository and all associated source code, documentation, assets, and configuration files are proprietary. +No license or other right to use, copy, modify, merge, publish, distribute, sublicense, sell, or create derivative works from this repository is granted except through prior written permission from the repository owner. + +If you submit a pull request, patch, issue text, documentation change, code sample, or other contribution to this repository, you represent that you have the necessary rights to submit that material. Unless a separate written agreement says otherwise, you agree that the repository owner may use, modify, adapt, and distribute your submitted contribution as part of this repository and its related materials without any additional obligation to you. Submission of a contribution does not, by itself, transfer ownership of your underlying copyright except to the extent required by separate written agreement. +`; + +const PASSING_CONTRIBUTING = `# Contributing to KodiAI + +KodiAI uses checked-in .gsd/ artifacts to make roadmap, slice, and task intent explicit. + +Current checked-in examples use the naming hierarchy M###, S##, and T##. + +Typical artifact layout: +- .gsd/milestones/M051/M051-ROADMAP.md +- .gsd/milestones/M051/slices/S01/S01-PLAN.md +- .gsd/milestones/M051/slices/S01/tasks/T01-SUMMARY.md +- .gsd/DECISIONS.md +- .gsd/REQUIREMENTS.md + +Migration behavior should match the code in src/db/migrate.ts. +Rollbacks use bun run src/db/migrate.ts down semantics and require a paired .down.sql file. +If a new migration intentionally does not have a rollback file, treat that as an explicit exception. +Do **not** assume every historical migration already meets the paired-file rule. + +The repository uses a mix of broad and targeted proof commands. +Run bun test and bunx tsc --noEmit when typed runtime surfaces change. +Targeted verifier commands such as verify:*, verify:m053, verify:m054:s01, and verify:m055:s01 should also be run when applicable. +See .github/workflows/ci.yml for the current CI verification contract. +`; + +const PASSING_PACKAGE_JSON = JSON.stringify( + { + name: "kodiai", + scripts: { + "verify:m055:s02": "bun scripts/verify-m055-s02.ts", + }, + }, + null, + 2, +); + +describe("verify m055 s02 docs truth harness", () => { + test("exports stable check ids and cli parsing", () => { + expect(M055_S02_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM055S02Args([])).toEqual({ json: false }); + expect(parseM055S02Args(["--json"])).toEqual({ json: true }); + expect(() => parseM055S02Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes for the current docs truth contract", async () => { + const report = await evaluateM055S02DocsTruth({ + generatedAt: "2026-04-21T07:00:00.000Z", + readTextFile: async (filePath: string) => { + if (filePath.endsWith("LICENSE")) return PASSING_LICENSE; + if (filePath.endsWith("CONTRIBUTING.md")) return PASSING_CONTRIBUTING; + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.command).toBe("verify:m055:s02"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M055-S02-LICENSE-CONTRACT", + passed: true, + status_code: "license_contract_ok", + }), + expect.objectContaining({ + id: "M055-S02-CONTRIBUTING-PLANNING", + passed: true, + status_code: "contributing_planning_markers_ok", + }), + expect.objectContaining({ + id: "M055-S02-CONTRIBUTING-MIGRATIONS", + passed: true, + status_code: "contributing_migration_markers_ok", + }), + expect.objectContaining({ + id: "M055-S02-CONTRIBUTING-VERIFICATION", + passed: true, + status_code: "contributing_verification_markers_ok", + }), + expect.objectContaining({ + id: "M055-S02-PACKAGE-WIRING", + passed: true, + status_code: "package_wiring_ok", + }), + ]); + + const rendered = renderM055S02Report(report); + expect(rendered).toContain("Docs contract proof surface: PASS"); + expect(rendered).toContain("M055-S02-LICENSE-CONTRACT PASS"); + expect(rendered).toContain("M055-S02-CONTRIBUTING-PLANNING PASS"); + expect(rendered).toContain("M055-S02-CONTRIBUTING-MIGRATIONS PASS"); + expect(rendered).toContain("M055-S02-CONTRIBUTING-VERIFICATION PASS"); + expect(rendered).toContain("M055-S02-PACKAGE-WIRING PASS"); + }); + + test("fails with named status codes for drifted docs and missing package wiring", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM055S02ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("LICENSE")) { + return PASSING_LICENSE.replace("All rights reserved.", "Some rights reserved."); + } + if (filePath.endsWith("CONTRIBUTING.md")) { + return PASSING_CONTRIBUTING + .replace(".gsd/", "planning artifacts") + .replace(".down.sql", "rollback SQL") + .replace("bunx tsc --noEmit", "tsc"); + } + if (filePath.endsWith("package.json")) { + return JSON.stringify({ name: "kodiai", scripts: {} }); + } + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M055-S02-LICENSE-CONTRACT", + passed: false, + status_code: "license_contract_missing", + }), + expect.objectContaining({ + id: "M055-S02-CONTRIBUTING-PLANNING", + passed: true, + status_code: "contributing_planning_markers_ok", + }), + expect.objectContaining({ + id: "M055-S02-CONTRIBUTING-MIGRATIONS", + passed: false, + status_code: "contributing_migration_markers_missing", + }), + expect.objectContaining({ + id: "M055-S02-CONTRIBUTING-VERIFICATION", + passed: false, + status_code: "contributing_verification_markers_missing", + }), + expect.objectContaining({ + id: "M055-S02-PACKAGE-WIRING", + passed: false, + status_code: "package_wiring_missing", + }), + ]); + expect(report.checks[0]?.detail).toContain("All rights reserved."); + expect(report.checks[1]?.detail).toContain(".gsd artifact model"); + expect(report.checks[2]?.detail).toContain(".down.sql"); + expect(report.checks[3]?.detail).toContain("bunx tsc --noEmit"); + expect(report.checks[4]?.detail).toContain("verify:m055:s02"); + expect(stderr.join(" ")).toContain("license_contract_missing"); + expect(stderr.join(" ")).not.toContain("contributing_planning_markers_missing"); + expect(stderr.join(" ")).toContain("contributing_migration_markers_missing"); + expect(stderr.join(" ")).toContain("contributing_verification_markers_missing"); + expect(stderr.join(" ")).toContain("package_wiring_missing"); + }); + + test("surfaces stable malformed-input and unreadable-file failures", async () => { + const malformedInputs = await evaluateM055S02DocsTruth({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("LICENSE")) return "KodiAI\nProprietary.\n"; + if (filePath.endsWith("CONTRIBUTING.md")) return "# Contributing\n\nNo current workflow markers here.\n"; + if (filePath.endsWith("package.json")) return "{ not valid json"; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(malformedInputs.checks[0]).toEqual( + expect.objectContaining({ + id: "M055-S02-LICENSE-CONTRACT", + passed: false, + status_code: "license_contract_missing", + }), + ); + expect(malformedInputs.checks[1]).toEqual( + expect.objectContaining({ + id: "M055-S02-CONTRIBUTING-PLANNING", + passed: false, + status_code: "contributing_planning_markers_missing", + }), + ); + expect(malformedInputs.checks[2]).toEqual( + expect.objectContaining({ + id: "M055-S02-CONTRIBUTING-MIGRATIONS", + passed: false, + status_code: "contributing_migration_markers_missing", + }), + ); + expect(malformedInputs.checks[3]).toEqual( + expect.objectContaining({ + id: "M055-S02-CONTRIBUTING-VERIFICATION", + passed: false, + status_code: "contributing_verification_markers_missing", + }), + ); + expect(malformedInputs.checks[4]).toEqual( + expect.objectContaining({ + id: "M055-S02-PACKAGE-WIRING", + passed: false, + status_code: "package_json_invalid", + }), + ); + + const unreadableLicense = await evaluateM055S02DocsTruth({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("LICENSE")) throw new Error("EACCES: LICENSE"); + if (filePath.endsWith("CONTRIBUTING.md")) return PASSING_CONTRIBUTING; + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(unreadableLicense.checks[0]).toEqual( + expect.objectContaining({ + id: "M055-S02-LICENSE-CONTRACT", + passed: false, + status_code: "license_file_unreadable", + }), + ); + + const unreadableContributing = await evaluateM055S02DocsTruth({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("LICENSE")) return PASSING_LICENSE; + if (filePath.endsWith("CONTRIBUTING.md")) throw new Error("EACCES: CONTRIBUTING.md"); + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(unreadableContributing.checks[1]).toEqual( + expect.objectContaining({ + id: "M055-S02-CONTRIBUTING-PLANNING", + passed: false, + status_code: "contributing_file_unreadable", + }), + ); + expect(unreadableContributing.checks[2]).toEqual( + expect.objectContaining({ + id: "M055-S02-CONTRIBUTING-MIGRATIONS", + passed: false, + status_code: "contributing_file_unreadable", + }), + ); + expect(unreadableContributing.checks[3]).toEqual( + expect.objectContaining({ + id: "M055-S02-CONTRIBUTING-VERIFICATION", + passed: false, + status_code: "contributing_file_unreadable", + }), + ); + + const unreadablePackage = await evaluateM055S02DocsTruth({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("LICENSE")) return PASSING_LICENSE; + if (filePath.endsWith("CONTRIBUTING.md")) return PASSING_CONTRIBUTING; + if (filePath.endsWith("package.json")) throw new Error("EACCES: package.json"); + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(unreadablePackage.checks[4]).toEqual( + expect.objectContaining({ + id: "M055-S02-PACKAGE-WIRING", + passed: false, + status_code: "package_file_unreadable", + }), + ); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m055:s02"]).toBe( + "bun scripts/verify-m055-s02.ts", + ); + }); +}); diff --git a/scripts/verify-m055-s02.ts b/scripts/verify-m055-s02.ts new file mode 100644 index 00000000..74e3fc9e --- /dev/null +++ b/scripts/verify-m055-s02.ts @@ -0,0 +1,387 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +const COMMAND_NAME = "verify:m055:s02" as const; +const LICENSE_PATH = path.resolve(import.meta.dir, "../LICENSE"); +const CONTRIBUTING_PATH = path.resolve(import.meta.dir, "../CONTRIBUTING.md"); +const PACKAGE_JSON_PATH = path.resolve(import.meta.dir, "../package.json"); +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/verify-m055-s02.ts"; + +const REQUIRED_LICENSE_MARKERS = [ + "Proprietary.", + "All rights reserved.", + "No license or other right to use, copy, modify, merge, publish, distribute, sublicense, sell, or create derivative works from this repository is granted except through prior written permission from the repository owner.", + "Submission of a contribution does not, by itself, transfer ownership of your underlying copyright except to the extent required by separate written agreement.", +] as const; + +const REQUIRED_CONTRIBUTING_PLANNING_MARKERS = [ + ".gsd/", + "M###", + "S##", + "T##", + "ROADMAP", + "PLAN", + "SUMMARY", + ".gsd/DECISIONS.md", + ".gsd/REQUIREMENTS.md", +] as const; + +const REQUIRED_CONTRIBUTING_MIGRATION_MARKERS = [ + "src/db/migrate.ts", + "bun run src/db/migrate.ts down ", + ".down.sql", + "explicit exception", + "Do **not** assume every historical migration already meets the paired-file rule.", +] as const; + +const REQUIRED_CONTRIBUTING_VERIFICATION_MARKERS = [ + "bun test", + "bunx tsc --noEmit", + "verify:*", + "verify:m053", + "verify:m054:s01", + "verify:m055:s01", + ".github/workflows/ci.yml", +] as const; + +export const M055_S02_CHECK_IDS = [ + "M055-S02-LICENSE-CONTRACT", + "M055-S02-CONTRIBUTING-PLANNING", + "M055-S02-CONTRIBUTING-MIGRATIONS", + "M055-S02-CONTRIBUTING-VERIFICATION", + "M055-S02-PACKAGE-WIRING", +] as const; + +export type M055S02CheckId = (typeof M055_S02_CHECK_IDS)[number]; + +export type Check = { + id: M055S02CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M055S02CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EvaluateOptions = { + generatedAt?: string; + readTextFile?: (filePath: string) => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +export async function evaluateM055S02DocsTruth( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const readTextFile = options.readTextFile ?? defaultReadTextFile; + + let licenseContent: string | null = null; + let licenseReadError: unknown = null; + try { + licenseContent = await readTextFile(LICENSE_PATH); + } catch (error) { + licenseReadError = error; + } + + let contributingContent: string | null = null; + let contributingReadError: unknown = null; + try { + contributingContent = await readTextFile(CONTRIBUTING_PATH); + } catch (error) { + contributingReadError = error; + } + + let packageJsonContent: string | null = null; + let packageJsonReadError: unknown = null; + try { + packageJsonContent = await readTextFile(PACKAGE_JSON_PATH); + } catch (error) { + packageJsonReadError = error; + } + + const checks: Check[] = [ + licenseContent == null + ? failCheck( + "M055-S02-LICENSE-CONTRACT", + "license_file_unreadable", + licenseReadError, + ) + : buildLicenseContractCheck(licenseContent), + contributingContent == null + ? failCheck( + "M055-S02-CONTRIBUTING-PLANNING", + "contributing_file_unreadable", + contributingReadError, + ) + : buildContributingPlanningCheck(contributingContent), + contributingContent == null + ? failCheck( + "M055-S02-CONTRIBUTING-MIGRATIONS", + "contributing_file_unreadable", + contributingReadError, + ) + : buildContributingMigrationCheck(contributingContent), + contributingContent == null + ? failCheck( + "M055-S02-CONTRIBUTING-VERIFICATION", + "contributing_file_unreadable", + contributingReadError, + ) + : buildContributingVerificationCheck(contributingContent), + packageJsonContent == null + ? failCheck( + "M055-S02-PACKAGE-WIRING", + "package_file_unreadable", + packageJsonReadError, + ) + : buildPackageWiringCheck(packageJsonContent), + ]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M055_S02_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM055S02Report(report: EvaluationReport): string { + const lines = [ + "M055 S02 docs contract verifier", + `Generated at: ${report.generatedAt}`, + `Docs contract proof surface: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM055S02ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM055S02DocsTruth(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM055S02Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`verify:m055:s02 failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM055S02Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +function buildLicenseContractCheck(licenseContent: string): Check { + const missingMarkers = REQUIRED_LICENSE_MARKERS.filter( + (marker) => !licenseContent.includes(marker), + ); + + if (missingMarkers.length > 0) { + return failCheck( + "M055-S02-LICENSE-CONTRACT", + "license_contract_missing", + `LICENSE is missing contract markers: ${missingMarkers.join(", ")}`, + ); + } + + return passCheck( + "M055-S02-LICENSE-CONTRACT", + "license_contract_ok", + "LICENSE preserves the proprietary, all-rights-reserved, no-grant, and contribution-use contract.", + ); +} + +function buildContributingPlanningCheck(contributingContent: string): Check { + const missingMarkers = REQUIRED_CONTRIBUTING_PLANNING_MARKERS.filter( + (marker) => !contributingContent.includes(marker), + ); + + if (missingMarkers.length > 0) { + return failCheck( + "M055-S02-CONTRIBUTING-PLANNING", + "contributing_planning_markers_missing", + `CONTRIBUTING.md is missing planning markers: ${missingMarkers.join(", ")}`, + ); + } + + return passCheck( + "M055-S02-CONTRIBUTING-PLANNING", + "contributing_planning_markers_ok", + "CONTRIBUTING.md documents the checked-in .gsd artifact model and naming/layout expectations.", + ); +} + +function buildContributingMigrationCheck(contributingContent: string): Check { + const missingMarkers = REQUIRED_CONTRIBUTING_MIGRATION_MARKERS.filter( + (marker) => !contributingContent.includes(marker), + ); + + if (missingMarkers.length > 0) { + return failCheck( + "M055-S02-CONTRIBUTING-MIGRATIONS", + "contributing_migration_markers_missing", + `CONTRIBUTING.md is missing migration markers: ${missingMarkers.join(", ")}`, + ); + } + + return passCheck( + "M055-S02-CONTRIBUTING-MIGRATIONS", + "contributing_migration_markers_ok", + "CONTRIBUTING.md documents src/db/migrate.ts rollback behavior, .down.sql expectations, and historical-drift caveats.", + ); +} + +function buildContributingVerificationCheck(contributingContent: string): Check { + const missingMarkers = REQUIRED_CONTRIBUTING_VERIFICATION_MARKERS.filter( + (marker) => !contributingContent.includes(marker), + ); + + if (missingMarkers.length > 0) { + return failCheck( + "M055-S02-CONTRIBUTING-VERIFICATION", + "contributing_verification_markers_missing", + `CONTRIBUTING.md is missing verification markers: ${missingMarkers.join(", ")}`, + ); + } + + return passCheck( + "M055-S02-CONTRIBUTING-VERIFICATION", + "contributing_verification_markers_ok", + "CONTRIBUTING.md documents baseline verification, targeted verify:* commands, and CI-backed proof expectations.", + ); +} + +function buildPackageWiringCheck(packageJsonContent: string): Check { + let packageJson: { scripts?: Record }; + try { + packageJson = JSON.parse(packageJsonContent) as { + scripts?: Record; + }; + } catch (error) { + return failCheck( + "M055-S02-PACKAGE-WIRING", + "package_json_invalid", + error, + ); + } + + const actualScript = packageJson.scripts?.[COMMAND_NAME]; + if (actualScript == null) { + return failCheck( + "M055-S02-PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT}`, + ); + } + + if (actualScript !== EXPECTED_PACKAGE_SCRIPT) { + return failCheck( + "M055-S02-PACKAGE-WIRING", + "package_wiring_incorrect", + `Expected scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT} but found ${actualScript}`, + ); + } + + return passCheck( + "M055-S02-PACKAGE-WIRING", + "package_wiring_ok", + `package.json wires ${COMMAND_NAME} to ${EXPECTED_PACKAGE_SCRIPT}`, + ); +} + +function passCheck(id: M055S02CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M055S02CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +if (import.meta.main) { + try { + const args = parseM055S02Args(process.argv.slice(2)); + const { exitCode } = await buildM055S02ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`verify:m055:s02 failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m055-s03.test.ts b/scripts/verify-m055-s03.test.ts new file mode 100644 index 00000000..cf1fea82 --- /dev/null +++ b/scripts/verify-m055-s03.test.ts @@ -0,0 +1,396 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import type { EvaluationReport } from "./verify-m055-s03.ts"; +import { + M055_S03_CHECK_IDS, + buildM055S03ProofHarness, + evaluateM055S03DocsTruth, + parseM055S03Args, + renderM055S03Report, +} from "./verify-m055-s03.ts"; + +const EXPECTED_CHECK_IDS = [ + "M055-S03-DOCS-INDEX-INVENTORY", + "M055-S03-REQUIRED-RUNBOOKS-PRESENT", + "M055-S03-RUNBOOK-COMMAND-REFERENCES", + "M055-S03-PACKAGE-WIRING", +] as const; + +const DOC_PATHS = [ + "docs/INDEX.md", + "docs/architecture.md", + "docs/configuration.md", + "docs/deployment.md", + "docs/GRACEFUL-RESTART-RUNBOOK.md", + "docs/guardrails.md", + "docs/issue-intelligence.md", + "docs/knowledge-system.md", + "docs/m029-s04-ops-runbook.md", + "docs/operations/embedding-integrity.md", + "docs/README.md", + "docs/runbooks/aca-job-debugging.md", + "docs/runbooks/deploy-rollback.md", + "docs/runbooks/key-rotation.md", + "docs/runbooks/mentions.md", + "docs/runbooks/nightly-sync-failures.md", + "docs/runbooks/recent-review-audit.md", + "docs/runbooks/review-requested-debug.md", + "docs/runbooks/scale.md", + "docs/runbooks/slack-integration.md", + "docs/runbooks/xbmc-cutover.md", + "docs/runbooks/xbmc-ops.md", + "docs/smoke/phase27-uat-notes.md", + "docs/smoke/phase72-telemetry-follow-through.md", + "docs/smoke/phase74-reliability-regression-gate.md", + "docs/smoke/phase75-live-ops-verification-closure.md", + "docs/smoke/phase80-slack-operator-hardening.md", + "docs/smoke/xbmc-kodiai-write-flow.md", + "docs/smoke/xbmc-xbmc-write-flow.md", +] as const; + +const REQUIRED_RUNBOOKS = [ + "docs/runbooks/deploy-rollback.md", + "docs/runbooks/key-rotation.md", + "docs/runbooks/aca-job-debugging.md", + "docs/runbooks/nightly-sync-failures.md", +] as const; + +const PASSING_INDEX = `# Documentation Index + +This file is the canonical inventory for the checked-in \`docs/\` tree. + +## Inventory + +- [\`docs/INDEX.md\`](INDEX.md) +- [\`docs/architecture.md\`](architecture.md) +- [\`docs/configuration.md\`](configuration.md) +- [\`docs/deployment.md\`](deployment.md) +- [\`docs/GRACEFUL-RESTART-RUNBOOK.md\`](GRACEFUL-RESTART-RUNBOOK.md) +- [\`docs/guardrails.md\`](guardrails.md) +- [\`docs/issue-intelligence.md\`](issue-intelligence.md) +- [\`docs/knowledge-system.md\`](knowledge-system.md) +- [\`docs/m029-s04-ops-runbook.md\`](m029-s04-ops-runbook.md) +- [\`docs/operations/embedding-integrity.md\`](operations/embedding-integrity.md) +- [\`docs/README.md\`](README.md) +- [\`docs/runbooks/aca-job-debugging.md\`](runbooks/aca-job-debugging.md) +- [\`docs/runbooks/deploy-rollback.md\`](runbooks/deploy-rollback.md) +- [\`docs/runbooks/key-rotation.md\`](runbooks/key-rotation.md) +- [\`docs/runbooks/mentions.md\`](runbooks/mentions.md) +- [\`docs/runbooks/nightly-sync-failures.md\`](runbooks/nightly-sync-failures.md) +- [\`docs/runbooks/recent-review-audit.md\`](runbooks/recent-review-audit.md) +- [\`docs/runbooks/review-requested-debug.md\`](runbooks/review-requested-debug.md) +- [\`docs/runbooks/scale.md\`](runbooks/scale.md) +- [\`docs/runbooks/slack-integration.md\`](runbooks/slack-integration.md) +- [\`docs/runbooks/xbmc-cutover.md\`](runbooks/xbmc-cutover.md) +- [\`docs/runbooks/xbmc-ops.md\`](runbooks/xbmc-ops.md) +- [\`docs/smoke/phase27-uat-notes.md\`](smoke/phase27-uat-notes.md) +- [\`docs/smoke/phase72-telemetry-follow-through.md\`](smoke/phase72-telemetry-follow-through.md) +- [\`docs/smoke/phase74-reliability-regression-gate.md\`](smoke/phase74-reliability-regression-gate.md) +- [\`docs/smoke/phase75-live-ops-verification-closure.md\`](smoke/phase75-live-ops-verification-closure.md) +- [\`docs/smoke/phase80-slack-operator-hardening.md\`](smoke/phase80-slack-operator-hardening.md) +- [\`docs/smoke/xbmc-kodiai-write-flow.md\`](smoke/xbmc-kodiai-write-flow.md) +- [\`docs/smoke/xbmc-xbmc-write-flow.md\`](smoke/xbmc-xbmc-write-flow.md) +`; + +const PASSING_DEPLOY_ROLLBACK = `# Deploy and Rollback Runbook + +Use \`az containerapp revision list\` to inspect revisions. +Use \`az containerapp ingress traffic set\` to move traffic. +Use \`bun run src/db/migrate.ts down \` for bounded DB rollback. +Use \`bun run verify:m055:s03\` as the docs proof command. +`; + +const PASSING_KEY_ROTATION = `# Key Rotation Runbook + +Redeploy with \`./deploy.sh\`. +Inspect app state with \`az containerapp show\` and job state with \`az containerapp job show\`. +Use \`gh workflow run nightly-issue-sync.yml\` after rotating workflow secrets. +Confirm embedding surfaces with \`bun run audit:embeddings --json\` and \`bun run repair:embeddings -- --status --corpus review_comments --json\`. +`; + +const PASSING_ACA_JOB_DEBUGGING = `# ACA Job Debugging Runbook + +Read the mounted workspace with \`ls -la /mnt/kodiai-workspaces/\` and \`cat /mnt/kodiai-workspaces//result.json\`. +Use \`bun run scripts/test-aca-job.ts --live\` for the live smoke proof. +Inspect executions with \`az containerapp job execution list\`, \`az containerapp job execution show\`, and the pure-code contract check \`bun run scripts/test-aca-job.ts\`. +`; + +const PASSING_NIGHTLY_SYNC_FAILURES = `# Nightly Sync Failures Runbook + +Issue sync runs \`bun scripts/backfill-issues.ts --sync\`. +Reaction sync runs \`bun scripts/sync-triage-reactions.ts\`. +Manual reruns use \`gh workflow run nightly-issue-sync.yml\` and \`gh workflow run nightly-reaction-sync.yml\`. +Follow-up commands include \`bun run repair:embeddings -- --corpus issues --status --json\`, \`bun run repair:embeddings -- --corpus issue_comments --status --json\`, and \`bun scripts/sync-triage-reactions.ts --days 7 --dry-run\`. +`; + +const PASSING_PACKAGE_JSON = JSON.stringify( + { + name: "kodiai", + scripts: { + "audit:embeddings": "bun scripts/embedding-audit.ts", + "repair:embeddings": "bun scripts/embedding-repair.ts", + "verify:m055:s03": "bun scripts/verify-m055-s03.ts", + }, + }, + null, + 2, +); + +describe("verify m055 s03 docs/runbooks verifier", () => { + test("exports stable check ids and cli parsing", () => { + expect(M055_S03_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM055S03Args([])).toEqual({ json: false }); + expect(parseM055S03Args(["--json"])).toEqual({ json: true }); + expect(() => parseM055S03Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes for the current docs inventory and runbook command contract", async () => { + const report = await evaluateM055S03DocsTruth({ + generatedAt: "2026-04-21T08:00:00.000Z", + readTextFile: async (filePath: string) => { + const normalized = normalize(filePath); + if (normalized === "docs/INDEX.md") return PASSING_INDEX; + if (normalized === "docs/runbooks/deploy-rollback.md") return PASSING_DEPLOY_ROLLBACK; + if (normalized === "docs/runbooks/key-rotation.md") return PASSING_KEY_ROTATION; + if (normalized === "docs/runbooks/aca-job-debugging.md") return PASSING_ACA_JOB_DEBUGGING; + if (normalized === "docs/runbooks/nightly-sync-failures.md") return PASSING_NIGHTLY_SYNC_FAILURES; + if (normalized === "package.json") return PASSING_PACKAGE_JSON; + throw new Error(`Unexpected path: ${filePath}`); + }, + listDocsPaths: async () => [...DOC_PATHS], + fileExists: async (filePath: string) => { + const normalized = normalize(filePath); + return [ + "src/db/migrate.ts", + "scripts/backfill-issues.ts", + "scripts/sync-triage-reactions.ts", + "scripts/test-aca-job.ts", + ].includes(normalized); + }, + }); + + expect(report.command).toBe("verify:m055:s03"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M055-S03-DOCS-INDEX-INVENTORY", + passed: true, + status_code: "docs_index_inventory_ok", + }), + expect.objectContaining({ + id: "M055-S03-REQUIRED-RUNBOOKS-PRESENT", + passed: true, + status_code: "required_runbooks_present", + }), + expect.objectContaining({ + id: "M055-S03-RUNBOOK-COMMAND-REFERENCES", + passed: true, + status_code: "runbook_command_references_ok", + }), + expect.objectContaining({ + id: "M055-S03-PACKAGE-WIRING", + passed: true, + status_code: "package_wiring_ok", + }), + ]); + + const rendered = renderM055S03Report(report); + expect(rendered).toContain("Docs/runbooks proof surface: PASS"); + expect(rendered).toContain("M055-S03-DOCS-INDEX-INVENTORY PASS"); + expect(rendered).toContain("M055-S03-REQUIRED-RUNBOOKS-PRESENT PASS"); + expect(rendered).toContain("M055-S03-RUNBOOK-COMMAND-REFERENCES PASS"); + expect(rendered).toContain("M055-S03-PACKAGE-WIRING PASS"); + }); + + test("fails with stable status codes for inventory drift, missing runbooks, unresolved commands, and missing wiring", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM055S03ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + readTextFile: async (filePath: string) => { + const normalized = normalize(filePath); + if (normalized === "docs/INDEX.md") { + return PASSING_INDEX.replace( + "- [`docs/runbooks/nightly-sync-failures.md`](runbooks/nightly-sync-failures.md)\n", + "", + ); + } + if (normalized === "docs/runbooks/deploy-rollback.md") { + return PASSING_DEPLOY_ROLLBACK.replace( + "`bun run verify:m055:s03`", + "`bun run verify:m055:s99`", + ); + } + if (normalized === "docs/runbooks/key-rotation.md") { + return PASSING_KEY_ROTATION.replace( + "`gh workflow run nightly-issue-sync.yml`", + "`bun scripts/missing-command.ts`", + ); + } + if (normalized === "docs/runbooks/aca-job-debugging.md") { + return PASSING_ACA_JOB_DEBUGGING; + } + if (normalized === "docs/runbooks/nightly-sync-failures.md") { + throw new Error("ENOENT: docs/runbooks/nightly-sync-failures.md"); + } + if (normalized === "package.json") { + return JSON.stringify({ name: "kodiai", scripts: {} }); + } + throw new Error(`Unexpected path: ${filePath}`); + }, + listDocsPaths: async () => [...DOC_PATHS], + fileExists: async () => false, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M055-S03-DOCS-INDEX-INVENTORY", + passed: false, + status_code: "docs_index_inventory_missing_entries", + }), + expect.objectContaining({ + id: "M055-S03-REQUIRED-RUNBOOKS-PRESENT", + passed: false, + status_code: "required_runbooks_missing", + }), + expect.objectContaining({ + id: "M055-S03-RUNBOOK-COMMAND-REFERENCES", + passed: false, + status_code: "runbook_command_references_unresolved", + }), + expect.objectContaining({ + id: "M055-S03-PACKAGE-WIRING", + passed: false, + status_code: "package_wiring_missing", + }), + ]); + expect(report.checks[0]?.detail).toContain("docs/runbooks/nightly-sync-failures.md"); + expect(report.checks[1]?.detail).toContain("docs/runbooks/nightly-sync-failures.md"); + expect(report.checks[2]?.detail).toContain("verify:m055:s99"); + expect(report.checks[2]?.detail).toContain("scripts/missing-command.ts"); + expect(report.checks[3]?.detail).toContain("verify:m055:s03"); + expect(stderr.join(" ")).toContain("docs_index_inventory_missing_entries"); + expect(stderr.join(" ")).toContain("required_runbooks_missing"); + expect(stderr.join(" ")).toContain("runbook_command_references_unresolved"); + expect(stderr.join(" ")).toContain("package_wiring_missing"); + }); + + test("surfaces stable malformed-input and unreadable-file failures", async () => { + const malformed = await evaluateM055S03DocsTruth({ + readTextFile: async (filePath: string) => { + const normalized = normalize(filePath); + if (normalized === "docs/INDEX.md") return "# Documentation Index\n\nNo inventory here.\n"; + if (normalized === "docs/runbooks/deploy-rollback.md") return PASSING_DEPLOY_ROLLBACK; + if (normalized === "docs/runbooks/key-rotation.md") return PASSING_KEY_ROTATION; + if (normalized === "docs/runbooks/aca-job-debugging.md") return PASSING_ACA_JOB_DEBUGGING; + if (normalized === "docs/runbooks/nightly-sync-failures.md") return PASSING_NIGHTLY_SYNC_FAILURES; + if (normalized === "package.json") return "{ not valid json"; + throw new Error(`Unexpected path: ${filePath}`); + }, + listDocsPaths: async () => [...DOC_PATHS], + fileExists: async (filePath: string) => + [ + "src/db/migrate.ts", + "scripts/backfill-issues.ts", + "scripts/sync-triage-reactions.ts", + "scripts/test-aca-job.ts", + ].includes(normalize(filePath)), + }); + + expect(malformed.checks[0]).toEqual( + expect.objectContaining({ + id: "M055-S03-DOCS-INDEX-INVENTORY", + passed: false, + status_code: "docs_index_inventory_missing_entries", + }), + ); + expect(malformed.checks[3]).toEqual( + expect.objectContaining({ + id: "M055-S03-PACKAGE-WIRING", + passed: false, + status_code: "package_json_invalid", + }), + ); + + const unreadableIndex = await evaluateM055S03DocsTruth({ + readTextFile: async (filePath: string) => { + const normalized = normalize(filePath); + if (normalized === "docs/INDEX.md") throw new Error("EACCES: docs/INDEX.md"); + if (normalized === "docs/runbooks/deploy-rollback.md") return PASSING_DEPLOY_ROLLBACK; + if (normalized === "docs/runbooks/key-rotation.md") return PASSING_KEY_ROTATION; + if (normalized === "docs/runbooks/aca-job-debugging.md") return PASSING_ACA_JOB_DEBUGGING; + if (normalized === "docs/runbooks/nightly-sync-failures.md") return PASSING_NIGHTLY_SYNC_FAILURES; + if (normalized === "package.json") return PASSING_PACKAGE_JSON; + throw new Error(`Unexpected path: ${filePath}`); + }, + listDocsPaths: async () => [...DOC_PATHS], + fileExists: async (filePath: string) => + [ + "src/db/migrate.ts", + "scripts/backfill-issues.ts", + "scripts/sync-triage-reactions.ts", + "scripts/test-aca-job.ts", + ].includes(normalize(filePath)), + }); + + expect(unreadableIndex.checks[0]).toEqual( + expect.objectContaining({ + id: "M055-S03-DOCS-INDEX-INVENTORY", + passed: false, + status_code: "docs_index_unreadable", + }), + ); + + const unreadablePackage = await evaluateM055S03DocsTruth({ + readTextFile: async (filePath: string) => { + const normalized = normalize(filePath); + if (normalized === "docs/INDEX.md") return PASSING_INDEX; + if (normalized === "docs/runbooks/deploy-rollback.md") return PASSING_DEPLOY_ROLLBACK; + if (normalized === "docs/runbooks/key-rotation.md") return PASSING_KEY_ROTATION; + if (normalized === "docs/runbooks/aca-job-debugging.md") return PASSING_ACA_JOB_DEBUGGING; + if (normalized === "docs/runbooks/nightly-sync-failures.md") return PASSING_NIGHTLY_SYNC_FAILURES; + if (normalized === "package.json") throw new Error("EACCES: package.json"); + throw new Error(`Unexpected path: ${filePath}`); + }, + listDocsPaths: async () => [...DOC_PATHS], + fileExists: async (filePath: string) => + [ + "src/db/migrate.ts", + "scripts/backfill-issues.ts", + "scripts/sync-triage-reactions.ts", + "scripts/test-aca-job.ts", + ].includes(normalize(filePath)), + }); + + expect(unreadablePackage.checks[3]).toEqual( + expect.objectContaining({ + id: "M055-S03-PACKAGE-WIRING", + passed: false, + status_code: "package_file_unreadable", + }), + ); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m055:s03"]).toBe( + "bun scripts/verify-m055-s03.ts", + ); + }); +}); + +function normalize(filePath: string): string { + return filePath.split(path.sep).join("/").replace(/^.*\/((docs|package\.json|scripts|src)\/?.*)$/, "$1"); +} diff --git a/scripts/verify-m055-s03.ts b/scripts/verify-m055-s03.ts new file mode 100644 index 00000000..3c17b2b2 --- /dev/null +++ b/scripts/verify-m055-s03.ts @@ -0,0 +1,564 @@ +import { access, readFile, readdir } from "node:fs/promises"; +import path from "node:path"; + +const COMMAND_NAME = "verify:m055:s03" as const; +const DOCS_INDEX_PATH = path.resolve(import.meta.dir, "../docs/INDEX.md"); +const PACKAGE_JSON_PATH = path.resolve(import.meta.dir, "../package.json"); +const DOCS_ROOT = path.resolve(import.meta.dir, "../docs"); +const REPO_ROOT = path.resolve(import.meta.dir, ".."); +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/verify-m055-s03.ts"; +const REQUIRED_RUNBOOK_PATHS = [ + path.resolve(import.meta.dir, "../docs/runbooks/deploy-rollback.md"), + path.resolve(import.meta.dir, "../docs/runbooks/key-rotation.md"), + path.resolve(import.meta.dir, "../docs/runbooks/aca-job-debugging.md"), + path.resolve(import.meta.dir, "../docs/runbooks/nightly-sync-failures.md"), +] as const; + +export const M055_S03_CHECK_IDS = [ + "M055-S03-DOCS-INDEX-INVENTORY", + "M055-S03-REQUIRED-RUNBOOKS-PRESENT", + "M055-S03-RUNBOOK-COMMAND-REFERENCES", + "M055-S03-PACKAGE-WIRING", +] as const; + +export type M055S03CheckId = (typeof M055_S03_CHECK_IDS)[number]; + +export type Check = { + id: M055S03CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M055S03CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EvaluateOptions = { + generatedAt?: string; + readTextFile?: (filePath: string) => Promise; + listDocsPaths?: () => Promise; + fileExists?: (filePath: string) => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +type CommandReference = { + runbookPath: string; + command: string; + target: string; + resolution: "package-script" | "typescript-file" | "unresolved"; +}; + +export async function evaluateM055S03DocsTruth( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const readTextFile = options.readTextFile ?? defaultReadTextFile; + const listDocsPaths = options.listDocsPaths ?? defaultListDocsPaths; + const fileExists = options.fileExists ?? defaultFileExists; + + let docsIndexContent: string | null = null; + let docsIndexReadError: unknown = null; + try { + docsIndexContent = await readTextFile(DOCS_INDEX_PATH); + } catch (error) { + docsIndexReadError = error; + } + + let docsPaths: string[] | null = null; + let docsPathsError: unknown = null; + try { + docsPaths = await listDocsPaths(); + } catch (error) { + docsPathsError = error; + } + + const runbookStates = await Promise.all( + REQUIRED_RUNBOOK_PATHS.map(async (runbookPath) => { + try { + const content = await readTextFile(runbookPath); + return { path: runbookPath, content, error: null as unknown }; + } catch (error) { + return { path: runbookPath, content: null as string | null, error }; + } + }), + ); + + let packageJsonContent: string | null = null; + let packageJsonReadError: unknown = null; + try { + packageJsonContent = await readTextFile(PACKAGE_JSON_PATH); + } catch (error) { + packageJsonReadError = error; + } + + const checks: Check[] = [ + docsIndexContent == null + ? failCheck( + "M055-S03-DOCS-INDEX-INVENTORY", + "docs_index_unreadable", + docsIndexReadError, + ) + : docsPaths == null + ? failCheck( + "M055-S03-DOCS-INDEX-INVENTORY", + "docs_tree_unreadable", + docsPathsError, + ) + : buildDocsIndexInventoryCheck(docsIndexContent, docsPaths), + buildRequiredRunbooksPresentCheck(runbookStates), + packageJsonContent == null + ? failCheck( + "M055-S03-RUNBOOK-COMMAND-REFERENCES", + "package_file_unreadable", + packageJsonReadError, + ) + : await buildRunbookCommandReferencesCheck(runbookStates, packageJsonContent, fileExists), + packageJsonContent == null + ? failCheck( + "M055-S03-PACKAGE-WIRING", + "package_file_unreadable", + packageJsonReadError, + ) + : buildPackageWiringCheck(packageJsonContent), + ]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M055_S03_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM055S03Report(report: EvaluationReport): string { + const lines = [ + "M055 S03 docs/runbooks verifier", + `Generated at: ${report.generatedAt}`, + `Docs/runbooks proof surface: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM055S03ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM055S03DocsTruth(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM055S03Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`verify:m055:s03 failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM055S03Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +function buildDocsIndexInventoryCheck(indexContent: string, docsPaths: string[]): Check { + const indexedPaths = parseIndexedDocsPaths(indexContent); + const expectedDocsPaths = [...docsPaths] + .map(normalizeRepoRelativePath) + .filter((candidate) => candidate.startsWith("docs/")) + .sort(); + + const missingEntries = expectedDocsPaths.filter((docPath) => !indexedPaths.has(docPath)); + const extraEntries = [...indexedPaths].filter((docPath) => !expectedDocsPaths.includes(docPath)); + + if (missingEntries.length > 0 || extraEntries.length > 0) { + const detailParts: string[] = []; + if (missingEntries.length > 0) { + detailParts.push(`missing: ${missingEntries.join(", ")}`); + } + if (extraEntries.length > 0) { + detailParts.push(`extra: ${extraEntries.join(", ")}`); + } + + return failCheck( + "M055-S03-DOCS-INDEX-INVENTORY", + "docs_index_inventory_missing_entries", + `docs/INDEX.md inventory drift detected (${detailParts.join("; ")})`, + ); + } + + return passCheck( + "M055-S03-DOCS-INDEX-INVENTORY", + "docs_index_inventory_ok", + `docs/INDEX.md inventories ${expectedDocsPaths.length} tracked docs paths.`, + ); +} + +function buildRequiredRunbooksPresentCheck( + runbookStates: Array<{ path: string; content: string | null; error: unknown }>, +): Check { + const missingPaths = runbookStates + .filter((state) => state.content == null) + .map((state) => normalizeRepoRelativePath(state.path)); + + if (missingPaths.length > 0) { + return failCheck( + "M055-S03-REQUIRED-RUNBOOKS-PRESENT", + "required_runbooks_missing", + `Required runbooks are missing or unreadable: ${missingPaths.join(", ")}`, + ); + } + + return passCheck( + "M055-S03-REQUIRED-RUNBOOKS-PRESENT", + "required_runbooks_present", + `All required runbooks are present: ${runbookStates.map((state) => normalizeRepoRelativePath(state.path)).join(", ")}`, + ); +} + +async function buildRunbookCommandReferencesCheck( + runbookStates: Array<{ path: string; content: string | null; error: unknown }>, + packageJsonContent: string, + fileExists: (filePath: string) => Promise, +): Promise { + const readableRunbooks = runbookStates.filter( + (state): state is { path: string; content: string; error: unknown } => state.content != null, + ); + + let packageJson: { scripts?: Record }; + try { + packageJson = JSON.parse(packageJsonContent) as { scripts?: Record }; + } catch (error) { + return failCheck( + "M055-S03-RUNBOOK-COMMAND-REFERENCES", + "package_json_invalid", + error, + ); + } + + const references = await collectCommandReferences(readableRunbooks, packageJson, fileExists); + const unresolved = references.filter((reference) => reference.resolution === "unresolved"); + + if (unresolved.length > 0) { + return failCheck( + "M055-S03-RUNBOOK-COMMAND-REFERENCES", + "runbook_command_references_unresolved", + unresolved + .map( + (reference) => + `${reference.target} from ${reference.runbookPath} via \`${reference.command}\``, + ) + .join("; "), + ); + } + + return passCheck( + "M055-S03-RUNBOOK-COMMAND-REFERENCES", + "runbook_command_references_ok", + references.length === 0 + ? "No Bun/package-script command references were detected in the required runbooks." + : `Resolved ${references.length} Bun/package-script command references across required runbooks.`, + ); +} + +function buildPackageWiringCheck(packageJsonContent: string): Check { + let packageJson: { scripts?: Record }; + try { + packageJson = JSON.parse(packageJsonContent) as { scripts?: Record }; + } catch (error) { + return failCheck( + "M055-S03-PACKAGE-WIRING", + "package_json_invalid", + error, + ); + } + + const actualScript = packageJson.scripts?.[COMMAND_NAME]; + if (actualScript == null) { + return failCheck( + "M055-S03-PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT}`, + ); + } + + if (actualScript !== EXPECTED_PACKAGE_SCRIPT) { + return failCheck( + "M055-S03-PACKAGE-WIRING", + "package_wiring_incorrect", + `Expected scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT} but found ${actualScript}`, + ); + } + + return passCheck( + "M055-S03-PACKAGE-WIRING", + "package_wiring_ok", + `package.json wires ${COMMAND_NAME} to ${EXPECTED_PACKAGE_SCRIPT}`, + ); +} + +async function collectCommandReferences( + runbookStates: Array<{ path: string; content: string; error: unknown }>, + packageJson: { scripts?: Record }, + fileExists: (filePath: string) => Promise, +): Promise { + const scripts = packageJson.scripts ?? {}; + const references: CommandReference[] = []; + const seen = new Set(); + + for (const runbookState of runbookStates) { + const runbookPath = normalizeRepoRelativePath(runbookState.path); + for (const command of extractCandidateCommands(runbookState.content)) { + const target = extractResolvableTarget(command); + if (target == null) { + continue; + } + + const key = `${runbookPath}:${command}`; + if (seen.has(key)) { + continue; + } + seen.add(key); + + const resolution = await resolveCommandTarget(target, scripts, fileExists); + references.push({ runbookPath, command, target, resolution }); + } + } + + return references; +} + +function extractCandidateCommands(markdown: string): string[] { + const matches = [ + ...markdown.matchAll(/```bash\n([\s\S]*?)```/g), + ...markdown.matchAll(/`([^`\n]+)`/g), + ]; + + const commands = new Set(); + + for (const match of matches) { + const blockOrInline = match[1]?.trim(); + if (blockOrInline == null || blockOrInline.length === 0) { + continue; + } + + const lines = blockOrInline + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0 && !line.startsWith("#")); + + for (const line of lines) { + if (line.startsWith("bun ")) { + commands.add(line); + } + } + } + + return [...commands]; +} + +function extractResolvableTarget(command: string): string | null { + let match = command.match(/^bun\s+run\s+([a-z0-9:-]+)(?:\s|$)/i); + if (match?.[1] != null && !match[1].includes("/") && !match[1].includes(".")) { + return match[1]; + } + + match = command.match(/^bun\s+run\s+((?:src|scripts)\/[^\s]+\.ts)(?:\s|$)/i); + if (match?.[1] != null) { + return match[1]; + } + + match = command.match(/^bun\s+((?:src|scripts)\/[^\s]+\.ts)(?:\s|$)/i); + if (match?.[1] != null) { + return match[1]; + } + + return null; +} + +async function resolveCommandTarget( + target: string, + scripts: Record, + fileExists: (filePath: string) => Promise, +): Promise { + if (!target.includes("/") && scripts[target] != null) { + return "package-script"; + } + + if (target.endsWith(".ts")) { + const absolutePath = path.resolve(REPO_ROOT, target); + if (await fileExists(absolutePath)) { + return "typescript-file"; + } + } + + return "unresolved"; +} + +function parseIndexedDocsPaths(indexContent: string): Set { + const indexedPaths = new Set(); + + for (const match of indexContent.matchAll(/\[([^\]]+)\]\(([^)]+)\)/g)) { + const linkText = match[1]?.trim(); + const href = match[2]?.trim(); + + const candidates = [linkText, href] + .filter((value): value is string => value != null && value.length > 0) + .map((value) => value.replace(/^`|`$/g, "")); + + for (const candidate of candidates) { + const normalized = normalizeIndexedDocsPath(candidate); + if (normalized != null) { + indexedPaths.add(normalized); + } + } + } + + return indexedPaths; +} + +function normalizeIndexedDocsPath(candidate: string): string | null { + if (candidate.startsWith("docs/")) { + return normalizeRepoRelativePath(candidate); + } + + if (candidate.startsWith("./")) { + return normalizeRepoRelativePath(path.posix.join("docs", candidate.slice(2))); + } + + if (candidate.startsWith("../")) { + return null; + } + + if (candidate.endsWith(".md")) { + return normalizeRepoRelativePath(path.posix.join("docs", candidate)); + } + + return null; +} + +function passCheck(id: M055S03CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M055S03CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +function normalizeRepoRelativePath(filePath: string): string { + const relativePath = path.isAbsolute(filePath) ? path.relative(REPO_ROOT, filePath) : filePath; + return relativePath.split(path.sep).join("/"); +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +async function defaultListDocsPaths(): Promise { + const results: string[] = []; + + async function walk(currentDir: string): Promise { + const entries = await readdir(currentDir, { withFileTypes: true }); + for (const entry of entries) { + const absolutePath = path.join(currentDir, entry.name); + if (entry.isDirectory()) { + await walk(absolutePath); + continue; + } + + results.push(normalizeRepoRelativePath(absolutePath)); + } + } + + await walk(DOCS_ROOT); + results.sort(); + return results; +} + +async function defaultFileExists(filePath: string): Promise { + try { + await access(filePath); + return true; + } catch { + return false; + } +} + +if (import.meta.main) { + try { + const args = parseM055S03Args(process.argv.slice(2)); + const { exitCode } = await buildM055S03ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`verify:m055:s03 failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m056-s01.test.ts b/scripts/verify-m056-s01.test.ts new file mode 100644 index 00000000..11b646ae --- /dev/null +++ b/scripts/verify-m056-s01.test.ts @@ -0,0 +1,263 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport, RuntimeResult } from "./verify-m056-s01.ts"; +import { + M056_S01_CHECK_IDS, + buildM056S01ProofHarness, + evaluateM056S01RollbackContract, + parseM056S01Args, + renderM056S01Report, +} from "./verify-m056-s01.ts"; + +const EXPECTED_CHECK_IDS = [ + "M056-S01-ROLLBACK-FILES", + "M056-S01-PACKAGE-WIRING", + "M056-S01-DATABASE-ACCESS", + "M056-S01-ROLLBACK-ROUNDTRIP", +] as const; + +const PASSING_PACKAGE_JSON = JSON.stringify( + { + name: "kodiai", + scripts: { + "verify:m056:s01": "bun scripts/verify-m056-s01.ts", + }, + }, + null, + 2, +); + +const REQUIRED_DOWN_FILES = [ + "src/db/migrations/012-wiki-staleness-run-state.down.sql", + "src/db/migrations/013-review-clusters.down.sql", + "src/db/migrations/016-issue-triage-state.down.sql", + "src/db/migrations/025-wiki-style-cache.down.sql", + "src/db/migrations/026-guardrail-audit.down.sql", +] as const; + +describe("verify m056 s01 rollback contract harness", () => { + test("exports stable check ids and cli parsing", () => { + expect(M056_S01_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM056S01Args([])).toEqual({ json: false }); + expect(parseM056S01Args(["--json"])).toEqual({ json: true }); + expect(() => parseM056S01Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes for the rollback contract when static and runtime checks succeed", async () => { + const report = await evaluateM056S01RollbackContract({ + generatedAt: "2026-04-21T08:30:00.000Z", + env: { TEST_DATABASE_URL: "postgres://test-db" }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith(".down.sql")) return "DROP TABLE IF EXISTS sample;\n"; + throw new Error(`Unexpected path: ${filePath}`); + }, + runRuntimeRoundTrip: async ({ connectionString }) => { + expect(connectionString).toBe("postgres://test-db"); + return { + ok: true, + status_code: "rollback_roundtrip_ok", + detail: + "All targeted tables existed after migrate-up, disappeared after rollback to 11, and returned after re-apply.", + } satisfies RuntimeResult; + }, + }); + + expect(report.command).toBe("verify:m056:s01"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M056-S01-ROLLBACK-FILES", + passed: true, + status_code: "rollback_files_ok", + }), + expect.objectContaining({ + id: "M056-S01-PACKAGE-WIRING", + passed: true, + status_code: "package_wiring_ok", + }), + expect.objectContaining({ + id: "M056-S01-DATABASE-ACCESS", + passed: true, + status_code: "database_url_ok", + }), + expect.objectContaining({ + id: "M056-S01-ROLLBACK-ROUNDTRIP", + passed: true, + status_code: "rollback_roundtrip_ok", + }), + ]); + + const rendered = renderM056S01Report(report); + expect(rendered).toContain("Rollback proof surface: PASS"); + expect(rendered).toContain("M056-S01-ROLLBACK-FILES PASS"); + expect(rendered).toContain("M056-S01-PACKAGE-WIRING PASS"); + expect(rendered).toContain("M056-S01-DATABASE-ACCESS PASS"); + expect(rendered).toContain("M056-S01-ROLLBACK-ROUNDTRIP PASS"); + }); + + test("fails with stable status codes for missing files, package drift, missing db access, and runtime drift", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM056S01ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + env: {}, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return JSON.stringify({ name: "kodiai", scripts: {} }); + } + if (filePath.endsWith("012-wiki-staleness-run-state.down.sql")) { + throw new Error("ENOENT: missing"); + } + return "DROP TABLE IF EXISTS sample;\n"; + }, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M056-S01-ROLLBACK-FILES", + passed: false, + status_code: "rollback_files_missing", + }), + expect.objectContaining({ + id: "M056-S01-PACKAGE-WIRING", + passed: false, + status_code: "package_wiring_missing", + }), + expect.objectContaining({ + id: "M056-S01-DATABASE-ACCESS", + passed: false, + status_code: "database_url_missing", + }), + expect.objectContaining({ + id: "M056-S01-ROLLBACK-ROUNDTRIP", + passed: false, + status_code: "database_url_missing", + }), + ]); + expect(report.checks[0]?.detail).toContain(REQUIRED_DOWN_FILES[0]); + expect(report.checks[1]?.detail).toContain("verify:m056:s01"); + expect(report.checks[2]?.detail).toContain("TEST_DATABASE_URL"); + expect(report.checks[3]?.detail).toContain("runtime check skipped"); + expect(stderr.join(" ")).toContain("rollback_files_missing"); + expect(stderr.join(" ")).toContain("package_wiring_missing"); + expect(stderr.join(" ")).toContain("database_url_missing"); + }); + + test("surfaces invalid json, unreadable files, and runtime failure details", async () => { + const invalidPackage = await evaluateM056S01RollbackContract({ + env: { DATABASE_URL: "postgres://fallback" }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return "{ not valid json"; + if (filePath.endsWith("013-review-clusters.down.sql")) { + throw new Error("EACCES: 013-review-clusters.down.sql"); + } + return "DROP TABLE IF EXISTS sample;\n"; + }, + runRuntimeRoundTrip: async () => ({ + ok: false, + status_code: "rollback_roundtrip_schema_drift", + detail: "Missing after re-apply: review_clusters", + }), + }); + + expect(invalidPackage.checks[0]).toEqual( + expect.objectContaining({ + id: "M056-S01-ROLLBACK-FILES", + passed: false, + status_code: "rollback_files_unreadable", + }), + ); + expect(invalidPackage.checks[1]).toEqual( + expect.objectContaining({ + id: "M056-S01-PACKAGE-WIRING", + passed: false, + status_code: "package_json_invalid", + }), + ); + expect(invalidPackage.checks[2]).toEqual( + expect.objectContaining({ + id: "M056-S01-DATABASE-ACCESS", + passed: true, + status_code: "database_url_ok", + }), + ); + expect(invalidPackage.checks[3]).toEqual( + expect.objectContaining({ + id: "M056-S01-ROLLBACK-ROUNDTRIP", + passed: false, + status_code: "rollback_roundtrip_schema_drift", + }), + ); + + const runtimeError = await evaluateM056S01RollbackContract({ + env: { DATABASE_URL: "postgres://fallback" }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + return "DROP TABLE IF EXISTS sample;\n"; + }, + runRuntimeRoundTrip: async () => { + throw new Error("connect ECONNREFUSED 127.0.0.1:5432"); + }, + }); + + expect(runtimeError.checks[3]).toEqual( + expect.objectContaining({ + id: "M056-S01-ROLLBACK-ROUNDTRIP", + passed: false, + status_code: "database_access_failed", + }), + ); + expect(runtimeError.checks[3]?.detail).toContain("ECONNREFUSED"); + }); + + test("prefers TEST_DATABASE_URL over DATABASE_URL for the runtime round trip", async () => { + let observedConnectionString = ""; + + const report = await evaluateM056S01RollbackContract({ + env: { + TEST_DATABASE_URL: "postgres://preferred-db", + DATABASE_URL: "postgres://fallback-db", + }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + return "DROP TABLE IF EXISTS sample;\n"; + }, + runRuntimeRoundTrip: async ({ connectionString }) => { + observedConnectionString = connectionString; + return { + ok: true, + status_code: "rollback_roundtrip_ok", + detail: "preferred test database used", + } satisfies RuntimeResult; + }, + }); + + expect(observedConnectionString).toBe("postgres://preferred-db"); + expect(report.checks[2]).toEqual( + expect.objectContaining({ + id: "M056-S01-DATABASE-ACCESS", + passed: true, + status_code: "database_url_ok", + }), + ); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m056:s01"]).toBe( + "bun scripts/verify-m056-s01.ts", + ); + }); +}); diff --git a/scripts/verify-m056-s01.ts b/scripts/verify-m056-s01.ts new file mode 100644 index 00000000..dfb2a795 --- /dev/null +++ b/scripts/verify-m056-s01.ts @@ -0,0 +1,443 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import pino from "pino"; +import { createDbClient } from "../src/db/client.ts"; +import { runMigrations, runRollback } from "../src/db/migrate.ts"; + +const COMMAND_NAME = "verify:m056:s01" as const; +const REPO_ROOT = path.resolve(import.meta.dir, ".."); +const PACKAGE_JSON_PATH = path.resolve(REPO_ROOT, "package.json"); +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/verify-m056-s01.ts"; +const TARGET_ROLLBACK_VERSION = 11; +const TARGET_TABLES = [ + "wiki_staleness_run_state", + "review_clusters", + "review_cluster_assignments", + "cluster_run_state", + "issue_triage_state", + "wiki_style_cache", + "guardrail_audit", +] as const; +const REQUIRED_ROLLBACK_FILES = [ + path.resolve(REPO_ROOT, "src/db/migrations/012-wiki-staleness-run-state.down.sql"), + path.resolve(REPO_ROOT, "src/db/migrations/013-review-clusters.down.sql"), + path.resolve(REPO_ROOT, "src/db/migrations/016-issue-triage-state.down.sql"), + path.resolve(REPO_ROOT, "src/db/migrations/025-wiki-style-cache.down.sql"), + path.resolve(REPO_ROOT, "src/db/migrations/026-guardrail-audit.down.sql"), +] as const; + +export const M056_S01_CHECK_IDS = [ + "M056-S01-ROLLBACK-FILES", + "M056-S01-PACKAGE-WIRING", + "M056-S01-DATABASE-ACCESS", + "M056-S01-ROLLBACK-ROUNDTRIP", +] as const; + +export type M056S01CheckId = (typeof M056_S01_CHECK_IDS)[number]; + +export type Check = { + id: M056S01CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M056S01CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +export type RuntimeResult = { + ok: boolean; + status_code: string; + detail?: string; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EnvShape = Partial>; + +type EvaluateOptions = { + generatedAt?: string; + env?: EnvShape; + readTextFile?: (filePath: string) => Promise; + runRuntimeRoundTrip?: (params: { connectionString: string }) => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +type RollbackFileState = { + path: string; + content: string | null; + error: unknown; +}; + +export async function evaluateM056S01RollbackContract( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const sourceEnv = options.env ?? process.env; + const env: EnvShape = { + TEST_DATABASE_URL: sourceEnv.TEST_DATABASE_URL, + DATABASE_URL: sourceEnv.DATABASE_URL, + }; + const readTextFile = options.readTextFile ?? defaultReadTextFile; + const runRuntimeRoundTrip = options.runRuntimeRoundTrip ?? defaultRunRuntimeRoundTrip; + + const rollbackFileStates = await Promise.all( + REQUIRED_ROLLBACK_FILES.map(async (filePath) => { + try { + const content = await readTextFile(filePath); + return { path: filePath, content, error: null } satisfies RollbackFileState; + } catch (error) { + return { path: filePath, content: null, error } satisfies RollbackFileState; + } + }), + ); + + let packageJsonContent: string | null = null; + let packageJsonReadError: unknown = null; + try { + packageJsonContent = await readTextFile(PACKAGE_JSON_PATH); + } catch (error) { + packageJsonReadError = error; + } + + const connectionString = env.TEST_DATABASE_URL ?? env.DATABASE_URL; + + const rollbackFilesCheck = buildRollbackFilesCheck(rollbackFileStates); + const packageWiringCheck = + packageJsonContent == null + ? failCheck("M056-S01-PACKAGE-WIRING", "package_file_unreadable", packageJsonReadError) + : buildPackageWiringCheck(packageJsonContent); + const databaseAccessCheck = buildDatabaseAccessCheck(env); + + let runtimeCheck: Check; + if (!connectionString) { + runtimeCheck = failCheck( + "M056-S01-ROLLBACK-ROUNDTRIP", + "database_url_missing", + "runtime check skipped because neither TEST_DATABASE_URL nor DATABASE_URL is set.", + ); + } else { + try { + const runtimeResult = await runRuntimeRoundTrip({ connectionString }); + runtimeCheck = runtimeResult.ok + ? passCheck("M056-S01-ROLLBACK-ROUNDTRIP", runtimeResult.status_code, runtimeResult.detail) + : failCheck("M056-S01-ROLLBACK-ROUNDTRIP", runtimeResult.status_code, runtimeResult.detail); + } catch (error) { + runtimeCheck = failCheck( + "M056-S01-ROLLBACK-ROUNDTRIP", + classifyRuntimeThrownError(error), + error, + ); + } + } + + const checks = [rollbackFilesCheck, packageWiringCheck, databaseAccessCheck, runtimeCheck]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M056_S01_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM056S01Report(report: EvaluationReport): string { + const lines = [ + "M056 S01 early rollback verifier", + `Generated at: ${report.generatedAt}`, + `Rollback proof surface: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM056S01ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM056S01RollbackContract(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM056S01Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`verify:m056:s01 failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM056S01Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +function buildRollbackFilesCheck(states: RollbackFileState[]): Check { + const missing = states.filter((state) => state.content == null && looksLikeMissingFileError(state.error)); + if (missing.length > 0) { + return failCheck( + "M056-S01-ROLLBACK-FILES", + "rollback_files_missing", + `Missing rollback files: ${missing.map((state) => normalizeRepoRelativePath(state.path)).join(", ")}`, + ); + } + + const unreadable = states.filter((state) => state.content == null); + if (unreadable.length > 0) { + return failCheck( + "M056-S01-ROLLBACK-FILES", + "rollback_files_unreadable", + unreadable + .map((state) => `${normalizeRepoRelativePath(state.path)} (${normalizeDetail(state.error)})`) + .join("; "), + ); + } + + return passCheck( + "M056-S01-ROLLBACK-FILES", + "rollback_files_ok", + `Resolved rollback siblings: ${states.map((state) => normalizeRepoRelativePath(state.path)).join(", ")}`, + ); +} + +function buildPackageWiringCheck(packageJsonContent: string): Check { + let packageJson: { scripts?: Record }; + try { + packageJson = JSON.parse(packageJsonContent) as { scripts?: Record }; + } catch (error) { + return failCheck("M056-S01-PACKAGE-WIRING", "package_json_invalid", error); + } + + const actualScript = packageJson.scripts?.[COMMAND_NAME]; + if (actualScript == null) { + return failCheck( + "M056-S01-PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT}`, + ); + } + + if (actualScript !== EXPECTED_PACKAGE_SCRIPT) { + return failCheck( + "M056-S01-PACKAGE-WIRING", + "package_wiring_incorrect", + `Expected scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT} but found ${actualScript}`, + ); + } + + return passCheck( + "M056-S01-PACKAGE-WIRING", + "package_wiring_ok", + `package.json wires ${COMMAND_NAME} to ${EXPECTED_PACKAGE_SCRIPT}`, + ); +} + +function buildDatabaseAccessCheck(env: EnvShape): Check { + if (env.TEST_DATABASE_URL) { + return passCheck( + "M056-S01-DATABASE-ACCESS", + "database_url_ok", + "Runtime round-trip will use TEST_DATABASE_URL.", + ); + } + + if (env.DATABASE_URL) { + return passCheck( + "M056-S01-DATABASE-ACCESS", + "database_url_ok", + "Runtime round-trip will fall back to DATABASE_URL because TEST_DATABASE_URL is unset.", + ); + } + + return failCheck( + "M056-S01-DATABASE-ACCESS", + "database_url_missing", + "Neither TEST_DATABASE_URL nor DATABASE_URL is set, so the rollback round-trip cannot run.", + ); +} + +async function defaultRunRuntimeRoundTrip({ connectionString }: { connectionString: string }): Promise { + const logger = pino({ level: "silent" }); + const client = createDbClient({ connectionString, logger }); + + try { + await resetMigrationState(client.sql); + await runMigrations(client.sql); + + const afterUp = await readTargetTablePresence(client.sql); + const missingAfterUp = TARGET_TABLES.filter((table) => !afterUp.has(table)); + if (missingAfterUp.length > 0) { + return { + ok: false, + status_code: "rollback_roundtrip_schema_drift", + detail: `Missing after migrate-up: ${missingAfterUp.join(", ")}`, + }; + } + + await runRollback(client.sql, TARGET_ROLLBACK_VERSION); + + const afterRollback = await readTargetTablePresence(client.sql); + const stillPresentAfterRollback = TARGET_TABLES.filter((table) => afterRollback.has(table)); + if (stillPresentAfterRollback.length > 0) { + return { + ok: false, + status_code: "rollback_roundtrip_schema_drift", + detail: `Still present after rollback to ${TARGET_ROLLBACK_VERSION}: ${stillPresentAfterRollback.join(", ")}`, + }; + } + + await runMigrations(client.sql); + + const afterReapply = await readTargetTablePresence(client.sql); + const missingAfterReapply = TARGET_TABLES.filter((table) => !afterReapply.has(table)); + if (missingAfterReapply.length > 0) { + return { + ok: false, + status_code: "rollback_roundtrip_schema_drift", + detail: `Missing after re-apply: ${missingAfterReapply.join(", ")}`, + }; + } + + return { + ok: true, + status_code: "rollback_roundtrip_ok", + detail: + "All targeted tables existed after migrate-up, disappeared after rollback to 11, and returned after re-apply.", + }; + } finally { + await client.close(); + } +} + +async function resetMigrationState(sql: ReturnType["sql"]): Promise { + await sql.unsafe(` + DROP SCHEMA IF EXISTS public CASCADE; + CREATE SCHEMA public; + GRANT ALL ON SCHEMA public TO CURRENT_USER; + GRANT ALL ON SCHEMA public TO PUBLIC; + `); +} + +async function readTargetTablePresence(sql: ReturnType["sql"]): Promise> { + const targetList = [...TARGET_TABLES] + .map((table) => `'${table}'`) + .join(", "); + const rows = await sql.unsafe(` + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name IN (${targetList}) + `) as Array<{ table_name: string }>; + + return new Set(rows.map((row) => row.table_name)); +} + +function classifyRuntimeThrownError(error: unknown): string { + const detail = normalizeDetail(error); + if (detail.includes("Missing rollback file:")) { + return "rollback_artifact_missing"; + } + if (/timeout/i.test(detail)) { + return "database_access_timeout"; + } + return "database_access_failed"; +} + +function looksLikeMissingFileError(error: unknown): boolean { + const detail = normalizeDetail(error); + return detail.includes("ENOENT") || detail.includes("no such file or directory"); +} + +function passCheck(id: M056S01CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M056S01CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +function normalizeRepoRelativePath(filePath: string): string { + const relativePath = path.isAbsolute(filePath) ? path.relative(REPO_ROOT, filePath) : filePath; + return relativePath.split(path.sep).join("/"); +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +if (import.meta.main) { + try { + const args = parseM056S01Args(process.argv.slice(2)); + const { exitCode } = await buildM056S01ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`verify:m056:s01 failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m056-s02.test.ts b/scripts/verify-m056-s02.test.ts new file mode 100644 index 00000000..ba3699ba --- /dev/null +++ b/scripts/verify-m056-s02.test.ts @@ -0,0 +1,315 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport, RuntimeResult } from "./verify-m056-s02.ts"; +import { + M056_S02_CHECK_IDS, + buildM056S02ProofHarness, + evaluateM056S02RollbackContract, + parseM056S02Args, + renderM056S02Report, +} from "./verify-m056-s02.ts"; + +const EXPECTED_CHECK_IDS = [ + "M056-S02-ROLLBACK-FILES", + "M056-S02-SLOT-030", + "M056-S02-PACKAGE-WIRING", + "M056-S02-DATABASE-ACCESS", + "M056-S02-ROLLBACK-ROUNDTRIP", +] as const; + +const PASSING_PACKAGE_JSON = JSON.stringify( + { + name: "kodiai", + scripts: { + "verify:m056:s02": "bun scripts/verify-m056-s02.ts", + }, + }, + null, + 2, +); + +const REQUIRED_DOWN_FILES = [ + "src/db/migrations/033-canonical-code-corpus.down.sql", + "src/db/migrations/034-review-graph.down.sql", + "src/db/migrations/035-generated-rules.down.sql", + "src/db/migrations/036-suggestion-cluster-models.down.sql", +] as const; + +const REQUIRED_SLOT_030_FILES = [ + "src/db/migrations/030-reserved.sql", + "src/db/migrations/030-reserved.down.sql", +] as const; + +describe("verify m056 s02 rollback contract harness", () => { + test("exports stable check ids and cli parsing", () => { + expect(M056_S02_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM056S02Args([])).toEqual({ json: false }); + expect(parseM056S02Args(["--json"])).toEqual({ json: true }); + expect(() => parseM056S02Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes when rollback files, slot 030, package wiring, db access, and runtime checks succeed", async () => { + const report = await evaluateM056S02RollbackContract({ + generatedAt: "2026-04-21T09:00:00.000Z", + env: { TEST_DATABASE_URL: "postgres://test-db" }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("030-reserved.sql")) { + return "-- reserved slot 030 up migration; intentionally schema-neutral\n"; + } + if (filePath.endsWith("030-reserved.down.sql")) { + return "-- reserved slot 030 down migration; intentionally schema-neutral\n"; + } + if (filePath.endsWith(".down.sql")) { + return "DROP TABLE IF EXISTS sample;\n"; + } + throw new Error(`Unexpected path: ${filePath}`); + }, + runRuntimeRoundTrip: async ({ connectionString }) => { + expect(connectionString).toBe("postgres://test-db"); + return { + ok: true, + status_code: "rollback_roundtrip_ok", + detail: + "All targeted tables existed after migrate-up, disappeared after rollback to 32, and returned after re-apply.", + } satisfies RuntimeResult; + }, + }); + + expect(report.command).toBe("verify:m056:s02"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M056-S02-ROLLBACK-FILES", + passed: true, + status_code: "rollback_files_ok", + }), + expect.objectContaining({ + id: "M056-S02-SLOT-030", + passed: true, + status_code: "slot_030_ok", + }), + expect.objectContaining({ + id: "M056-S02-PACKAGE-WIRING", + passed: true, + status_code: "package_wiring_ok", + }), + expect.objectContaining({ + id: "M056-S02-DATABASE-ACCESS", + passed: true, + status_code: "database_url_ok", + }), + expect.objectContaining({ + id: "M056-S02-ROLLBACK-ROUNDTRIP", + passed: true, + status_code: "rollback_roundtrip_ok", + }), + ]); + + const rendered = renderM056S02Report(report); + expect(rendered).toContain("Rollback proof surface: PASS"); + expect(rendered).toContain("M056-S02-ROLLBACK-FILES PASS"); + expect(rendered).toContain("M056-S02-SLOT-030 PASS"); + expect(rendered).toContain("M056-S02-PACKAGE-WIRING PASS"); + expect(rendered).toContain("M056-S02-DATABASE-ACCESS PASS"); + expect(rendered).toContain("M056-S02-ROLLBACK-ROUNDTRIP PASS"); + }); + + test("fails with stable status codes for missing rollback files, ambiguous slot 030, package drift, and missing db access", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM056S02ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + env: {}, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return JSON.stringify({ name: "kodiai", scripts: {} }); + } + if (filePath.endsWith("033-canonical-code-corpus.down.sql")) { + throw new Error("ENOENT: missing"); + } + if (filePath.endsWith("030-reserved.down.sql")) { + throw new Error("ENOENT: missing"); + } + return "-- present\n"; + }, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M056-S02-ROLLBACK-FILES", + passed: false, + status_code: "rollback_files_missing", + }), + expect.objectContaining({ + id: "M056-S02-SLOT-030", + passed: false, + status_code: "slot_030_ambiguous", + }), + expect.objectContaining({ + id: "M056-S02-PACKAGE-WIRING", + passed: false, + status_code: "package_wiring_missing", + }), + expect.objectContaining({ + id: "M056-S02-DATABASE-ACCESS", + passed: false, + status_code: "database_url_missing", + }), + expect.objectContaining({ + id: "M056-S02-ROLLBACK-ROUNDTRIP", + passed: false, + status_code: "database_url_missing", + }), + ]); + expect(report.checks[0]?.detail).toContain(REQUIRED_DOWN_FILES[0]); + expect(report.checks[1]?.detail).toContain(REQUIRED_SLOT_030_FILES[1]); + expect(report.checks[2]?.detail).toContain("verify:m056:s02"); + expect(report.checks[3]?.detail).toContain("TEST_DATABASE_URL"); + expect(report.checks[4]?.detail).toContain("runtime check skipped"); + expect(stderr.join(" ")).toContain("rollback_files_missing"); + expect(stderr.join(" ")).toContain("slot_030_ambiguous"); + expect(stderr.join(" ")).toContain("package_wiring_missing"); + expect(stderr.join(" ")).toContain("database_url_missing"); + }); + + test("surfaces invalid json, unreadable slot files, runtime schema drift, and thrown db access failures", async () => { + const invalidPackage = await evaluateM056S02RollbackContract({ + env: { DATABASE_URL: "postgres://fallback" }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return "{ not valid json"; + if (filePath.endsWith("030-reserved.sql")) { + throw new Error("EACCES: 030-reserved.sql"); + } + if (filePath.endsWith(".down.sql")) { + return "DROP TABLE IF EXISTS sample;\n"; + } + return "-- present\n"; + }, + runRuntimeRoundTrip: async () => ({ + ok: false, + status_code: "rollback_roundtrip_schema_drift", + detail: "Still present after rollback to 32: canonical_code_chunks, review_graph_builds", + }), + }); + + expect(invalidPackage.checks[0]).toEqual( + expect.objectContaining({ + id: "M056-S02-ROLLBACK-FILES", + passed: true, + status_code: "rollback_files_ok", + }), + ); + expect(invalidPackage.checks[1]).toEqual( + expect.objectContaining({ + id: "M056-S02-SLOT-030", + passed: false, + status_code: "slot_030_unreadable", + }), + ); + expect(invalidPackage.checks[2]).toEqual( + expect.objectContaining({ + id: "M056-S02-PACKAGE-WIRING", + passed: false, + status_code: "package_json_invalid", + }), + ); + expect(invalidPackage.checks[3]).toEqual( + expect.objectContaining({ + id: "M056-S02-DATABASE-ACCESS", + passed: true, + status_code: "database_url_ok", + }), + ); + expect(invalidPackage.checks[4]).toEqual( + expect.objectContaining({ + id: "M056-S02-ROLLBACK-ROUNDTRIP", + passed: false, + status_code: "rollback_roundtrip_schema_drift", + }), + ); + expect(invalidPackage.checks[4]?.detail).toContain("rollback to 32"); + + const runtimeError = await evaluateM056S02RollbackContract({ + env: { DATABASE_URL: "postgres://fallback" }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("030-reserved.sql")) { + return "-- reserved\n"; + } + if (filePath.endsWith("030-reserved.down.sql")) { + return "-- reserved\n"; + } + return "DROP TABLE IF EXISTS sample;\n"; + }, + runRuntimeRoundTrip: async () => { + throw new Error("connect ECONNREFUSED 127.0.0.1:5432"); + }, + }); + + expect(runtimeError.checks[4]).toEqual( + expect.objectContaining({ + id: "M056-S02-ROLLBACK-ROUNDTRIP", + passed: false, + status_code: "database_access_failed", + }), + ); + expect(runtimeError.checks[4]?.detail).toContain("ECONNREFUSED"); + }); + + test("prefers TEST_DATABASE_URL over DATABASE_URL for the runtime round trip", async () => { + let observedConnectionString = ""; + + const report = await evaluateM056S02RollbackContract({ + env: { + TEST_DATABASE_URL: "postgres://preferred-db", + DATABASE_URL: "postgres://fallback-db", + }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("030-reserved.sql")) { + return "-- reserved\n"; + } + if (filePath.endsWith("030-reserved.down.sql")) { + return "-- reserved\n"; + } + return "DROP TABLE IF EXISTS sample;\n"; + }, + runRuntimeRoundTrip: async ({ connectionString }) => { + observedConnectionString = connectionString; + return { + ok: true, + status_code: "rollback_roundtrip_ok", + detail: "preferred test database used", + } satisfies RuntimeResult; + }, + }); + + expect(observedConnectionString).toBe("postgres://preferred-db"); + expect(report.checks[3]).toEqual( + expect.objectContaining({ + id: "M056-S02-DATABASE-ACCESS", + passed: true, + status_code: "database_url_ok", + }), + ); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m056:s02"]).toBe( + "bun scripts/verify-m056-s02.ts", + ); + }); +}); diff --git a/scripts/verify-m056-s02.ts b/scripts/verify-m056-s02.ts new file mode 100644 index 00000000..df6f0a36 --- /dev/null +++ b/scripts/verify-m056-s02.ts @@ -0,0 +1,551 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import pino from "pino"; +import { createDbClient } from "../src/db/client.ts"; +import { runMigrations, runRollback } from "../src/db/migrate.ts"; + +const COMMAND_NAME = "verify:m056:s02" as const; +const REPO_ROOT = path.resolve(import.meta.dir, ".."); +const PACKAGE_JSON_PATH = path.resolve(REPO_ROOT, "package.json"); +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/verify-m056-s02.ts"; +const TARGET_ROLLBACK_VERSION = 32; +const TARGET_TABLES = [ + "canonical_code_chunks", + "canonical_corpus_backfill_state", + "review_graph_builds", + "review_graph_files", + "review_graph_nodes", + "review_graph_edges", + "generated_rules", + "suggestion_cluster_models", +] as const; +const REQUIRED_ROLLBACK_FILES = [ + path.resolve(REPO_ROOT, "src/db/migrations/033-canonical-code-corpus.down.sql"), + path.resolve(REPO_ROOT, "src/db/migrations/034-review-graph.down.sql"), + path.resolve(REPO_ROOT, "src/db/migrations/035-generated-rules.down.sql"), + path.resolve(REPO_ROOT, "src/db/migrations/036-suggestion-cluster-models.down.sql"), +] as const; +const REQUIRED_SLOT_030_FILES = [ + path.resolve(REPO_ROOT, "src/db/migrations/030-reserved.sql"), + path.resolve(REPO_ROOT, "src/db/migrations/030-reserved.down.sql"), +] as const; +const RESERVED_SLOT_FORBIDDEN_DDL = /\b(create|alter|drop|truncate)\b/i; + +export const M056_S02_CHECK_IDS = [ + "M056-S02-ROLLBACK-FILES", + "M056-S02-SLOT-030", + "M056-S02-PACKAGE-WIRING", + "M056-S02-DATABASE-ACCESS", + "M056-S02-ROLLBACK-ROUNDTRIP", +] as const; + +export type M056S02CheckId = (typeof M056_S02_CHECK_IDS)[number]; + +export type Check = { + id: M056S02CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M056S02CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +export type RuntimeResult = { + ok: boolean; + status_code: string; + detail?: string; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EnvShape = Partial>; + +type EvaluateOptions = { + generatedAt?: string; + env?: EnvShape; + readTextFile?: (filePath: string) => Promise; + runRuntimeRoundTrip?: (params: { connectionString: string }) => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +type FileState = { + path: string; + content: string | null; + error: unknown; +}; + +export async function evaluateM056S02RollbackContract( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const sourceEnv = options.env ?? process.env; + const env: EnvShape = { + TEST_DATABASE_URL: sourceEnv.TEST_DATABASE_URL, + DATABASE_URL: sourceEnv.DATABASE_URL, + }; + const readTextFile = options.readTextFile ?? defaultReadTextFile; + const runRuntimeRoundTrip = options.runRuntimeRoundTrip ?? defaultRunRuntimeRoundTrip; + + const rollbackFileStates = await Promise.all(REQUIRED_ROLLBACK_FILES.map((filePath) => readFileState(filePath, readTextFile))); + const slot030States = await Promise.all(REQUIRED_SLOT_030_FILES.map((filePath) => readFileState(filePath, readTextFile))); + const packageState = await readFileState(PACKAGE_JSON_PATH, readTextFile); + const connectionString = env.TEST_DATABASE_URL ?? env.DATABASE_URL; + + const rollbackFilesCheck = buildRollbackFilesCheck(rollbackFileStates); + const slot030Check = buildSlot030Check(slot030States); + const packageWiringCheck = + packageState.content == null + ? failCheck("M056-S02-PACKAGE-WIRING", "package_file_unreadable", packageState.error) + : buildPackageWiringCheck(packageState.content); + const databaseAccessCheck = buildDatabaseAccessCheck(env); + + let runtimeCheck: Check; + if (!connectionString) { + runtimeCheck = failCheck( + "M056-S02-ROLLBACK-ROUNDTRIP", + "database_url_missing", + "runtime check skipped because neither TEST_DATABASE_URL nor DATABASE_URL is set.", + ); + } else { + try { + const runtimeResult = await runRuntimeRoundTrip({ connectionString }); + runtimeCheck = runtimeResult.ok + ? passCheck("M056-S02-ROLLBACK-ROUNDTRIP", runtimeResult.status_code, runtimeResult.detail) + : failCheck("M056-S02-ROLLBACK-ROUNDTRIP", runtimeResult.status_code, runtimeResult.detail); + } catch (error) { + runtimeCheck = failCheck( + "M056-S02-ROLLBACK-ROUNDTRIP", + classifyRuntimeThrownError(error), + error, + ); + } + } + + const checks = [ + rollbackFilesCheck, + slot030Check, + packageWiringCheck, + databaseAccessCheck, + runtimeCheck, + ]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M056_S02_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM056S02Report(report: EvaluationReport): string { + const lines = [ + "M056 S02 late rollback verifier", + `Generated at: ${report.generatedAt}`, + `Rollback proof surface: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM056S02ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM056S02RollbackContract(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM056S02Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`verify:m056:s02 failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM056S02Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +function buildRollbackFilesCheck(states: FileState[]): Check { + const missing = states.filter((state) => state.content == null && looksLikeMissingFileError(state.error)); + if (missing.length > 0) { + return failCheck( + "M056-S02-ROLLBACK-FILES", + "rollback_files_missing", + `Missing rollback files: ${missing.map((state) => normalizeRepoRelativePath(state.path)).join(", ")}`, + ); + } + + const unreadable = states.filter((state) => state.content == null); + if (unreadable.length > 0) { + return failCheck( + "M056-S02-ROLLBACK-FILES", + "rollback_files_unreadable", + unreadable + .map((state) => `${normalizeRepoRelativePath(state.path)} (${normalizeDetail(state.error)})`) + .join("; "), + ); + } + + return passCheck( + "M056-S02-ROLLBACK-FILES", + "rollback_files_ok", + `Resolved rollback siblings: ${states.map((state) => normalizeRepoRelativePath(state.path)).join(", ")}`, + ); +} + +function buildSlot030Check(states: FileState[]): Check { + const missing = states.filter((state) => state.content == null && looksLikeMissingFileError(state.error)); + if (missing.length > 0) { + return failCheck( + "M056-S02-SLOT-030", + "slot_030_ambiguous", + `Reserved migration slot 030 is ambiguous; missing files: ${missing.map((state) => normalizeRepoRelativePath(state.path)).join(", ")}`, + ); + } + + const unreadable = states.filter((state) => state.content == null); + if (unreadable.length > 0) { + return failCheck( + "M056-S02-SLOT-030", + "slot_030_unreadable", + unreadable + .map((state) => `${normalizeRepoRelativePath(state.path)} (${normalizeDetail(state.error)})`) + .join("; "), + ); + } + + const ddlMutations = states + .filter((state) => RESERVED_SLOT_FORBIDDEN_DDL.test(state.content ?? "")) + .map((state) => normalizeRepoRelativePath(state.path)); + if (ddlMutations.length > 0) { + return failCheck( + "M056-S02-SLOT-030", + "slot_030_not_neutral", + `Reserved migration slot 030 must stay schema-neutral: DDL detected in ${ddlMutations.join(", ")}`, + ); + } + + return passCheck( + "M056-S02-SLOT-030", + "slot_030_ok", + `Reserved migration slot 030 resolves via ${states.map((state) => normalizeRepoRelativePath(state.path)).join(" + ")}`, + ); +} + +function buildPackageWiringCheck(packageJsonContent: string): Check { + let packageJson: { scripts?: Record }; + try { + packageJson = JSON.parse(packageJsonContent) as { scripts?: Record }; + } catch (error) { + return failCheck("M056-S02-PACKAGE-WIRING", "package_json_invalid", error); + } + + const actualScript = packageJson.scripts?.[COMMAND_NAME]; + if (actualScript == null) { + return failCheck( + "M056-S02-PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT}`, + ); + } + + if (actualScript !== EXPECTED_PACKAGE_SCRIPT) { + return failCheck( + "M056-S02-PACKAGE-WIRING", + "package_wiring_incorrect", + `Expected scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT} but found ${actualScript}`, + ); + } + + return passCheck( + "M056-S02-PACKAGE-WIRING", + "package_wiring_ok", + `package.json wires ${COMMAND_NAME} to ${EXPECTED_PACKAGE_SCRIPT}`, + ); +} + +function buildDatabaseAccessCheck(env: EnvShape): Check { + if (env.TEST_DATABASE_URL) { + return passCheck( + "M056-S02-DATABASE-ACCESS", + "database_url_ok", + "Runtime round-trip will use TEST_DATABASE_URL.", + ); + } + + if (env.DATABASE_URL) { + return passCheck( + "M056-S02-DATABASE-ACCESS", + "database_url_ok", + "Runtime round-trip will fall back to DATABASE_URL because TEST_DATABASE_URL is unset.", + ); + } + + return failCheck( + "M056-S02-DATABASE-ACCESS", + "database_url_missing", + "Neither TEST_DATABASE_URL nor DATABASE_URL is set, so the rollback round-trip cannot run.", + ); +} + +async function defaultRunRuntimeRoundTrip({ connectionString }: { connectionString: string }): Promise { + const logger = pino({ level: "silent" }); + const client = createDbClient({ connectionString, logger }); + + try { + await withSilencedConsole(async () => { + await resetMigrationState(client.sql); + await runMigrations(client.sql); + }); + + const afterUp = await readTargetTablePresence(client.sql); + const missingAfterUp = TARGET_TABLES.filter((table) => !afterUp.has(table)); + if (missingAfterUp.length > 0) { + return { + ok: false, + status_code: "rollback_roundtrip_schema_drift", + detail: `Missing after migrate-up: ${missingAfterUp.join(", ")}`, + }; + } + + const slot030MutationAfterUp = await readReservedSlotMutationCount(client.sql); + if (slot030MutationAfterUp !== 0) { + return { + ok: false, + status_code: "slot_030_not_neutral", + detail: `Reserved migration slot 030 mutated schema state after migrate-up: observed ${slot030MutationAfterUp} unexpected public tables with 030-specific names.`, + }; + } + + await withSilencedConsole(async () => { + await runRollback(client.sql, TARGET_ROLLBACK_VERSION); + }); + + const afterRollback = await readTargetTablePresence(client.sql); + const stillPresentAfterRollback = TARGET_TABLES.filter((table) => afterRollback.has(table)); + if (stillPresentAfterRollback.length > 0) { + return { + ok: false, + status_code: "rollback_roundtrip_schema_drift", + detail: `Still present after rollback to ${TARGET_ROLLBACK_VERSION}: ${stillPresentAfterRollback.join(", ")}`, + }; + } + + const slot030MutationAfterRollback = await readReservedSlotMutationCount(client.sql); + if (slot030MutationAfterRollback !== 0) { + return { + ok: false, + status_code: "slot_030_not_neutral", + detail: `Reserved migration slot 030 mutated schema state during rollback to ${TARGET_ROLLBACK_VERSION}: observed ${slot030MutationAfterRollback} unexpected public tables with 030-specific names.`, + }; + } + + await withSilencedConsole(async () => { + await runMigrations(client.sql); + }); + + const afterReapply = await readTargetTablePresence(client.sql); + const missingAfterReapply = TARGET_TABLES.filter((table) => !afterReapply.has(table)); + if (missingAfterReapply.length > 0) { + return { + ok: false, + status_code: "rollback_roundtrip_schema_drift", + detail: `Missing after re-apply: ${missingAfterReapply.join(", ")}`, + }; + } + + const slot030MutationAfterReapply = await readReservedSlotMutationCount(client.sql); + if (slot030MutationAfterReapply !== 0) { + return { + ok: false, + status_code: "slot_030_not_neutral", + detail: `Reserved migration slot 030 mutated schema state after re-apply: observed ${slot030MutationAfterReapply} unexpected public tables with 030-specific names.`, + }; + } + + return { + ok: true, + status_code: "rollback_roundtrip_ok", + detail: + "All targeted tables existed after migrate-up, disappeared after rollback to 32, and returned after re-apply.", + }; + } finally { + await client.close(); + } +} + +async function resetMigrationState(sql: ReturnType["sql"]): Promise { + await sql.unsafe(` + DROP SCHEMA IF EXISTS public CASCADE; + CREATE SCHEMA public; + GRANT ALL ON SCHEMA public TO CURRENT_USER; + GRANT ALL ON SCHEMA public TO PUBLIC; + `); +} + +async function readTargetTablePresence(sql: ReturnType["sql"]): Promise> { + const targetList = [...TARGET_TABLES].map((table) => `'${table}'`).join(", "); + const rows = (await sql.unsafe(` + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name IN (${targetList}) + `)) as Array<{ table_name: string }>; + + return new Set(rows.map((row) => row.table_name)); +} + +async function readReservedSlotMutationCount(sql: ReturnType["sql"]): Promise { + const rows = (await sql.unsafe(` + SELECT COUNT(*)::int AS count + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name LIKE '030%' + `)) as Array<{ count: number }>; + + return rows[0]?.count ?? 0; +} + +function classifyRuntimeThrownError(error: unknown): string { + const detail = normalizeDetail(error); + if (detail.includes("Missing rollback file:")) { + return "rollback_artifact_missing"; + } + if (/timeout/i.test(detail)) { + return "database_access_timeout"; + } + return "database_access_failed"; +} + +function looksLikeMissingFileError(error: unknown): boolean { + const detail = normalizeDetail(error); + return detail.includes("ENOENT") || detail.includes("no such file or directory"); +} + +function passCheck(id: M056S02CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M056S02CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +function normalizeRepoRelativePath(filePath: string): string { + const relativePath = path.isAbsolute(filePath) ? path.relative(REPO_ROOT, filePath) : filePath; + return relativePath.split(path.sep).join("/"); +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +async function withSilencedConsole(fn: () => Promise): Promise { + const originalLog = console.log; + const originalError = console.error; + const originalWarn = console.warn; + + console.log = () => {}; + console.error = () => {}; + console.warn = () => {}; + + try { + return await fn(); + } finally { + console.log = originalLog; + console.error = originalError; + console.warn = originalWarn; + } +} + +async function readFileState( + filePath: string, + readTextFile: (filePath: string) => Promise, +): Promise { + try { + const content = await readTextFile(filePath); + return { path: filePath, content, error: null }; + } catch (error) { + return { path: filePath, content: null, error }; + } +} + +if (import.meta.main) { + try { + const args = parseM056S02Args(process.argv.slice(2)); + const { exitCode } = await buildM056S02ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`verify:m056:s02 failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m056-s03.test.ts b/scripts/verify-m056-s03.test.ts new file mode 100644 index 00000000..451e5743 --- /dev/null +++ b/scripts/verify-m056-s03.test.ts @@ -0,0 +1,328 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { CheckerReport } from "./check-migrations-have-downs.ts"; +import type { EvaluationReport } from "./verify-m056-s03.ts"; +import { + M056_S03_CHECK_IDS, + buildM056S03ProofHarness, + evaluateM056S03Proof, + parseM056S03Args, + renderM056S03Report, +} from "./verify-m056-s03.ts"; + +const EXPECTED_CHECK_IDS = [ + "M056-S03-CHECKER-STATE", + "M056-S03-PACKAGE-WIRING", + "M056-S03-CI-WIRING", + "M056-S03-DECISION-RECORD", + "M056-S03-CONTRIBUTING-TRUTH", +] as const; + +const PASSING_CHECKER_REPORT: CheckerReport = { + command: "check:migrations-have-downs", + generatedAt: "2026-04-21T10:00:00.000Z", + check_ids: [ + "MIGRATIONS-DIR-STATE", + "MIGRATION-ALLOWLIST-STATE", + "MIGRATION-PAIRS", + "PACKAGE-WIRING", + ], + overallPassed: true, + checks: [ + { + id: "MIGRATIONS-DIR-STATE", + passed: true, + skipped: false, + status_code: "migrations_dir_ok", + detail: "Scanned migrations.", + }, + { + id: "MIGRATION-ALLOWLIST-STATE", + passed: true, + skipped: false, + status_code: "allowlist_empty", + detail: "No allowlist entries.", + }, + { + id: "MIGRATION-PAIRS", + passed: true, + skipped: false, + status_code: "all_rollbacks_present", + detail: "All migrations paired.", + }, + { + id: "PACKAGE-WIRING", + passed: true, + skipped: false, + status_code: "package_wiring_ok", + detail: "Package script present.", + }, + ], +}; + +const PASSING_PACKAGE_JSON = JSON.stringify( + { + name: "kodiai", + scripts: { + "check:migrations-have-downs": "bun scripts/check-migrations-have-downs.ts", + "verify:m056:s03": "bun scripts/verify-m056-s03.ts", + }, + }, + null, + 2, +); + +const PASSING_CI = `name: ci +jobs: + test: + steps: + - run: bun install + - run: bun run verify:m056:s03 + - run: bun test --max-concurrency=2 scripts src + - run: bunx tsc --noEmit +`; + +const PASSING_DECISIONS = `# Decisions Register +| # | When | Scope | Decision | Choice | Rationale | Revisable? | Made By | +|---|------|-------|----------|--------|-----------|------------|---------| +| D999 | M056/S03/T02 | migrations | Paired migration contract | Every forward migration requires a rollback sibling or an explicit allowlisted rationale. | The repo and CI should fail closed on unpaired forward migrations unless a recorded exception exists. | Yes | agent | +`; + +const PASSING_CONTRIBUTING = `# Contributing to KodiAI + +## Database Migration Expectations + +For new migrations, add: +- a forward migration: NNN-name.sql +- a rollback migration: NNN-name.down.sql + +If a new migration intentionally does not have a rollback file, record an explicit allowlisted rationale in the migration gate decision/log rather than treating unpaired forward migrations as normal history. + +Run bun run check:migrations-have-downs and bun run verify:m056:s03 when touching migration policy or migration files. +`; + +describe("verify m056 s03 proof harness", () => { + test("exports stable check ids and cli parsing", () => { + expect(M056_S03_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM056S03Args([])).toEqual({ json: false }); + expect(parseM056S03Args(["--json"])).toEqual({ json: true }); + expect(() => parseM056S03Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes for the paired-migration proof surface", async () => { + const report = await evaluateM056S03Proof({ + generatedAt: "2026-04-21T10:05:00.000Z", + runChecker: async () => PASSING_CHECKER_REPORT, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("ci.yml")) return PASSING_CI; + if (filePath.endsWith("DECISIONS.md")) return PASSING_DECISIONS; + if (filePath.endsWith("CONTRIBUTING.md")) return PASSING_CONTRIBUTING; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.command).toBe("verify:m056:s03"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M056-S03-CHECKER-STATE", + passed: true, + status_code: "checker_passed", + }), + expect.objectContaining({ + id: "M056-S03-PACKAGE-WIRING", + passed: true, + status_code: "package_wiring_ok", + }), + expect.objectContaining({ + id: "M056-S03-CI-WIRING", + passed: true, + status_code: "ci_wiring_ok", + }), + expect.objectContaining({ + id: "M056-S03-DECISION-RECORD", + passed: true, + status_code: "decision_record_ok", + }), + expect.objectContaining({ + id: "M056-S03-CONTRIBUTING-TRUTH", + passed: true, + status_code: "contributing_truth_ok", + }), + ]); + + const rendered = renderM056S03Report(report); + expect(rendered).toContain("M056 S03 paired migration proof verifier"); + expect(rendered).toContain("Paired migration proof surface: PASS"); + expect(rendered).toContain("M056-S03-CHECKER-STATE PASS"); + expect(rendered).toContain("M056-S03-PACKAGE-WIRING PASS"); + expect(rendered).toContain("M056-S03-CI-WIRING PASS"); + expect(rendered).toContain("M056-S03-DECISION-RECORD PASS"); + expect(rendered).toContain("M056-S03-CONTRIBUTING-TRUTH PASS"); + }); + + test("fails with stable status codes for checker failure, missing wiring, missing decision marker, and stale docs", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM056S03ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + runChecker: async () => ({ + ...PASSING_CHECKER_REPORT, + overallPassed: false, + checks: PASSING_CHECKER_REPORT.checks.map((check) => + check.id === "MIGRATION-PAIRS" + ? { ...check, passed: false, status_code: "rollback_missing", detail: "Missing rollback sibling for 999-test.sql" } + : check, + ), + }), + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return JSON.stringify({ name: "kodiai", scripts: {} }); + } + if (filePath.endsWith("ci.yml")) { + return `name: ci\njobs:\n test:\n steps:\n - run: bun test --max-concurrency=2 scripts src\n - run: bun run verify:m056:s03\n`; + } + if (filePath.endsWith("DECISIONS.md")) { + return PASSING_DECISIONS.replace("", ""); + } + if (filePath.endsWith("CONTRIBUTING.md")) { + return `${PASSING_CONTRIBUTING}\nHistorical drift still exists for 012-wiki-staleness-run-state.sql and 013-review-clusters.sql.\n`; + } + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M056-S03-CHECKER-STATE", + passed: false, + status_code: "checker_failed", + }), + expect.objectContaining({ + id: "M056-S03-PACKAGE-WIRING", + passed: false, + status_code: "package_wiring_missing", + }), + expect.objectContaining({ + id: "M056-S03-CI-WIRING", + passed: false, + status_code: "ci_verify_step_misordered", + }), + expect.objectContaining({ + id: "M056-S03-DECISION-RECORD", + passed: false, + status_code: "decision_marker_missing", + }), + expect.objectContaining({ + id: "M056-S03-CONTRIBUTING-TRUTH", + passed: false, + status_code: "contributing_truth_stale", + }), + ]); + expect(report.checks[0]?.detail).toContain("rollback_missing"); + expect(report.checks[1]?.detail).toContain("verify:m056:s03"); + expect(report.checks[2]?.detail).toContain("bun run verify:m056:s03"); + expect(report.checks[3]?.detail).toContain("M056-S03-PAIRED-MIGRATION-CONTRACT"); + expect(report.checks[4]?.detail).toContain("012-wiki-staleness-run-state.sql"); + expect(stderr.join(" ")).toContain("checker_failed"); + expect(stderr.join(" ")).toContain("package_wiring_missing"); + expect(stderr.join(" ")).toContain("ci_verify_step_misordered"); + expect(stderr.join(" ")).toContain("decision_marker_missing"); + expect(stderr.join(" ")).toContain("contributing_truth_stale"); + }); + + test("surfaces malformed checker responses, unreadable files, invalid package json, and missing ci step in a clean json envelope", async () => { + const malformedChecker = await evaluateM056S03Proof({ + runChecker: async () => ({ + command: "check:migrations-have-downs", + generatedAt: "2026-04-21T10:00:00.000Z", + check_ids: PASSING_CHECKER_REPORT.check_ids, + overallPassed: true, + checks: "nope", + } as unknown as CheckerReport), + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return "{ not valid json"; + if (filePath.endsWith("ci.yml")) return "name: ci\n"; + if (filePath.endsWith("DECISIONS.md")) return "# Decisions Register\n"; + if (filePath.endsWith("CONTRIBUTING.md")) return "# Contributing\n"; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(malformedChecker.checks[0]).toEqual( + expect.objectContaining({ + id: "M056-S03-CHECKER-STATE", + passed: false, + status_code: "checker_report_invalid", + }), + ); + expect(malformedChecker.checks[1]).toEqual( + expect.objectContaining({ + id: "M056-S03-PACKAGE-WIRING", + passed: false, + status_code: "package_json_invalid", + }), + ); + expect(malformedChecker.checks[2]).toEqual( + expect.objectContaining({ + id: "M056-S03-CI-WIRING", + passed: false, + status_code: "ci_verify_step_missing", + }), + ); + expect(malformedChecker.checks[3]).toEqual( + expect.objectContaining({ + id: "M056-S03-DECISION-RECORD", + passed: false, + status_code: "decision_marker_missing", + }), + ); + expect(malformedChecker.checks[4]).toEqual( + expect.objectContaining({ + id: "M056-S03-CONTRIBUTING-TRUTH", + passed: false, + status_code: "contributing_truth_missing", + }), + ); + + const unreadableFiles = await evaluateM056S03Proof({ + runChecker: async () => { + throw new Error("checker invocation failed"); + }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) throw new Error("EACCES: package.json"); + if (filePath.endsWith("ci.yml")) throw new Error("EACCES: ci.yml"); + if (filePath.endsWith("DECISIONS.md")) throw new Error("EACCES: DECISIONS.md"); + if (filePath.endsWith("CONTRIBUTING.md")) throw new Error("EACCES: CONTRIBUTING.md"); + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(unreadableFiles.checks).toEqual([ + expect.objectContaining({ id: "M056-S03-CHECKER-STATE", status_code: "checker_invocation_failed" }), + expect.objectContaining({ id: "M056-S03-PACKAGE-WIRING", status_code: "package_file_unreadable" }), + expect.objectContaining({ id: "M056-S03-CI-WIRING", status_code: "ci_file_unreadable" }), + expect.objectContaining({ id: "M056-S03-DECISION-RECORD", status_code: "decision_file_unreadable" }), + expect.objectContaining({ id: "M056-S03-CONTRIBUTING-TRUTH", status_code: "contributing_file_unreadable" }), + ]); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m056:s03"]).toBe( + "bun scripts/verify-m056-s03.ts", + ); + }); +}); diff --git a/scripts/verify-m056-s03.ts b/scripts/verify-m056-s03.ts new file mode 100644 index 00000000..8d402a56 --- /dev/null +++ b/scripts/verify-m056-s03.ts @@ -0,0 +1,421 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import type { CheckerReport } from "./check-migrations-have-downs.ts"; +import { evaluateMigrationPairing } from "./check-migrations-have-downs.ts"; + +const COMMAND_NAME = "verify:m056:s03" as const; +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/verify-m056-s03.ts"; +const EXPECTED_CHECKER_COMMAND = "check:migrations-have-downs" as const; +const EXPECTED_CI_STEP = "bun run verify:m056:s03"; +const CI_TEST_STEP_MARKERS = [ + "bun test --max-concurrency=2 scripts", + "bun test --max-concurrency=2 src/knowledge", + "bunx tsc --noEmit", +] as const; +const DECISION_MARKER = "M056-S03-PAIRED-MIGRATION-CONTRACT"; +const DECISION_REQUIRED_TEXT = + "Every forward migration requires a rollback sibling or an explicit allowlisted rationale."; +const REQUIRED_CONTRIBUTING_MARKERS = [ + "NNN-name.sql", + "NNN-name.down.sql", + "explicit allowlisted rationale", + "bun run check:migrations-have-downs", + "bun run verify:m056:s03", +] as const; +const STALE_CONTRIBUTING_MARKERS = [ + "012-wiki-staleness-run-state.sql", + "013-review-clusters.sql", + "016-issue-triage-state.sql", + "025-wiki-style-cache.sql", + "026-guardrail-audit.sql", + "033-canonical-code-corpus.sql", + "034-review-graph.sql", + "035-generated-rules.sql", + "036-suggestion-cluster-models.sql", + "both paired and unpaired historical files", +] as const; + +const REPO_ROOT = path.resolve(import.meta.dir, ".."); +const PACKAGE_JSON_PATH = path.resolve(REPO_ROOT, "package.json"); +const CI_WORKFLOW_PATH = path.resolve(REPO_ROOT, ".github/workflows/ci.yml"); +const DECISIONS_PATH = path.resolve(REPO_ROOT, ".gsd/DECISIONS.md"); +const CONTRIBUTING_PATH = path.resolve(REPO_ROOT, "CONTRIBUTING.md"); + +export const M056_S03_CHECK_IDS = [ + "M056-S03-CHECKER-STATE", + "M056-S03-PACKAGE-WIRING", + "M056-S03-CI-WIRING", + "M056-S03-DECISION-RECORD", + "M056-S03-CONTRIBUTING-TRUTH", +] as const; + +export type M056S03CheckId = (typeof M056_S03_CHECK_IDS)[number]; + +export type Check = { + id: M056S03CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M056S03CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EvaluateOptions = { + generatedAt?: string; + runChecker?: () => Promise; + readTextFile?: (filePath: string) => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +export async function evaluateM056S03Proof( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const runChecker = options.runChecker ?? defaultRunChecker; + const readTextFile = options.readTextFile ?? defaultReadTextFile; + + const checkerCheck = await buildCheckerStateCheck(runChecker); + + const packageContent = await readOptionalTextFile(readTextFile, PACKAGE_JSON_PATH); + const ciContent = await readOptionalTextFile(readTextFile, CI_WORKFLOW_PATH); + const decisionsContent = await readOptionalTextFile(readTextFile, DECISIONS_PATH); + const contributingContent = await readOptionalTextFile(readTextFile, CONTRIBUTING_PATH); + + const checks: Check[] = [ + checkerCheck, + packageContent.ok + ? buildPackageWiringCheck(packageContent.content) + : failCheck("M056-S03-PACKAGE-WIRING", "package_file_unreadable", packageContent.error), + ciContent.ok + ? buildCiWiringCheck(ciContent.content) + : failCheck("M056-S03-CI-WIRING", "ci_file_unreadable", ciContent.error), + decisionsContent.ok + ? buildDecisionRecordCheck(decisionsContent.content) + : failCheck( + "M056-S03-DECISION-RECORD", + "decision_file_unreadable", + decisionsContent.error, + ), + contributingContent.ok + ? buildContributingTruthCheck(contributingContent.content) + : failCheck( + "M056-S03-CONTRIBUTING-TRUTH", + "contributing_file_unreadable", + contributingContent.error, + ), + ]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M056_S03_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM056S03Report(report: EvaluationReport): string { + const lines = [ + "M056 S03 paired migration proof verifier", + `Generated at: ${report.generatedAt}`, + `Paired migration proof surface: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM056S03ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM056S03Proof(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM056S03Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`${COMMAND_NAME} failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM056S03Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +async function buildCheckerStateCheck( + runChecker: () => Promise, +): Promise { + let report: CheckerReport; + try { + report = await runChecker(); + } catch (error) { + return failCheck("M056-S03-CHECKER-STATE", "checker_invocation_failed", error); + } + + if (!isCheckerReport(report)) { + return failCheck( + "M056-S03-CHECKER-STATE", + "checker_report_invalid", + "check:migrations-have-downs did not return the expected report envelope.", + ); + } + + if (!report.overallPassed) { + const failingChecks = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + + return failCheck( + "M056-S03-CHECKER-STATE", + "checker_failed", + failingChecks.length > 0 + ? `check:migrations-have-downs reported failures: ${failingChecks}` + : "check:migrations-have-downs reported overallPassed=false.", + ); + } + + return passCheck( + "M056-S03-CHECKER-STATE", + "checker_passed", + "check:migrations-have-downs passed with a valid machine-readable report envelope.", + ); +} + +function buildPackageWiringCheck(packageJsonContent: string): Check { + let packageJson: { scripts?: Record }; + try { + packageJson = JSON.parse(packageJsonContent) as { scripts?: Record }; + } catch (error) { + return failCheck("M056-S03-PACKAGE-WIRING", "package_json_invalid", error); + } + + const actualScript = packageJson.scripts?.[COMMAND_NAME]; + if (actualScript == null) { + return failCheck( + "M056-S03-PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT}`, + ); + } + + if (actualScript !== EXPECTED_PACKAGE_SCRIPT) { + return failCheck( + "M056-S03-PACKAGE-WIRING", + "package_wiring_incorrect", + `Expected scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT} but found ${actualScript}`, + ); + } + + return passCheck( + "M056-S03-PACKAGE-WIRING", + "package_wiring_ok", + `package.json wires ${COMMAND_NAME} to ${EXPECTED_PACKAGE_SCRIPT}`, + ); +} + +function buildCiWiringCheck(ciContent: string): Check { + const verifyStepIndex = ciContent.indexOf(EXPECTED_CI_STEP); + if (verifyStepIndex === -1) { + return failCheck( + "M056-S03-CI-WIRING", + "ci_verify_step_missing", + `.github/workflows/ci.yml must run ${EXPECTED_CI_STEP} before the broader Bun test steps.`, + ); + } + + for (const marker of CI_TEST_STEP_MARKERS) { + const markerIndex = ciContent.indexOf(marker); + if (markerIndex !== -1 && verifyStepIndex > markerIndex) { + return failCheck( + "M056-S03-CI-WIRING", + "ci_verify_step_misordered", + `${EXPECTED_CI_STEP} must appear before ${marker} in .github/workflows/ci.yml.`, + ); + } + } + + return passCheck( + "M056-S03-CI-WIRING", + "ci_wiring_ok", + `.github/workflows/ci.yml runs ${EXPECTED_CI_STEP} before the broader Bun test steps.`, + ); +} + +function buildDecisionRecordCheck(decisionsContent: string): Check { + if (!decisionsContent.includes(DECISION_MARKER)) { + return failCheck( + "M056-S03-DECISION-RECORD", + "decision_marker_missing", + `.gsd/DECISIONS.md must include the ${DECISION_MARKER} marker alongside the paired-migration rationale.`, + ); + } + + if (!decisionsContent.includes(DECISION_REQUIRED_TEXT)) { + return failCheck( + "M056-S03-DECISION-RECORD", + "decision_contract_text_missing", + `.gsd/DECISIONS.md must record: ${DECISION_REQUIRED_TEXT}`, + ); + } + + return passCheck( + "M056-S03-DECISION-RECORD", + "decision_record_ok", + `.gsd/DECISIONS.md records the paired-migration contract with marker ${DECISION_MARKER}.`, + ); +} + +function buildContributingTruthCheck(contributingContent: string): Check { + const missingMarkers = REQUIRED_CONTRIBUTING_MARKERS.filter( + (marker) => !contributingContent.includes(marker), + ); + if (missingMarkers.length > 0) { + return failCheck( + "M056-S03-CONTRIBUTING-TRUTH", + "contributing_truth_missing", + `CONTRIBUTING.md is missing paired-migration contract markers: ${missingMarkers.join(", ")}`, + ); + } + + const staleMarkers = STALE_CONTRIBUTING_MARKERS.filter((marker) => + contributingContent.includes(marker), + ); + if (staleMarkers.length > 0) { + return failCheck( + "M056-S03-CONTRIBUTING-TRUTH", + "contributing_truth_stale", + `CONTRIBUTING.md still claims historical unpaired-migration drift: ${staleMarkers.join(", ")}`, + ); + } + + return passCheck( + "M056-S03-CONTRIBUTING-TRUTH", + "contributing_truth_ok", + "CONTRIBUTING.md documents the enforced paired-migration contract without stale historical-drift claims.", + ); +} + +function isCheckerReport(value: unknown): value is CheckerReport { + if (typeof value !== "object" || value == null) { + return false; + } + + const candidate = value as Partial; + return ( + candidate.command === EXPECTED_CHECKER_COMMAND && + typeof candidate.generatedAt === "string" && + typeof candidate.overallPassed === "boolean" && + Array.isArray(candidate.check_ids) && + Array.isArray(candidate.checks) + ); +} + +function passCheck(id: M056S03CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M056S03CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +async function readOptionalTextFile( + readTextFile: (filePath: string) => Promise, + filePath: string, +): Promise<{ ok: true; content: string } | { ok: false; error: unknown }> { + try { + return { ok: true, content: await readTextFile(filePath) }; + } catch (error) { + return { ok: false, error }; + } +} + +async function defaultRunChecker(): Promise { + return evaluateMigrationPairing(); +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +if (import.meta.main) { + try { + const args = parseM056S03Args(process.argv.slice(2)); + const { exitCode } = await buildM056S03ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${COMMAND_NAME} failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m057-s02.ts b/scripts/verify-m057-s02.ts new file mode 100644 index 00000000..45e4dbeb --- /dev/null +++ b/scripts/verify-m057-s02.ts @@ -0,0 +1,30 @@ +const COMMAND_NAME = "verify:m057:s02" as const; +const TEST_COMMANDS = [ + ["bun", "test", "./src/handlers/ci-failure.test.ts"], + ["bun", "test", "./src/lib/ci-failure-classifier.test.ts"], +] as const; + +type TestCommand = (typeof TEST_COMMANDS)[number]; + +function formatCommand(command: TestCommand): string { + return command.join(" "); +} + +for (const command of TEST_COMMANDS) { + const [cmd, ...args] = command; + const formatted = formatCommand(command); + process.stdout.write(`→ ${formatted}\n`); + + const result = Bun.spawnSync({ + cmd: [cmd, ...args], + stdout: "inherit", + stderr: "inherit", + }); + + if (result.exitCode !== 0) { + process.stderr.write(`${COMMAND_NAME} failed: ${formatted}\n`); + process.exit(result.exitCode); + } +} + +process.stdout.write(`${COMMAND_NAME} passed\n`); diff --git a/scripts/verify-m057-s03.ts b/scripts/verify-m057-s03.ts new file mode 100644 index 00000000..cae7a3d1 --- /dev/null +++ b/scripts/verify-m057-s03.ts @@ -0,0 +1,32 @@ +const COMMAND_NAME = "verify:m057:s03" as const; +const TEST_COMMANDS = [ + ["bun", "test", "./src/jobs/fork-manager.test.ts"], + ["bun", "test", "./src/jobs/gist-publisher.test.ts"], + ["bun", "test", "./src/slack/write-runner.test.ts"], + ["bun", "test", "./src/handlers/mention.test.ts"], +] as const; + +type TestCommand = (typeof TEST_COMMANDS)[number]; + +function formatCommand(command: TestCommand): string { + return command.join(" "); +} + +for (const command of TEST_COMMANDS) { + const [cmd, ...args] = command; + const formatted = formatCommand(command); + process.stdout.write(`→ ${formatted}\n`); + + const result = Bun.spawnSync({ + cmd: [cmd, ...args], + stdout: "inherit", + stderr: "inherit", + }); + + if (result.exitCode !== 0) { + process.stderr.write(`${COMMAND_NAME} failed: ${formatted}\n`); + process.exit(result.exitCode); + } +} + +process.stdout.write(`${COMMAND_NAME} passed\n`); diff --git a/scripts/verify-m057-s04.test.ts b/scripts/verify-m057-s04.test.ts new file mode 100644 index 00000000..27c87ff4 --- /dev/null +++ b/scripts/verify-m057-s04.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; + +describe("verify m057 s04", () => { + test("exports the expected verifier command bundle", async () => { + const verifier = await import("./verify-m057-s04.ts"); + + expect(verifier.COMMAND_NAME).toBe("verify:m057:s04"); + expect(verifier.VERIFY_COMMANDS).toEqual([ + ["bun", "test", "./scripts/check-orphaned-tests.test.ts"], + ["bun", "test", "./src/execution/executor.test.ts"], + ["bun", "run", "./scripts/check-orphaned-tests.ts", "--json"], + ]); + }); + + test("wires the package verifier command to the local script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m057:s04"]).toBe( + "bun scripts/verify-m057-s04.ts", + ); + }); +}); diff --git a/scripts/verify-m057-s04.ts b/scripts/verify-m057-s04.ts new file mode 100644 index 00000000..5135792a --- /dev/null +++ b/scripts/verify-m057-s04.ts @@ -0,0 +1,36 @@ +export const COMMAND_NAME = "verify:m057:s04" as const; +export const VERIFY_COMMANDS = [ + ["bun", "test", "./scripts/check-orphaned-tests.test.ts"], + ["bun", "test", "./src/execution/executor.test.ts"], + ["bun", "run", "./scripts/check-orphaned-tests.ts", "--json"], +] as const; + +type VerifyCommand = (typeof VERIFY_COMMANDS)[number]; + +function formatCommand(command: readonly string[]): string { + return command.join(" "); +} + +export function runVerifyM057S04(commands: readonly VerifyCommand[] = VERIFY_COMMANDS): void { + for (const command of commands) { + const formatted = formatCommand(command); + process.stdout.write(`→ ${formatted}\n`); + + const result = Bun.spawnSync({ + cmd: [...command], + stdout: "inherit", + stderr: "inherit", + }); + + if (result.exitCode !== 0) { + process.stderr.write(`${COMMAND_NAME} failed: ${formatted}\n`); + process.exit(result.exitCode); + } + } + + process.stdout.write(`${COMMAND_NAME} passed\n`); +} + +if (import.meta.main) { + runVerifyM057S04(); +} diff --git a/scripts/verify-m058-s01.test.ts b/scripts/verify-m058-s01.test.ts new file mode 100644 index 00000000..189c9a38 --- /dev/null +++ b/scripts/verify-m058-s01.test.ts @@ -0,0 +1,267 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport } from "./verify-m058-s01.ts"; +import { + M058_S01_CHECK_IDS, + buildM058S01ProofHarness, + evaluateM058S01Proof, + parseM058S01Args, + renderM058S01Report, +} from "./verify-m058-s01.ts"; + +const EXPECTED_CHECK_IDS = [ + "M058-S01-CI-COVERAGE-BREADTH", + "M058-S01-CI-SPLIT-PRESERVED", + "M058-S01-PACKAGE-WIRING", + "M058-S01-CI-ORDERING-RATIONALE", +] as const; + +const PASSING_PACKAGE_JSON = JSON.stringify( + { + name: "kodiai", + scripts: { + "verify:m058:s01": "bun scripts/verify-m058-s01.ts", + }, + }, + null, + 2, +); + +const PASSING_CI = `name: ci +jobs: + test: + steps: + - run: bun install + - run: bun run verify:m056:s03 + # Bun has been unstable on GitHub runners with one monolithic test process. + # Keep DB-backed tests on a low concurrency cap and split the suite into + # two shorter invocations to avoid cross-file schema interference and runner crashes. + - run: bun test --max-concurrency=2 scripts src + - run: bun test --max-concurrency=2 src/knowledge + - run: bunx tsc --noEmit +`; + +describe("verify m058 s01 proof harness", () => { + test("exports stable check ids and cli parsing", () => { + expect(M058_S01_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM058S01Args([])).toEqual({ json: false }); + expect(parseM058S01Args(["--json"])).toEqual({ json: true }); + expect(() => parseM058S01Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes for the broadened CI coverage contract", async () => { + const report = await evaluateM058S01Proof({ + generatedAt: "2026-04-21T12:00:00.000Z", + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("ci.yml")) return PASSING_CI; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.command).toBe("verify:m058:s01"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M058-S01-CI-COVERAGE-BREADTH", + passed: true, + status_code: "ci_coverage_breadth_ok", + }), + expect.objectContaining({ + id: "M058-S01-CI-SPLIT-PRESERVED", + passed: true, + status_code: "ci_split_preserved_ok", + }), + expect.objectContaining({ + id: "M058-S01-PACKAGE-WIRING", + passed: true, + status_code: "package_wiring_ok", + }), + expect.objectContaining({ + id: "M058-S01-CI-ORDERING-RATIONALE", + passed: true, + status_code: "ci_ordering_rationale_ok", + }), + ]); + + const rendered = renderM058S01Report(report); + expect(rendered).toContain("M058 S01 CI coverage verifier"); + expect(rendered).toContain("CI coverage proof surface: PASS"); + expect(rendered).toContain("M058-S01-CI-COVERAGE-BREADTH PASS"); + expect(rendered).toContain("M058-S01-CI-SPLIT-PRESERVED PASS"); + expect(rendered).toContain("M058-S01-PACKAGE-WIRING PASS"); + expect(rendered).toContain("M058-S01-CI-ORDERING-RATIONALE PASS"); + }); + + test("fails with stable status codes for missing coverage, stale split step, and missing wiring", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM058S01ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return JSON.stringify({ name: "kodiai", scripts: {} }); + } + if (filePath.endsWith("ci.yml")) { + return `name: ci +jobs: + test: + steps: + # Bun has been unstable on GitHub runners with one monolithic test process. + # Keep DB-backed tests on a low concurrency cap and split the suite into + # two shorter invocations to avoid cross-file schema interference and runner crashes. + - run: bun test --max-concurrency=2 scripts src/contributor src/handlers src/webhook + - run: bun run verify:m056:s03 + - run: bun test --max-concurrency=2 src/webhook + - run: bunx tsc --noEmit +`; + } + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M058-S01-CI-COVERAGE-BREADTH", + passed: false, + status_code: "ci_coverage_step_missing", + }), + expect.objectContaining({ + id: "M058-S01-CI-SPLIT-PRESERVED", + passed: false, + status_code: "ci_split_step_missing", + }), + expect.objectContaining({ + id: "M058-S01-PACKAGE-WIRING", + passed: false, + status_code: "package_wiring_missing", + }), + expect.objectContaining({ + id: "M058-S01-CI-ORDERING-RATIONALE", + passed: true, + status_code: "ci_ordering_rationale_ok", + }), + ]); + expect(report.checks[0]?.detail).toContain("bun test --max-concurrency=2 scripts src"); + expect(report.checks[1]?.detail).toContain("bun test --max-concurrency=2 src/knowledge"); + expect(report.checks[2]?.detail).toContain("verify:m058:s01"); + expect(stderr.join(" ")).toContain("ci_coverage_step_missing"); + expect(stderr.join(" ")).toContain("ci_split_step_missing"); + expect(stderr.join(" ")).toContain("package_wiring_missing"); + }); + + test("fails with a stable status code when the verifier step is ordered after the broadened test step", async () => { + const report = await evaluateM058S01Proof({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("ci.yml")) { + return `name: ci +jobs: + test: + steps: + # Bun has been unstable on GitHub runners with one monolithic test process. + # Keep DB-backed tests on a low concurrency cap and split the suite into + # two shorter invocations to avoid cross-file schema interference and runner crashes. + - run: bun test --max-concurrency=2 scripts src + - run: bun run verify:m056:s03 + - run: bun test --max-concurrency=2 src/knowledge + - run: bunx tsc --noEmit +`; + } + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.checks[3]).toEqual( + expect.objectContaining({ + id: "M058-S01-CI-ORDERING-RATIONALE", + passed: false, + status_code: "ci_verify_step_misordered", + }), + ); + expect(report.checks[3]?.detail).toContain("bun run verify:m056:s03"); + expect(report.checks[3]?.detail).toContain("bun test --max-concurrency=2 scripts src"); + }); + + test("flags stale rationale comments, invalid package json, unreadable files, and malformed cli usage", async () => { + const staleCommentReport = await evaluateM058S01Proof({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return "{ not valid json"; + if (filePath.endsWith("ci.yml")) { + return `name: ci +jobs: + test: + steps: + - run: bun run verify:m056:s03 + # Bun used to be flaky, but the old src list is still here. + - run: bun test --max-concurrency=2 scripts src + - run: bun test --max-concurrency=2 src/knowledge +`; + } + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(staleCommentReport.checks[0]).toEqual( + expect.objectContaining({ + id: "M058-S01-CI-COVERAGE-BREADTH", + passed: true, + status_code: "ci_coverage_breadth_ok", + }), + ); + expect(staleCommentReport.checks[1]).toEqual( + expect.objectContaining({ + id: "M058-S01-CI-SPLIT-PRESERVED", + passed: true, + status_code: "ci_split_preserved_ok", + }), + ); + expect(staleCommentReport.checks[2]).toEqual( + expect.objectContaining({ + id: "M058-S01-PACKAGE-WIRING", + passed: false, + status_code: "package_json_invalid", + }), + ); + expect(staleCommentReport.checks[3]).toEqual( + expect.objectContaining({ + id: "M058-S01-CI-ORDERING-RATIONALE", + passed: false, + status_code: "ci_split_rationale_comment_missing", + }), + ); + + const unreadableFiles = await evaluateM058S01Proof({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) throw new Error("EACCES: package.json"); + if (filePath.endsWith("ci.yml")) throw new Error("EACCES: ci.yml"); + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(unreadableFiles.checks).toEqual([ + expect.objectContaining({ id: "M058-S01-CI-COVERAGE-BREADTH", status_code: "ci_file_unreadable" }), + expect.objectContaining({ id: "M058-S01-CI-SPLIT-PRESERVED", status_code: "ci_file_unreadable" }), + expect.objectContaining({ id: "M058-S01-PACKAGE-WIRING", status_code: "package_file_unreadable" }), + expect.objectContaining({ id: "M058-S01-CI-ORDERING-RATIONALE", status_code: "ci_file_unreadable" }), + ]); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m058:s01"]).toBe( + "bun scripts/verify-m058-s01.ts", + ); + }); +}); diff --git a/scripts/verify-m058-s01.ts b/scripts/verify-m058-s01.ts new file mode 100644 index 00000000..6c3ce050 --- /dev/null +++ b/scripts/verify-m058-s01.ts @@ -0,0 +1,334 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +const COMMAND_NAME = "verify:m058:s01" as const; +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/verify-m058-s01.ts"; +const EXPECTED_VERIFY_STEP = "bun run verify:m056:s03"; +const EXPECTED_BROAD_TEST_STEP = "bun test --max-concurrency=2 scripts src"; +const EXPECTED_KNOWLEDGE_TEST_STEP = "bun test --max-concurrency=2 src/knowledge"; +const REQUIRED_SPLIT_RATIONALE_MARKERS = [ + "Bun has been unstable on GitHub runners with one monolithic test process.", + "Keep DB-backed tests on a low concurrency cap and split the suite into", + "two shorter invocations to avoid cross-file schema interference and runner crashes.", +] as const; + +const REPO_ROOT = path.resolve(import.meta.dir, ".."); +const PACKAGE_JSON_PATH = path.resolve(REPO_ROOT, "package.json"); +const CI_WORKFLOW_PATH = path.resolve(REPO_ROOT, ".github/workflows/ci.yml"); + +export const M058_S01_CHECK_IDS = [ + "M058-S01-CI-COVERAGE-BREADTH", + "M058-S01-CI-SPLIT-PRESERVED", + "M058-S01-PACKAGE-WIRING", + "M058-S01-CI-ORDERING-RATIONALE", +] as const; + +export type M058S01CheckId = (typeof M058_S01_CHECK_IDS)[number]; + +export type Check = { + id: M058S01CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M058S01CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EvaluateOptions = { + generatedAt?: string; + readTextFile?: (filePath: string) => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +export async function evaluateM058S01Proof( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const readTextFile = options.readTextFile ?? defaultReadTextFile; + + const packageContent = await readOptionalTextFile(readTextFile, PACKAGE_JSON_PATH); + const ciContent = await readOptionalTextFile(readTextFile, CI_WORKFLOW_PATH); + + const checks: Check[] = [ + ciContent.ok + ? buildCiCoverageBreadthCheck(ciContent.content) + : failCheck("M058-S01-CI-COVERAGE-BREADTH", "ci_file_unreadable", ciContent.error), + ciContent.ok + ? buildCiSplitPreservedCheck(ciContent.content) + : failCheck("M058-S01-CI-SPLIT-PRESERVED", "ci_file_unreadable", ciContent.error), + packageContent.ok + ? buildPackageWiringCheck(packageContent.content) + : failCheck("M058-S01-PACKAGE-WIRING", "package_file_unreadable", packageContent.error), + ciContent.ok + ? buildCiOrderingAndRationaleCheck(ciContent.content) + : failCheck("M058-S01-CI-ORDERING-RATIONALE", "ci_file_unreadable", ciContent.error), + ]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M058_S01_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM058S01Report(report: EvaluationReport): string { + const lines = [ + "M058 S01 CI coverage verifier", + `Generated at: ${report.generatedAt}`, + `CI coverage proof surface: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM058S01ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM058S01Proof(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM058S01Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`${COMMAND_NAME} failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM058S01Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +function buildCiCoverageBreadthCheck(ciContent: string): Check { + if (!hasExactRunStep(ciContent, EXPECTED_BROAD_TEST_STEP)) { + return failCheck( + "M058-S01-CI-COVERAGE-BREADTH", + "ci_coverage_step_missing", + `.github/workflows/ci.yml must include ${EXPECTED_BROAD_TEST_STEP} so the full src tree, including src/webhook, is exercised.`, + ); + } + + return passCheck( + "M058-S01-CI-COVERAGE-BREADTH", + "ci_coverage_breadth_ok", + `.github/workflows/ci.yml includes ${EXPECTED_BROAD_TEST_STEP}.`, + ); +} + +function buildCiSplitPreservedCheck(ciContent: string): Check { + if (!hasExactRunStep(ciContent, EXPECTED_KNOWLEDGE_TEST_STEP)) { + return failCheck( + "M058-S01-CI-SPLIT-PRESERVED", + "ci_split_step_missing", + `.github/workflows/ci.yml must retain ${EXPECTED_KNOWLEDGE_TEST_STEP} as the isolated second Bun test step.`, + ); + } + + return passCheck( + "M058-S01-CI-SPLIT-PRESERVED", + "ci_split_preserved_ok", + `.github/workflows/ci.yml retains ${EXPECTED_KNOWLEDGE_TEST_STEP} as the isolated second Bun test step.`, + ); +} + +function buildPackageWiringCheck(packageJsonContent: string): Check { + let packageJson: { scripts?: Record }; + try { + packageJson = JSON.parse(packageJsonContent) as { scripts?: Record }; + } catch (error) { + return failCheck("M058-S01-PACKAGE-WIRING", "package_json_invalid", error); + } + + const actualScript = packageJson.scripts?.[COMMAND_NAME]; + if (actualScript == null) { + return failCheck( + "M058-S01-PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT}`, + ); + } + + if (actualScript !== EXPECTED_PACKAGE_SCRIPT) { + return failCheck( + "M058-S01-PACKAGE-WIRING", + "package_wiring_incorrect", + `Expected scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT} but found ${actualScript}`, + ); + } + + return passCheck( + "M058-S01-PACKAGE-WIRING", + "package_wiring_ok", + `package.json wires ${COMMAND_NAME} to ${EXPECTED_PACKAGE_SCRIPT}`, + ); +} + +function buildCiOrderingAndRationaleCheck(ciContent: string): Check { + const verifyStepIndex = indexOfRunStep(ciContent, EXPECTED_VERIFY_STEP); + if (verifyStepIndex === -1) { + return failCheck( + "M058-S01-CI-ORDERING-RATIONALE", + "ci_verify_step_missing", + `.github/workflows/ci.yml must run ${EXPECTED_VERIFY_STEP} before the broadened Bun test steps.`, + ); + } + + const broadStepIndex = indexOfRunStep(ciContent, EXPECTED_BROAD_TEST_STEP); + const knowledgeStepIndex = indexOfRunStep(ciContent, EXPECTED_KNOWLEDGE_TEST_STEP); + + if (broadStepIndex !== -1 && verifyStepIndex > broadStepIndex) { + return failCheck( + "M058-S01-CI-ORDERING-RATIONALE", + "ci_verify_step_misordered", + `${EXPECTED_VERIFY_STEP} must appear before ${EXPECTED_BROAD_TEST_STEP} in .github/workflows/ci.yml.`, + ); + } + + if (knowledgeStepIndex !== -1 && verifyStepIndex > knowledgeStepIndex) { + return failCheck( + "M058-S01-CI-ORDERING-RATIONALE", + "ci_verify_step_misordered", + `${EXPECTED_VERIFY_STEP} must appear before ${EXPECTED_KNOWLEDGE_TEST_STEP} in .github/workflows/ci.yml.`, + ); + } + + const missingRationaleMarkers = REQUIRED_SPLIT_RATIONALE_MARKERS.filter( + (marker) => !ciContent.includes(marker), + ); + if (missingRationaleMarkers.length > 0) { + return failCheck( + "M058-S01-CI-ORDERING-RATIONALE", + "ci_split_rationale_comment_missing", + `.github/workflows/ci.yml must preserve the split rationale comment markers: ${missingRationaleMarkers.join(", ")}`, + ); + } + + return passCheck( + "M058-S01-CI-ORDERING-RATIONALE", + "ci_ordering_rationale_ok", + `.github/workflows/ci.yml keeps ${EXPECTED_VERIFY_STEP} ahead of the split Bun test steps and preserves the split rationale comment.`, + ); +} + +function hasExactRunStep(ciContent: string, command: string): boolean { + return runStepMatch(ciContent, command) != null; +} + +function indexOfRunStep(ciContent: string, command: string): number { + return runStepMatch(ciContent, command)?.index ?? -1; +} + +function runStepMatch(ciContent: string, command: string): RegExpMatchArray | null { + return ciContent.match(new RegExp(`^\\s*- run: ${escapeForRegex(command)}\\s*$`, "m")); +} + +function escapeForRegex(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function passCheck(id: M058S01CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M058S01CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +async function readOptionalTextFile( + readTextFile: (filePath: string) => Promise, + filePath: string, +): Promise<{ ok: true; content: string } | { ok: false; error: unknown }> { + try { + return { ok: true, content: await readTextFile(filePath) }; + } catch (error) { + return { ok: false, error }; + } +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +if (import.meta.main) { + try { + const args = parseM058S01Args(process.argv.slice(2)); + const { exitCode } = await buildM058S01ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${COMMAND_NAME} failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m058-s02.test.ts b/scripts/verify-m058-s02.test.ts new file mode 100644 index 00000000..ce0cac4e --- /dev/null +++ b/scripts/verify-m058-s02.test.ts @@ -0,0 +1,317 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport } from "./verify-m058-s02.ts"; +import { + M058_S02_CHECK_IDS, + buildM058S02ProofHarness, + evaluateM058S02Proof, + parseM058S02Args, + renderM058S02Report, +} from "./verify-m058-s02.ts"; + +const EXPECTED_CHECK_IDS = [ + "M058-S02-PACKAGE-CONTRACT", + "M058-S02-PACKAGE-WIRING", + "M058-S02-WORKFLOW-ALIGNMENT", + "M058-S02-DOCS-TRUTH", +] as const; + +const PINNED_BUN_VERSION = "1.3.8" as const; +const EXPECTED_PACKAGE_MANAGER = `bun@${PINNED_BUN_VERSION}`; +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/verify-m058-s02.ts"; + +const PASSING_PACKAGE_JSON = JSON.stringify( + { + name: "kodiai", + packageManager: EXPECTED_PACKAGE_MANAGER, + engines: { + bun: PINNED_BUN_VERSION, + }, + scripts: { + "verify:m058:s02": EXPECTED_PACKAGE_SCRIPT, + }, + devDependencies: { + "@types/bun": "latest", + }, + }, + null, + 2, +); + +const PASSING_CI = `name: ci +jobs: + test: + steps: + - uses: oven-sh/setup-bun@v2 + with: + bun-version: ${PINNED_BUN_VERSION} + - run: bun install +`; + +const PASSING_NIGHTLY_ISSUE = `name: nightly-issue-sync +jobs: + sync: + steps: + - uses: oven-sh/setup-bun@v2 + with: + bun-version: ${PINNED_BUN_VERSION} + - run: bun install --frozen-lockfile +`; + +const PASSING_NIGHTLY_REACTION = `name: nightly-reaction-sync +jobs: + sync: + steps: + - uses: oven-sh/setup-bun@v2 + with: + bun-version: ${PINNED_BUN_VERSION} + - run: bun install --frozen-lockfile +`; + +const PASSING_CONTRIBUTING = `# Contributing + +- package.json pins \`packageManager\` to \`${EXPECTED_PACKAGE_MANAGER}\`. +- package.json pins \`engines.bun\` to \`${PINNED_BUN_VERSION}\`. +- GitHub Actions workflows use \`oven-sh/setup-bun@v2\` with \`bun-version: ${PINNED_BUN_VERSION}\`. +- \`@types/bun\` remains a separate devDependency surface and is not the runtime pin. +- Run \`bun run verify:m058:s02\` to inspect Bun contract drift. +`; + +describe("verify m058 s02 proof harness", () => { + test("exports stable check ids and cli parsing", () => { + expect(M058_S02_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM058S02Args([])).toEqual({ json: false }); + expect(parseM058S02Args(["--json"])).toEqual({ json: true }); + expect(() => parseM058S02Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes when package, workflows, and docs all align to one pinned Bun contract", async () => { + const report = await evaluateM058S02Proof({ + generatedAt: "2026-04-21T12:00:00.000Z", + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("ci.yml")) return PASSING_CI; + if (filePath.endsWith("nightly-issue-sync.yml")) return PASSING_NIGHTLY_ISSUE; + if (filePath.endsWith("nightly-reaction-sync.yml")) return PASSING_NIGHTLY_REACTION; + if (filePath.endsWith("CONTRIBUTING.md")) return PASSING_CONTRIBUTING; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.command).toBe("verify:m058:s02"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M058-S02-PACKAGE-CONTRACT", + passed: true, + status_code: "package_contract_ok", + }), + expect.objectContaining({ + id: "M058-S02-PACKAGE-WIRING", + passed: true, + status_code: "package_wiring_ok", + }), + expect.objectContaining({ + id: "M058-S02-WORKFLOW-ALIGNMENT", + passed: true, + status_code: "workflow_alignment_ok", + }), + expect.objectContaining({ + id: "M058-S02-DOCS-TRUTH", + passed: true, + status_code: "docs_truth_ok", + }), + ]); + + const rendered = renderM058S02Report(report); + expect(rendered).toContain("M058 S02 Bun contract verifier"); + expect(rendered).toContain("Bun contract proof surface: PASS"); + expect(rendered).toContain("M058-S02-PACKAGE-CONTRACT PASS"); + expect(rendered).toContain("M058-S02-PACKAGE-WIRING PASS"); + expect(rendered).toContain("M058-S02-WORKFLOW-ALIGNMENT PASS"); + expect(rendered).toContain("M058-S02-DOCS-TRUTH PASS"); + }); + + test("flags missing exact contract fields and package script wiring", async () => { + const report = await evaluateM058S02Proof({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return JSON.stringify({ + name: "kodiai", + engines: {}, + scripts: {}, + }); + } + if (filePath.endsWith("ci.yml")) return PASSING_CI; + if (filePath.endsWith("nightly-issue-sync.yml")) return PASSING_NIGHTLY_ISSUE; + if (filePath.endsWith("nightly-reaction-sync.yml")) return PASSING_NIGHTLY_REACTION; + if (filePath.endsWith("CONTRIBUTING.md")) return PASSING_CONTRIBUTING; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.overallPassed).toBe(false); + expect(report.checks[0]).toEqual( + expect.objectContaining({ + id: "M058-S02-PACKAGE-CONTRACT", + passed: false, + status_code: "package_manager_missing", + }), + ); + expect(report.checks[1]).toEqual( + expect.objectContaining({ + id: "M058-S02-PACKAGE-WIRING", + passed: false, + status_code: "package_wiring_missing", + }), + ); + }); + + test("flags malformed package data, mismatched exact pins, and invalid package json", async () => { + const mismatchedReport = await evaluateM058S02Proof({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return JSON.stringify({ + name: "kodiai", + packageManager: "bun@1.3.7", + engines: { bun: PINNED_BUN_VERSION }, + scripts: { "verify:m058:s02": EXPECTED_PACKAGE_SCRIPT }, + }); + } + if (filePath.endsWith("ci.yml")) return PASSING_CI; + if (filePath.endsWith("nightly-issue-sync.yml")) return PASSING_NIGHTLY_ISSUE; + if (filePath.endsWith("nightly-reaction-sync.yml")) return PASSING_NIGHTLY_REACTION; + if (filePath.endsWith("CONTRIBUTING.md")) return PASSING_CONTRIBUTING; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(mismatchedReport.checks[0]).toEqual( + expect.objectContaining({ + id: "M058-S02-PACKAGE-CONTRACT", + passed: false, + status_code: "bun_version_mismatch", + }), + ); + + const malformedReport = await evaluateM058S02Proof({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return "{ not valid json"; + if (filePath.endsWith("ci.yml")) return PASSING_CI; + if (filePath.endsWith("nightly-issue-sync.yml")) return PASSING_NIGHTLY_ISSUE; + if (filePath.endsWith("nightly-reaction-sync.yml")) return PASSING_NIGHTLY_REACTION; + if (filePath.endsWith("CONTRIBUTING.md")) return PASSING_CONTRIBUTING; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(malformedReport.checks[0]).toEqual( + expect.objectContaining({ + id: "M058-S02-PACKAGE-CONTRACT", + passed: false, + status_code: "package_json_invalid", + }), + ); + expect(malformedReport.checks[1]).toEqual( + expect.objectContaining({ + id: "M058-S02-PACKAGE-WIRING", + passed: false, + status_code: "package_json_invalid", + }), + ); + }); + + test("isolates stale workflow and docs drift from an otherwise valid package contract", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM058S02ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("ci.yml")) return PASSING_CI; + if (filePath.endsWith("nightly-issue-sync.yml")) return PASSING_NIGHTLY_ISSUE.replace( + PINNED_BUN_VERSION, + "latest", + ); + if (filePath.endsWith("nightly-reaction-sync.yml")) return PASSING_NIGHTLY_REACTION; + if (filePath.endsWith("CONTRIBUTING.md")) { + return `# Contributing\n\nThis repository does not pin a single Bun version in source control yet.\n`; + } + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + + expect(result.exitCode).toBe(1); + expect(report.checks[0]).toEqual( + expect.objectContaining({ + id: "M058-S02-PACKAGE-CONTRACT", + passed: true, + status_code: "package_contract_ok", + }), + ); + expect(report.checks[2]).toEqual( + expect.objectContaining({ + id: "M058-S02-WORKFLOW-ALIGNMENT", + passed: false, + status_code: "workflow_bun_version_drift", + }), + ); + expect(report.checks[3]).toEqual( + expect.objectContaining({ + id: "M058-S02-DOCS-TRUTH", + passed: false, + status_code: "docs_truth_stale", + }), + ); + expect(stderr.join(" ")).toContain("workflow_bun_version_drift"); + expect(stderr.join(" ")).toContain("docs_truth_stale"); + }); + + test("flags unreadable workflow and docs files with stable status codes", async () => { + const report = await evaluateM058S02Proof({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("ci.yml")) throw new Error("EACCES: ci.yml"); + if (filePath.endsWith("nightly-issue-sync.yml")) throw new Error("EACCES: nightly issue"); + if (filePath.endsWith("nightly-reaction-sync.yml")) throw new Error("EACCES: nightly reaction"); + if (filePath.endsWith("CONTRIBUTING.md")) throw new Error("EACCES: CONTRIBUTING.md"); + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.checks[2]).toEqual( + expect.objectContaining({ + id: "M058-S02-WORKFLOW-ALIGNMENT", + passed: false, + status_code: "workflow_file_unreadable", + }), + ); + expect(report.checks[3]).toEqual( + expect.objectContaining({ + id: "M058-S02-DOCS-TRUTH", + passed: false, + status_code: "contributing_file_unreadable", + }), + ); + }); + + test("wires the canonical package script and Bun contract in package.json", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { + packageManager?: string; + engines?: { bun?: string }; + scripts?: Record; + }; + + expect(packageJson.packageManager).toBe(EXPECTED_PACKAGE_MANAGER); + expect(packageJson.engines?.bun).toBe(PINNED_BUN_VERSION); + expect(packageJson.scripts?.["verify:m058:s02"]).toBe(EXPECTED_PACKAGE_SCRIPT); + }); +}); diff --git a/scripts/verify-m058-s02.ts b/scripts/verify-m058-s02.ts new file mode 100644 index 00000000..ef0002ae --- /dev/null +++ b/scripts/verify-m058-s02.ts @@ -0,0 +1,436 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +const COMMAND_NAME = "verify:m058:s02" as const; +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/verify-m058-s02.ts"; +const PINNED_BUN_VERSION = "1.3.8" as const; +const EXPECTED_PACKAGE_MANAGER = `bun@${PINNED_BUN_VERSION}`; +const WORKFLOW_VERSION_SNIPPET = `bun-version: ${PINNED_BUN_VERSION}`; +const WORKFLOW_STALE_SNIPPET = "bun-version: latest"; +const REQUIRED_DOC_MARKERS = [ + `packageManager\` to \`${EXPECTED_PACKAGE_MANAGER}\``, + `engines.bun\` to \`${PINNED_BUN_VERSION}\``, + `bun-version: ${PINNED_BUN_VERSION}`, + "@types/bun", + COMMAND_NAME, +] as const; +const STALE_DOC_MARKERS = [ + "does **not** pin a single Bun version", + "does not pin a single Bun version", + "bun-version: latest", + "use a current Bun release that is compatible with the repo", +] as const; + +const REPO_ROOT = path.resolve(import.meta.dir, ".."); +const PACKAGE_JSON_PATH = path.resolve(REPO_ROOT, "package.json"); +const CONTRIBUTING_PATH = path.resolve(REPO_ROOT, "CONTRIBUTING.md"); +const WORKFLOW_PATHS = [ + ".github/workflows/ci.yml", + ".github/workflows/nightly-issue-sync.yml", + ".github/workflows/nightly-reaction-sync.yml", +] as const; + +export const M058_S02_CHECK_IDS = [ + "M058-S02-PACKAGE-CONTRACT", + "M058-S02-PACKAGE-WIRING", + "M058-S02-WORKFLOW-ALIGNMENT", + "M058-S02-DOCS-TRUTH", +] as const; + +export type M058S02CheckId = (typeof M058_S02_CHECK_IDS)[number]; + +export type Check = { + id: M058S02CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M058S02CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EvaluateOptions = { + generatedAt?: string; + readTextFile?: (filePath: string) => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +type PackageJsonShape = { + packageManager?: unknown; + engines?: { bun?: unknown } | unknown; + scripts?: Record | unknown; +}; + +export async function evaluateM058S02Proof( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const readTextFile = options.readTextFile ?? defaultReadTextFile; + + const packageContent = await readOptionalTextFile(readTextFile, PACKAGE_JSON_PATH); + const contributingContent = await readOptionalTextFile(readTextFile, CONTRIBUTING_PATH); + const workflowContents = await Promise.all( + WORKFLOW_PATHS.map(async (relativePath) => ({ + relativePath, + result: await readOptionalTextFile(readTextFile, path.resolve(REPO_ROOT, relativePath)), + })), + ); + + const parsedPackageJson = packageContent.ok + ? parsePackageJson(packageContent.content) + : { ok: false as const, error: packageContent.error }; + + const checks: Check[] = [ + buildPackageContractCheck(parsedPackageJson), + buildPackageWiringCheck(parsedPackageJson), + buildWorkflowAlignmentCheck(workflowContents), + contributingContent.ok + ? buildDocsTruthCheck(contributingContent.content) + : failCheck( + "M058-S02-DOCS-TRUTH", + "contributing_file_unreadable", + contributingContent.error, + ), + ]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M058_S02_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM058S02Report(report: EvaluationReport): string { + const lines = [ + "M058 S02 Bun contract verifier", + `Generated at: ${report.generatedAt}`, + `Bun contract proof surface: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM058S02ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM058S02Proof(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM058S02Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`${COMMAND_NAME} failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM058S02Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +function buildPackageContractCheck( + parsedPackageJson: ReturnType, +): Check { + if (!parsedPackageJson.ok) { + return failCheck("M058-S02-PACKAGE-CONTRACT", "package_json_invalid", parsedPackageJson.error); + } + + const packageJson = parsedPackageJson.value; + if (typeof packageJson.packageManager !== "string") { + return failCheck( + "M058-S02-PACKAGE-CONTRACT", + "package_manager_missing", + `package.json must define packageManager=${EXPECTED_PACKAGE_MANAGER}`, + ); + } + + if (!packageJson.packageManager.startsWith("bun@")) { + return failCheck( + "M058-S02-PACKAGE-CONTRACT", + "package_manager_invalid_shape", + `packageManager must be an exact bun@ string; found ${packageJson.packageManager}`, + ); + } + + if (typeof packageJson.engines !== "object" || packageJson.engines == null || Array.isArray(packageJson.engines)) { + return failCheck( + "M058-S02-PACKAGE-CONTRACT", + "engines_bun_missing", + `package.json must define engines.bun=${PINNED_BUN_VERSION}`, + ); + } + + const engines = packageJson.engines as Record; + const bunEngine = engines.bun; + if (typeof bunEngine !== "string") { + return failCheck( + "M058-S02-PACKAGE-CONTRACT", + "engines_bun_missing", + `package.json must define engines.bun=${PINNED_BUN_VERSION}`, + ); + } + + const packageManagerVersion = packageJson.packageManager.slice("bun@".length); + if (packageManagerVersion !== bunEngine) { + return failCheck( + "M058-S02-PACKAGE-CONTRACT", + "bun_version_mismatch", + `packageManager declares ${packageJson.packageManager} while engines.bun declares ${bunEngine}`, + ); + } + + if (packageJson.packageManager !== EXPECTED_PACKAGE_MANAGER || bunEngine !== PINNED_BUN_VERSION) { + return failCheck( + "M058-S02-PACKAGE-CONTRACT", + "bun_version_drift", + `Expected packageManager=${EXPECTED_PACKAGE_MANAGER} and engines.bun=${PINNED_BUN_VERSION}, found packageManager=${packageJson.packageManager} engines.bun=${bunEngine}`, + ); + } + + return passCheck( + "M058-S02-PACKAGE-CONTRACT", + "package_contract_ok", + `package.json pins packageManager=${EXPECTED_PACKAGE_MANAGER} and engines.bun=${PINNED_BUN_VERSION}`, + ); +} + +function buildPackageWiringCheck(parsedPackageJson: ReturnType): Check { + if (!parsedPackageJson.ok) { + return failCheck("M058-S02-PACKAGE-WIRING", "package_json_invalid", parsedPackageJson.error); + } + + const scripts = parsedPackageJson.value.scripts; + if (typeof scripts !== "object" || scripts == null || Array.isArray(scripts)) { + return failCheck( + "M058-S02-PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT}`, + ); + } + + const scriptMap = scripts as Record; + const actualScript = scriptMap[COMMAND_NAME]; + if (actualScript == null) { + return failCheck( + "M058-S02-PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT}`, + ); + } + + if (actualScript !== EXPECTED_PACKAGE_SCRIPT) { + return failCheck( + "M058-S02-PACKAGE-WIRING", + "package_wiring_incorrect", + `Expected scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT} but found ${actualScript}`, + ); + } + + return passCheck( + "M058-S02-PACKAGE-WIRING", + "package_wiring_ok", + `package.json wires ${COMMAND_NAME} to ${EXPECTED_PACKAGE_SCRIPT}`, + ); +} + +function buildWorkflowAlignmentCheck( + workflowContents: Array<{ + relativePath: (typeof WORKFLOW_PATHS)[number]; + result: Awaited>; + }>, +): Check { + for (const workflow of workflowContents) { + if (!workflow.result.ok) { + return failCheck( + "M058-S02-WORKFLOW-ALIGNMENT", + "workflow_file_unreadable", + `${workflow.relativePath}: ${normalizeDetail(workflow.result.error)}`, + ); + } + } + + const stalePaths: string[] = []; + const missingSetupPaths: string[] = []; + + for (const workflow of workflowContents) { + if (!workflow.result.ok) { + continue; + } + + const content = workflow.result.content; + if (!content.includes("oven-sh/setup-bun@v2")) { + missingSetupPaths.push(workflow.relativePath); + continue; + } + + if (content.includes(WORKFLOW_STALE_SNIPPET) || !content.includes(WORKFLOW_VERSION_SNIPPET)) { + stalePaths.push(workflow.relativePath); + } + } + + if (missingSetupPaths.length > 0) { + return failCheck( + "M058-S02-WORKFLOW-ALIGNMENT", + "workflow_setup_missing", + `Missing oven-sh/setup-bun@v2 in: ${missingSetupPaths.join(", ")}`, + ); + } + + if (stalePaths.length > 0) { + return failCheck( + "M058-S02-WORKFLOW-ALIGNMENT", + "workflow_bun_version_drift", + `Expected ${WORKFLOW_VERSION_SNIPPET} in all Bun-installing workflows; drift found in: ${stalePaths.join(", ")}`, + ); + } + + return passCheck( + "M058-S02-WORKFLOW-ALIGNMENT", + "workflow_alignment_ok", + `All Bun-installing workflows pin ${WORKFLOW_VERSION_SNIPPET}`, + ); +} + +function buildDocsTruthCheck(contributingContent: string): Check { + const staleMarkers = STALE_DOC_MARKERS.filter((marker) => contributingContent.includes(marker)); + if (staleMarkers.length > 0) { + return failCheck( + "M058-S02-DOCS-TRUTH", + "docs_truth_stale", + `CONTRIBUTING.md still contains stale Bun guidance: ${staleMarkers.join(", ")}`, + ); + } + + const missingMarkers = REQUIRED_DOC_MARKERS.filter( + (marker) => !contributingContent.includes(marker), + ); + if (missingMarkers.length > 0) { + return failCheck( + "M058-S02-DOCS-TRUTH", + "docs_truth_missing", + `CONTRIBUTING.md must document the Bun contract markers: ${missingMarkers.join(", ")}`, + ); + } + + return passCheck( + "M058-S02-DOCS-TRUTH", + "docs_truth_ok", + "CONTRIBUTING.md documents the pinned Bun contract, workflow alignment, verifier command, and the separate @types/bun caveat.", + ); +} + +function parsePackageJson(content: string): + | { ok: true; value: PackageJsonShape } + | { ok: false; error: unknown } { + try { + return { ok: true, value: JSON.parse(content) as PackageJsonShape }; + } catch (error) { + return { ok: false, error }; + } +} + +function passCheck(id: M058S02CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M058S02CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +async function readOptionalTextFile( + readTextFile: (filePath: string) => Promise, + filePath: string, +): Promise<{ ok: true; content: string } | { ok: false; error: unknown }> { + try { + return { ok: true, content: await readTextFile(filePath) }; + } catch (error) { + return { ok: false, error }; + } +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +if (import.meta.main) { + try { + const args = parseM058S02Args(process.argv.slice(2)); + const { exitCode } = await buildM058S02ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${COMMAND_NAME} failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m058-s03.test.ts b/scripts/verify-m058-s03.test.ts new file mode 100644 index 00000000..cc80e4d2 --- /dev/null +++ b/scripts/verify-m058-s03.test.ts @@ -0,0 +1,308 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport } from "./verify-m058-s03.ts"; +import { + M058_S03_CHECK_IDS, + buildM058S03ProofHarness, + evaluateM058S03Proof, + parseM058S03Args, + renderM058S03Report, +} from "./verify-m058-s03.ts"; + +const EXPECTED_CHECK_IDS = [ + "M058-S03-PACKAGE-WIRING", + "M058-S03-CI-WIRING", + "M058-S03-CI-RATIONALE", + "M058-S03-DECISION-RECORD", +] as const; + +const EXPECTED_VERIFY_SCRIPT = "bun scripts/verify-m058-s03.ts"; +const EXPECTED_LINT_SCRIPT = "eslint src scripts"; + +const PASSING_PACKAGE_JSON = JSON.stringify( + { + name: "kodiai", + scripts: { + lint: EXPECTED_LINT_SCRIPT, + "verify:m058:s03": EXPECTED_VERIFY_SCRIPT, + }, + }, + null, + 2, +); + +const PASSING_CI = `name: ci +jobs: + test: + steps: + - run: bun install + - run: bun run lint + - run: bun run verify:m056:s03 + - run: bun run check:orphaned-tests + # Bun has been unstable on GitHub runners with one monolithic test process. + # Keep DB-backed tests on a low concurrency cap and split the suite into + # two shorter invocations to avoid cross-file schema interference and runner crashes. + - run: bun test --max-concurrency=2 scripts src + - run: bun test --max-concurrency=2 src/knowledge + - run: bunx tsc --noEmit +`; + +const PASSING_DECISIONS = `# Decisions Register +| # | When | Scope | Decision | Choice | Rationale | Revisable? | Made By | +|---|------|-------|----------|--------|-----------|------------|---------| +| D999 | M058/S03/T02 | tooling | Lint tool contract for M058/S03 CI hardening | Adopt ESLint as the repo-owned linter for src/ and scripts/, with an explicit carve-out for operator-facing migration CLI console output. | The roadmap and enforcement tooling already model ESLint config names, the repo has no existing lint surface to preserve, and ESLint gives the lowest-friction path to a PR-blocking lint gate while allowing a narrow exception for src/db/migrate.ts instead of weakening the broader no-console policy. | Yes | agent | +`; + +describe("verify m058 s03 proof harness", () => { + test("exports stable check ids and cli parsing", () => { + expect(M058_S03_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM058S03Args([])).toEqual({ json: false }); + expect(parseM058S03Args(["--json"])).toEqual({ json: true }); + expect(() => parseM058S03Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes for the S03 CI gate contract", async () => { + const report = await evaluateM058S03Proof({ + generatedAt: "2026-04-21T12:00:00.000Z", + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("ci.yml")) return PASSING_CI; + if (filePath.endsWith("DECISIONS.md")) return PASSING_DECISIONS; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.command).toBe("verify:m058:s03"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M058-S03-PACKAGE-WIRING", + passed: true, + status_code: "package_wiring_ok", + }), + expect.objectContaining({ + id: "M058-S03-CI-WIRING", + passed: true, + status_code: "ci_wiring_ok", + }), + expect.objectContaining({ + id: "M058-S03-CI-RATIONALE", + passed: true, + status_code: "ci_rationale_ok", + }), + expect.objectContaining({ + id: "M058-S03-DECISION-RECORD", + passed: true, + status_code: "decision_record_ok", + }), + ]); + + const rendered = renderM058S03Report(report); + expect(rendered).toContain("M058 S03 CI gate contract verifier"); + expect(rendered).toContain("CI gate proof surface: PASS"); + expect(rendered).toContain("M058-S03-PACKAGE-WIRING PASS"); + expect(rendered).toContain("M058-S03-CI-WIRING PASS"); + expect(rendered).toContain("M058-S03-CI-RATIONALE PASS"); + expect(rendered).toContain("M058-S03-DECISION-RECORD PASS"); + }); + + test("fails with stable status codes for missing package wiring, missing CI steps, and missing decision marker", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM058S03ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return JSON.stringify({ name: "kodiai", scripts: {} }); + } + if (filePath.endsWith("ci.yml")) { + return `name: ci +jobs: + test: + steps: + - run: bun run verify:m056:s03 + # Bun has been unstable on GitHub runners with one monolithic test process. + # Keep DB-backed tests on a low concurrency cap and split the suite into + # two shorter invocations to avoid cross-file schema interference and runner crashes. + - run: bun test --max-concurrency=2 scripts src + - run: bun test --max-concurrency=2 src/knowledge +`; + } + if (filePath.endsWith("DECISIONS.md")) { + return PASSING_DECISIONS.replace("", ""); + } + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M058-S03-PACKAGE-WIRING", + passed: false, + status_code: "package_wiring_missing", + }), + expect.objectContaining({ + id: "M058-S03-CI-WIRING", + passed: false, + status_code: "ci_lint_step_missing", + }), + expect.objectContaining({ + id: "M058-S03-CI-RATIONALE", + passed: true, + status_code: "ci_rationale_ok", + }), + expect.objectContaining({ + id: "M058-S03-DECISION-RECORD", + passed: false, + status_code: "decision_marker_missing", + }), + ]); + expect(report.checks[0]?.detail).toContain("scripts.lint"); + expect(report.checks[1]?.detail).toContain("bun run lint"); + expect(report.checks[3]?.detail).toContain("M058-S03-LINT-TOOL-CONTRACT"); + expect(stderr.join(" ")).toContain("package_wiring_missing"); + expect(stderr.join(" ")).toContain("ci_lint_step_missing"); + expect(stderr.join(" ")).toContain("decision_marker_missing"); + }); + + test("fails when CI gate steps are misordered or the orphaned-test step is missing", async () => { + const misorderedReport = await evaluateM058S03Proof({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("ci.yml")) { + return `name: ci +jobs: + test: + steps: + - run: bun run lint + - run: bun run check:orphaned-tests + - run: bun run verify:m056:s03 + # Bun has been unstable on GitHub runners with one monolithic test process. + # Keep DB-backed tests on a low concurrency cap and split the suite into + # two shorter invocations to avoid cross-file schema interference and runner crashes. + - run: bun test --max-concurrency=2 scripts src + - run: bun test --max-concurrency=2 src/knowledge +`; + } + if (filePath.endsWith("DECISIONS.md")) return PASSING_DECISIONS; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(misorderedReport.checks[1]).toEqual( + expect.objectContaining({ + id: "M058-S03-CI-WIRING", + passed: false, + status_code: "ci_gate_steps_misordered", + }), + ); + expect(misorderedReport.checks[1]?.detail).toContain("bun run verify:m056:s03"); + expect(misorderedReport.checks[1]?.detail).toContain("bun run check:orphaned-tests"); + + const missingOrphanedReport = await evaluateM058S03Proof({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("ci.yml")) { + return `name: ci +jobs: + test: + steps: + - run: bun run lint + - run: bun run verify:m056:s03 + # Bun has been unstable on GitHub runners with one monolithic test process. + # Keep DB-backed tests on a low concurrency cap and split the suite into + # two shorter invocations to avoid cross-file schema interference and runner crashes. + - run: bun test --max-concurrency=2 scripts src + - run: bun test --max-concurrency=2 src/knowledge +`; + } + if (filePath.endsWith("DECISIONS.md")) return PASSING_DECISIONS; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(missingOrphanedReport.checks[1]).toEqual( + expect.objectContaining({ + id: "M058-S03-CI-WIRING", + passed: false, + status_code: "ci_orphaned_test_step_missing", + }), + ); + }); + + test("flags stale split rationale, malformed package json, and unreadable files with stable status codes", async () => { + const staleRationaleReport = await evaluateM058S03Proof({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return "{ not valid json"; + if (filePath.endsWith("ci.yml")) { + return `name: ci +jobs: + test: + steps: + - run: bun run lint + - run: bun run verify:m056:s03 + - run: bun run check:orphaned-tests + - run: bun test --max-concurrency=2 scripts src + - run: bun test --max-concurrency=2 src/knowledge +`; + } + if (filePath.endsWith("DECISIONS.md")) return "# Decisions Register\n"; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(staleRationaleReport.checks[0]).toEqual( + expect.objectContaining({ + id: "M058-S03-PACKAGE-WIRING", + passed: false, + status_code: "package_json_invalid", + }), + ); + expect(staleRationaleReport.checks[2]).toEqual( + expect.objectContaining({ + id: "M058-S03-CI-RATIONALE", + passed: false, + status_code: "ci_split_rationale_comment_missing", + }), + ); + expect(staleRationaleReport.checks[3]).toEqual( + expect.objectContaining({ + id: "M058-S03-DECISION-RECORD", + passed: false, + status_code: "decision_marker_missing", + }), + ); + + const unreadableReport = await evaluateM058S03Proof({ + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) throw new Error("EACCES: package.json"); + if (filePath.endsWith("ci.yml")) throw new Error("EACCES: ci.yml"); + if (filePath.endsWith("DECISIONS.md")) throw new Error("EACCES: DECISIONS.md"); + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(unreadableReport.checks).toEqual([ + expect.objectContaining({ id: "M058-S03-PACKAGE-WIRING", status_code: "package_file_unreadable" }), + expect.objectContaining({ id: "M058-S03-CI-WIRING", status_code: "ci_file_unreadable" }), + expect.objectContaining({ id: "M058-S03-CI-RATIONALE", status_code: "ci_file_unreadable" }), + expect.objectContaining({ id: "M058-S03-DECISION-RECORD", status_code: "decision_file_unreadable" }), + ]); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m058:s03"]).toBe(EXPECTED_VERIFY_SCRIPT); + }); +}); diff --git a/scripts/verify-m058-s03.ts b/scripts/verify-m058-s03.ts new file mode 100644 index 00000000..d334a718 --- /dev/null +++ b/scripts/verify-m058-s03.ts @@ -0,0 +1,423 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +const COMMAND_NAME = "verify:m058:s03" as const; +const EXPECTED_VERIFY_SCRIPT = "bun scripts/verify-m058-s03.ts"; +const EXPECTED_LINT_SCRIPT = "eslint src scripts"; +const EXPECTED_LINT_STEP = "bun run lint"; +const EXPECTED_MIGRATION_VERIFY_STEP = "bun run verify:m056:s03"; +const EXPECTED_ORPHANED_TEST_STEP = "bun run check:orphaned-tests"; +const EXPECTED_BROAD_TEST_STEP = "bun test --max-concurrency=2 scripts src"; +const EXPECTED_KNOWLEDGE_TEST_STEP = "bun test --max-concurrency=2 src/knowledge"; +const REQUIRED_SPLIT_RATIONALE_MARKERS = [ + "Bun has been unstable on GitHub runners with one monolithic test process.", + "Keep DB-backed tests on a low concurrency cap and split the suite into", + "two shorter invocations to avoid cross-file schema interference and runner crashes.", +] as const; +const DECISION_MARKER = "M058-S03-LINT-TOOL-CONTRACT"; +const DECISION_REQUIRED_TEXT = + "Adopt ESLint as the repo-owned linter for src/ and scripts/, with an explicit carve-out for operator-facing migration CLI console output."; + +const REPO_ROOT = path.resolve(import.meta.dir, ".."); +const PACKAGE_JSON_PATH = path.resolve(REPO_ROOT, "package.json"); +const CI_WORKFLOW_PATH = path.resolve(REPO_ROOT, ".github/workflows/ci.yml"); +const DECISIONS_PATH = path.resolve(REPO_ROOT, ".gsd/DECISIONS.md"); + +export const M058_S03_CHECK_IDS = [ + "M058-S03-PACKAGE-WIRING", + "M058-S03-CI-WIRING", + "M058-S03-CI-RATIONALE", + "M058-S03-DECISION-RECORD", +] as const; + +export type M058S03CheckId = (typeof M058_S03_CHECK_IDS)[number]; + +export type Check = { + id: M058S03CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M058S03CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EvaluateOptions = { + generatedAt?: string; + readTextFile?: (filePath: string) => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +type PackageJsonShape = { + scripts?: Record | unknown; +}; + +export async function evaluateM058S03Proof( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const readTextFile = options.readTextFile ?? defaultReadTextFile; + + const packageContent = await readOptionalTextFile(readTextFile, PACKAGE_JSON_PATH); + const ciContent = await readOptionalTextFile(readTextFile, CI_WORKFLOW_PATH); + const decisionsContent = await readOptionalTextFile(readTextFile, DECISIONS_PATH); + + const parsedPackageJson = packageContent.ok + ? parsePackageJson(packageContent.content) + : { ok: false as const, error: packageContent.error }; + + const checks: Check[] = [ + packageContent.ok + ? buildPackageWiringCheck(parsedPackageJson) + : failCheck("M058-S03-PACKAGE-WIRING", "package_file_unreadable", packageContent.error), + ciContent.ok + ? buildCiWiringCheck(ciContent.content) + : failCheck("M058-S03-CI-WIRING", "ci_file_unreadable", ciContent.error), + ciContent.ok + ? buildCiRationaleCheck(ciContent.content) + : failCheck("M058-S03-CI-RATIONALE", "ci_file_unreadable", ciContent.error), + decisionsContent.ok + ? buildDecisionRecordCheck(decisionsContent.content) + : failCheck( + "M058-S03-DECISION-RECORD", + "decision_file_unreadable", + decisionsContent.error, + ), + ]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M058_S03_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM058S03Report(report: EvaluationReport): string { + const lines = [ + "M058 S03 CI gate contract verifier", + `Generated at: ${report.generatedAt}`, + `CI gate proof surface: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM058S03ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM058S03Proof(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM058S03Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`${COMMAND_NAME} failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM058S03Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +function buildPackageWiringCheck(parsedPackageJson: ReturnType): Check { + if (!parsedPackageJson.ok) { + return failCheck("M058-S03-PACKAGE-WIRING", "package_json_invalid", parsedPackageJson.error); + } + + const scripts = parsedPackageJson.value.scripts; + if (typeof scripts !== "object" || scripts == null || Array.isArray(scripts)) { + return failCheck( + "M058-S03-PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.lint=${EXPECTED_LINT_SCRIPT} and scripts.${COMMAND_NAME}=${EXPECTED_VERIFY_SCRIPT}`, + ); + } + + const scriptMap = scripts as Record; + const lintScript = scriptMap.lint; + const verifyScript = scriptMap[COMMAND_NAME]; + + if (lintScript == null || verifyScript == null) { + return failCheck( + "M058-S03-PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.lint=${EXPECTED_LINT_SCRIPT} and scripts.${COMMAND_NAME}=${EXPECTED_VERIFY_SCRIPT}`, + ); + } + + if (lintScript !== EXPECTED_LINT_SCRIPT) { + return failCheck( + "M058-S03-PACKAGE-WIRING", + "lint_script_incorrect", + `Expected scripts.lint=${EXPECTED_LINT_SCRIPT} but found ${lintScript}`, + ); + } + + if (verifyScript !== EXPECTED_VERIFY_SCRIPT) { + return failCheck( + "M058-S03-PACKAGE-WIRING", + "verify_script_incorrect", + `Expected scripts.${COMMAND_NAME}=${EXPECTED_VERIFY_SCRIPT} but found ${verifyScript}`, + ); + } + + return passCheck( + "M058-S03-PACKAGE-WIRING", + "package_wiring_ok", + `package.json wires lint to ${EXPECTED_LINT_SCRIPT} and ${COMMAND_NAME} to ${EXPECTED_VERIFY_SCRIPT}`, + ); +} + +function buildCiWiringCheck(ciContent: string): Check { + const lintStepIndex = indexOfRunStep(ciContent, EXPECTED_LINT_STEP); + if (lintStepIndex === -1) { + return failCheck( + "M058-S03-CI-WIRING", + "ci_lint_step_missing", + `.github/workflows/ci.yml must include ${EXPECTED_LINT_STEP} before the heavier Bun test steps.`, + ); + } + + const migrationVerifyIndex = indexOfRunStep(ciContent, EXPECTED_MIGRATION_VERIFY_STEP); + if (migrationVerifyIndex === -1) { + return failCheck( + "M058-S03-CI-WIRING", + "ci_migration_verify_step_missing", + `.github/workflows/ci.yml must include ${EXPECTED_MIGRATION_VERIFY_STEP} in the structural gate bundle.`, + ); + } + + const orphanedTestIndex = indexOfRunStep(ciContent, EXPECTED_ORPHANED_TEST_STEP); + if (orphanedTestIndex === -1) { + return failCheck( + "M058-S03-CI-WIRING", + "ci_orphaned_test_step_missing", + `.github/workflows/ci.yml must include ${EXPECTED_ORPHANED_TEST_STEP} before the heavier Bun test steps.`, + ); + } + + if (lintStepIndex > migrationVerifyIndex) { + return failCheck( + "M058-S03-CI-WIRING", + "ci_gate_steps_misordered", + `${EXPECTED_LINT_STEP} must appear before ${EXPECTED_MIGRATION_VERIFY_STEP} in .github/workflows/ci.yml.`, + ); + } + + if (migrationVerifyIndex > orphanedTestIndex) { + return failCheck( + "M058-S03-CI-WIRING", + "ci_gate_steps_misordered", + `${EXPECTED_MIGRATION_VERIFY_STEP} must appear before ${EXPECTED_ORPHANED_TEST_STEP} in .github/workflows/ci.yml.`, + ); + } + + for (const marker of [EXPECTED_BROAD_TEST_STEP, EXPECTED_KNOWLEDGE_TEST_STEP] as const) { + const markerIndex = indexOfRunStep(ciContent, marker); + if (markerIndex !== -1 && orphanedTestIndex > markerIndex) { + return failCheck( + "M058-S03-CI-WIRING", + "ci_gate_steps_misordered", + `${EXPECTED_ORPHANED_TEST_STEP} must appear before ${marker} in .github/workflows/ci.yml.`, + ); + } + } + + return passCheck( + "M058-S03-CI-WIRING", + "ci_wiring_ok", + `.github/workflows/ci.yml runs ${EXPECTED_LINT_STEP}, ${EXPECTED_MIGRATION_VERIFY_STEP}, and ${EXPECTED_ORPHANED_TEST_STEP} before the heavier Bun test steps.`, + ); +} + +function buildCiRationaleCheck(ciContent: string): Check { + const missingRationaleMarkers = REQUIRED_SPLIT_RATIONALE_MARKERS.filter( + (marker) => !ciContent.includes(marker), + ); + if (missingRationaleMarkers.length > 0) { + return failCheck( + "M058-S03-CI-RATIONALE", + "ci_split_rationale_comment_missing", + `.github/workflows/ci.yml must preserve the split rationale comment markers: ${missingRationaleMarkers.join(", ")}`, + ); + } + + if (!hasExactRunStep(ciContent, EXPECTED_BROAD_TEST_STEP)) { + return failCheck( + "M058-S03-CI-RATIONALE", + "ci_broad_test_step_missing", + `.github/workflows/ci.yml must retain ${EXPECTED_BROAD_TEST_STEP} as the broadened test step.`, + ); + } + + if (!hasExactRunStep(ciContent, EXPECTED_KNOWLEDGE_TEST_STEP)) { + return failCheck( + "M058-S03-CI-RATIONALE", + "ci_knowledge_test_step_missing", + `.github/workflows/ci.yml must retain ${EXPECTED_KNOWLEDGE_TEST_STEP} as the isolated knowledge test step.`, + ); + } + + return passCheck( + "M058-S03-CI-RATIONALE", + "ci_rationale_ok", + `.github/workflows/ci.yml preserves the split Bun-test rationale comment plus both split test steps.`, + ); +} + +function buildDecisionRecordCheck(decisionsContent: string): Check { + if (!decisionsContent.includes(DECISION_MARKER)) { + return failCheck( + "M058-S03-DECISION-RECORD", + "decision_marker_missing", + `.gsd/DECISIONS.md must include the ${DECISION_MARKER} marker alongside the lint-tool rationale.`, + ); + } + + if (!decisionsContent.includes(DECISION_REQUIRED_TEXT)) { + return failCheck( + "M058-S03-DECISION-RECORD", + "decision_contract_text_missing", + `.gsd/DECISIONS.md must record: ${DECISION_REQUIRED_TEXT}`, + ); + } + + return passCheck( + "M058-S03-DECISION-RECORD", + "decision_record_ok", + `.gsd/DECISIONS.md records the lint-tool contract with marker ${DECISION_MARKER}.`, + ); +} + +function parsePackageJson(content: string): + | { ok: true; value: PackageJsonShape } + | { ok: false; error: unknown } { + try { + return { ok: true, value: JSON.parse(content) as PackageJsonShape }; + } catch (error) { + return { ok: false, error }; + } +} + +function hasExactRunStep(ciContent: string, command: string): boolean { + return runStepMatch(ciContent, command) != null; +} + +function indexOfRunStep(ciContent: string, command: string): number { + return runStepMatch(ciContent, command)?.index ?? -1; +} + +function runStepMatch(ciContent: string, command: string): RegExpMatchArray | null { + return ciContent.match(new RegExp(`^\\s*- run: ${escapeForRegex(command)}\\s*$`, "m")); +} + +function escapeForRegex(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function passCheck(id: M058S03CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M058S03CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +async function readOptionalTextFile( + readTextFile: (filePath: string) => Promise, + filePath: string, +): Promise<{ ok: true; content: string } | { ok: false; error: unknown }> { + try { + return { ok: true, content: await readTextFile(filePath) }; + } catch (error) { + return { ok: false, error }; + } +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +if (import.meta.main) { + try { + const args = parseM058S03Args(process.argv.slice(2)); + const { exitCode } = await buildM058S03ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${COMMAND_NAME} failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m059-s01.test.ts b/scripts/verify-m059-s01.test.ts new file mode 100644 index 00000000..ae67c238 --- /dev/null +++ b/scripts/verify-m059-s01.test.ts @@ -0,0 +1,391 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport } from "./verify-m059-s01.ts"; +import { + M059_S01_CHECK_IDS, + buildM059S01ProofHarness, + evaluateM059S01Proof, + parseM059S01Args, + renderM059S01Report, +} from "./verify-m059-s01.ts"; + +const EXPECTED_CHECK_IDS = [ + "M059-S01-REGISTRY-COVERAGE", + "M059-S01-REGISTRY-DUPLICATES", + "M059-S01-REGISTRY-USAGE-TRUTH", + "M059-S01-SCOPE-CONTRACT", + "M059-S01-PACKAGE-WIRING", +] as const; + +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/verify-m059-s01.ts"; +const TRACKED_FILES = [ + "scripts/verify-m059-s01.ts", + "scripts/backfill-issues.ts", + "scripts/sync-triage-reactions.ts", + "scripts/helpers/shared.sh", +] as const; + +const PASSING_PACKAGE_JSON = JSON.stringify( + { + name: "kodiai", + scripts: { + "verify:m059:s01": EXPECTED_PACKAGE_SCRIPT, + "backfill:issues": "bun scripts/backfill-issues.ts --sync", + }, + }, + null, + 2, +); + +const PASSING_CI = `name: ci +jobs: + test: + steps: + - run: bun run verify:m059:s01 --json +`; + +const PASSING_NIGHTLY_ISSUE = `name: nightly-issue-sync +jobs: + sync: + steps: + - run: bun scripts/backfill-issues.ts --sync +`; + +const PASSING_NIGHTLY_REACTION = `name: nightly-reaction-sync +jobs: + sync: + steps: + - run: bun scripts/sync-triage-reactions.ts +`; + +const PASSING_REGISTRY = `# Script Registry + + + +| path | purpose | owner | lifecycle | usage | +| --- | --- | --- | --- | --- | +| scripts/verify-m059-s01.ts | Verifies the script registry contract. | M059 | active | package:verify:m059:s01, workflow:.github/workflows/ci.yml#bun run verify:m059:s01 --json | +| scripts/backfill-issues.ts | Syncs GitHub issues into the local store. | M012 | active | package:backfill:issues, workflow:.github/workflows/nightly-issue-sync.yml#bun scripts/backfill-issues.ts --sync | +| scripts/sync-triage-reactions.ts | Syncs triage comment reactions. | M021 | active | workflow:.github/workflows/nightly-reaction-sync.yml#bun scripts/sync-triage-reactions.ts | +| scripts/helpers/shared.sh | Shared shell helpers for script wrappers. | M059 | internal | none | +`; + +describe("verify m059 s01 proof harness", () => { + test("exports stable check ids and cli parsing", () => { + expect(M059_S01_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM059S01Args([])).toEqual({ json: false }); + expect(parseM059S01Args(["--json"])).toEqual({ json: true }); + expect(() => parseM059S01Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes when registry coverage, usage truth, scope contract, and package wiring all align", async () => { + const report = await evaluateM059S01Proof({ + generatedAt: "2026-04-21T12:00:00.000Z", + listTrackedScriptFiles: async () => [...TRACKED_FILES], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("scripts/REGISTRY.md")) return PASSING_REGISTRY; + if (filePath.endsWith("ci.yml")) return PASSING_CI; + if (filePath.endsWith("nightly-issue-sync.yml")) return PASSING_NIGHTLY_ISSUE; + if (filePath.endsWith("nightly-reaction-sync.yml")) return PASSING_NIGHTLY_REACTION; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.command).toBe("verify:m059:s01"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M059-S01-REGISTRY-COVERAGE", + passed: true, + status_code: "registry_coverage_ok", + }), + expect.objectContaining({ + id: "M059-S01-REGISTRY-DUPLICATES", + passed: true, + status_code: "registry_duplicates_ok", + }), + expect.objectContaining({ + id: "M059-S01-REGISTRY-USAGE-TRUTH", + passed: true, + status_code: "registry_usage_truth_ok", + }), + expect.objectContaining({ + id: "M059-S01-SCOPE-CONTRACT", + passed: true, + status_code: "scope_contract_ok", + }), + expect.objectContaining({ + id: "M059-S01-PACKAGE-WIRING", + passed: true, + status_code: "package_wiring_ok", + }), + ]); + + const rendered = renderM059S01Report(report); + expect(rendered).toContain("M059 S01 script registry verifier"); + expect(rendered).toContain("Script registry proof surface: PASS"); + expect(rendered).toContain("M059-S01-REGISTRY-COVERAGE PASS"); + expect(rendered).toContain("M059-S01-REGISTRY-DUPLICATES PASS"); + expect(rendered).toContain("M059-S01-REGISTRY-USAGE-TRUTH PASS"); + expect(rendered).toContain("M059-S01-SCOPE-CONTRACT PASS"); + expect(rendered).toContain("M059-S01-PACKAGE-WIRING PASS"); + }); + + test("ignores non-script tracked files under scripts/ when checking coverage", async () => { + const report = await evaluateM059S01Proof({ + listTrackedScriptFiles: async () => [...TRACKED_FILES, "scripts/REGISTRY.md"], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("scripts/REGISTRY.md")) return PASSING_REGISTRY; + if (filePath.endsWith("ci.yml")) return PASSING_CI; + if (filePath.endsWith("nightly-issue-sync.yml")) return PASSING_NIGHTLY_ISSUE; + if (filePath.endsWith("nightly-reaction-sync.yml")) return PASSING_NIGHTLY_REACTION; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.checks[0]).toEqual( + expect.objectContaining({ + id: "M059-S01-REGISTRY-COVERAGE", + passed: true, + status_code: "registry_coverage_ok", + }), + ); + }); + + test("ignores deleted working-tree paths when listTrackedScriptFiles still reports them", async () => { + const report = await evaluateM059S01Proof({ + listTrackedScriptFiles: async () => [...TRACKED_FILES], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("scripts/REGISTRY.md")) { + return `${PASSING_REGISTRY}| scripts/deleted-but-tracked.ts | Removed local file awaiting commit. | M059 | sunset | none |\n`; + } + if (filePath.endsWith("ci.yml")) return PASSING_CI; + if (filePath.endsWith("nightly-issue-sync.yml")) return PASSING_NIGHTLY_ISSUE; + if (filePath.endsWith("nightly-reaction-sync.yml")) return PASSING_NIGHTLY_REACTION; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.checks[0]).toEqual( + expect.objectContaining({ + id: "M059-S01-REGISTRY-COVERAGE", + passed: false, + status_code: "registry_rows_missing", + }), + ); + expect(report.checks[0]?.detail).toContain("Registry rows without tracked files: scripts/deleted-but-tracked.ts"); + }); + + test("flags missing registry coverage, duplicate rows, and missing package wiring with stable status codes", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM059S01ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + listTrackedScriptFiles: async () => [...TRACKED_FILES], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return JSON.stringify({ name: "kodiai", scripts: {} }); + } + if (filePath.endsWith("scripts/REGISTRY.md")) { + return `# Script Registry + +| path | purpose | owner | lifecycle | usage | +| --- | --- | --- | --- | --- | +| scripts/backfill-issues.ts | Syncs issues. | M012 | active | package:backfill:issues | +| scripts/backfill-issues.ts | Duplicate row. | M012 | active | package:backfill:issues | +| scripts/sync-triage-reactions.ts | Syncs reactions. | M021 | active | workflow:.github/workflows/nightly-reaction-sync.yml#bun scripts/sync-triage-reactions.ts | +`; + } + if (filePath.endsWith("ci.yml")) return PASSING_CI; + if (filePath.endsWith("nightly-issue-sync.yml")) return PASSING_NIGHTLY_ISSUE; + if (filePath.endsWith("nightly-reaction-sync.yml")) return PASSING_NIGHTLY_REACTION; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + + expect(result.exitCode).toBe(1); + expect(report.checks[0]).toEqual( + expect.objectContaining({ + id: "M059-S01-REGISTRY-COVERAGE", + passed: false, + status_code: "registry_rows_missing", + }), + ); + expect(report.checks[1]).toEqual( + expect.objectContaining({ + id: "M059-S01-REGISTRY-DUPLICATES", + passed: false, + status_code: "duplicate_row", + }), + ); + expect(report.checks[4]).toEqual( + expect.objectContaining({ + id: "M059-S01-PACKAGE-WIRING", + passed: false, + status_code: "package_wiring_missing", + }), + ); + expect(stderr.join(" ")).toContain("registry_rows_missing"); + expect(stderr.join(" ")).toContain("duplicate_row"); + expect(stderr.join(" ")).toContain("package_wiring_missing"); + }); + + test("flags malformed registry schema, unknown lifecycle, and missing usage values", async () => { + const report = await evaluateM059S01Proof({ + listTrackedScriptFiles: async () => [...TRACKED_FILES], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("scripts/REGISTRY.md")) { + return `# Script Registry + +| path | purpose | owner | lifecycle | usage | +| --- | --- | --- | --- | --- | +| scripts/verify-m059-s01.ts | Missing usage value | M059 | sunset | +`; + } + if (filePath.endsWith("ci.yml")) return PASSING_CI; + if (filePath.endsWith("nightly-issue-sync.yml")) return PASSING_NIGHTLY_ISSUE; + if (filePath.endsWith("nightly-reaction-sync.yml")) return PASSING_NIGHTLY_REACTION; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.checks[0]).toEqual( + expect.objectContaining({ + id: "M059-S01-REGISTRY-COVERAGE", + passed: false, + status_code: "registry_schema_invalid", + }), + ); + expect(report.checks[2]).toEqual( + expect.objectContaining({ + id: "M059-S01-REGISTRY-USAGE-TRUTH", + passed: false, + status_code: "registry_schema_invalid", + }), + ); + }); + + test("flags stale package and workflow usage references, including usage none drift", async () => { + const report = await evaluateM059S01Proof({ + listTrackedScriptFiles: async () => [...TRACKED_FILES], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("scripts/REGISTRY.md")) { + return `# Script Registry + + + +| path | purpose | owner | lifecycle | usage | +| --- | --- | --- | --- | --- | +| scripts/verify-m059-s01.ts | Verifies the script registry contract. | M059 | active | none | +| scripts/backfill-issues.ts | Syncs GitHub issues into the local store. | M012 | active | package:backfill:missing | +| scripts/sync-triage-reactions.ts | Syncs triage comment reactions. | M021 | active | workflow:.github/workflows/nightly-reaction-sync.yml#bun scripts/does-not-exist.ts | +| scripts/helpers/shared.sh | Shared shell helpers for script wrappers. | M059 | internal | none | +`; + } + if (filePath.endsWith("ci.yml")) return PASSING_CI; + if (filePath.endsWith("nightly-issue-sync.yml")) return PASSING_NIGHTLY_ISSUE; + if (filePath.endsWith("nightly-reaction-sync.yml")) return PASSING_NIGHTLY_REACTION; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.checks[2]).toEqual( + expect.objectContaining({ + id: "M059-S01-REGISTRY-USAGE-TRUTH", + passed: false, + status_code: "usage_reference_invalid", + }), + ); + expect(report.checks[2]?.detail).toContain("package:backfill:missing"); + expect(report.checks[2]?.detail).toContain("scripts/does-not-exist.ts"); + expect(report.checks[2]?.detail).toContain("usage none"); + }); + + test("flags missing scope declaration when tracked shell helpers are omitted", async () => { + const report = await evaluateM059S01Proof({ + listTrackedScriptFiles: async () => [...TRACKED_FILES], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("scripts/REGISTRY.md")) { + return `# Script Registry + +| path | purpose | owner | lifecycle | usage | +| --- | --- | --- | --- | --- | +| scripts/verify-m059-s01.ts | Verifies the script registry contract. | M059 | active | package:verify:m059:s01 | +| scripts/backfill-issues.ts | Syncs GitHub issues into the local store. | M012 | active | package:backfill:issues | +| scripts/sync-triage-reactions.ts | Syncs triage comment reactions. | M021 | active | workflow:.github/workflows/nightly-reaction-sync.yml#bun scripts/sync-triage-reactions.ts | +`; + } + if (filePath.endsWith("ci.yml")) return PASSING_CI; + if (filePath.endsWith("nightly-issue-sync.yml")) return PASSING_NIGHTLY_ISSUE; + if (filePath.endsWith("nightly-reaction-sync.yml")) return PASSING_NIGHTLY_REACTION; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.checks[3]).toEqual( + expect.objectContaining({ + id: "M059-S01-SCOPE-CONTRACT", + passed: false, + status_code: "scope_contract_missing", + }), + ); + }); + + test("flags missing registry file, unreadable workflow file, and invalid package json", async () => { + const report = await evaluateM059S01Proof({ + listTrackedScriptFiles: async () => [...TRACKED_FILES], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return "{ not valid json"; + if (filePath.endsWith("scripts/REGISTRY.md")) throw new Error("ENOENT: REGISTRY.md"); + if (filePath.endsWith("ci.yml")) throw new Error("EACCES: ci.yml"); + if (filePath.endsWith("nightly-issue-sync.yml")) return PASSING_NIGHTLY_ISSUE; + if (filePath.endsWith("nightly-reaction-sync.yml")) return PASSING_NIGHTLY_REACTION; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.checks[0]).toEqual( + expect.objectContaining({ + id: "M059-S01-REGISTRY-COVERAGE", + passed: false, + status_code: "registry_missing", + }), + ); + expect(report.checks[2]).toEqual( + expect.objectContaining({ + id: "M059-S01-REGISTRY-USAGE-TRUTH", + passed: false, + status_code: "workflow_file_unreadable", + }), + ); + expect(report.checks[4]).toEqual( + expect.objectContaining({ + id: "M059-S01-PACKAGE-WIRING", + passed: false, + status_code: "package_json_invalid", + }), + ); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m059:s01"]).toBe( + "bun scripts/verify-m059-s01.ts", + ); + }); +}); diff --git a/scripts/verify-m059-s01.ts b/scripts/verify-m059-s01.ts new file mode 100644 index 00000000..ef69b361 --- /dev/null +++ b/scripts/verify-m059-s01.ts @@ -0,0 +1,710 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +const COMMAND_NAME = "verify:m059:s01" as const; +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/verify-m059-s01.ts"; +const REPO_ROOT = path.resolve(import.meta.dir, ".."); +const PACKAGE_JSON_PATH = path.resolve(REPO_ROOT, "package.json"); +const REGISTRY_PATH = path.resolve(REPO_ROOT, "scripts/REGISTRY.md"); +const WORKFLOW_PATHS = [ + ".github/workflows/ci.yml", + ".github/workflows/nightly-issue-sync.yml", + ".github/workflows/nightly-reaction-sync.yml", +] as const; +const ALLOWED_LIFECYCLES = ["active", "internal", "deprecated", "sunset"] as const; +export const REGISTRY_HEADER = ["path", "purpose", "owner", "lifecycle", "usage"] as const; +const TRACKED_SCRIPT_EXTENSIONS = [".ts", ".sh"] as const; + +export const M059_S01_CHECK_IDS = [ + "M059-S01-REGISTRY-COVERAGE", + "M059-S01-REGISTRY-DUPLICATES", + "M059-S01-REGISTRY-USAGE-TRUTH", + "M059-S01-SCOPE-CONTRACT", + "M059-S01-PACKAGE-WIRING", +] as const; + +export type M059S01CheckId = (typeof M059_S01_CHECK_IDS)[number]; + +export type Check = { + id: M059S01CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M059S01CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { write: (chunk: string) => boolean | void }; + +type EvaluateOptions = { + generatedAt?: string; + readTextFile?: (filePath: string) => Promise; + listTrackedScriptFiles?: () => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +type ReadResult = + | { ok: true; content: string } + | { ok: false; error: unknown }; + +type PackageJsonShape = { + scripts?: Record | unknown; +}; + +type UsageRef = + | { kind: "none" } + | { kind: "package"; name: string } + | { kind: "workflow"; workflowPath: string; command: string }; + +export type RegistryRow = { + path: string; + purpose: string; + owner: string; + lifecycle: string; + usageRaw: string; + usageRefs: UsageRef[]; + sourceLine: number; +}; + +export type ParsedRegistry = + | { ok: true; rows: RegistryRow[]; scopeDeclarationPresent: boolean } + | { ok: false; status_code: string; detail: string; scopeDeclarationPresent: boolean }; + +export async function evaluateM059S01Proof( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const readTextFile = options.readTextFile ?? defaultReadTextFile; + const listTrackedScriptFiles = options.listTrackedScriptFiles ?? defaultListTrackedScriptFiles; + + const [packageContent, registryContent, workflowContents, trackedFilesResult] = await Promise.all([ + readOptionalTextFile(readTextFile, PACKAGE_JSON_PATH), + readOptionalTextFile(readTextFile, REGISTRY_PATH), + Promise.all( + WORKFLOW_PATHS.map(async (relativePath) => ({ + relativePath, + result: await readOptionalTextFile(readTextFile, path.resolve(REPO_ROOT, relativePath)), + })), + ), + readTrackedScriptFiles(listTrackedScriptFiles), + ]); + + const parsedRegistry = parseRegistryReadResult(registryContent); + const parsedPackageJson = parsePackageJsonResult(packageContent); + + const checks: Check[] = [ + buildRegistryCoverageCheck(parsedRegistry, trackedFilesResult), + buildRegistryDuplicatesCheck(parsedRegistry), + buildRegistryUsageTruthCheck(parsedRegistry, parsedPackageJson, workflowContents), + buildScopeContractCheck(parsedRegistry, trackedFilesResult), + buildPackageWiringCheck(parsedPackageJson), + ]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M059_S01_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM059S01Report(report: EvaluationReport): string { + const lines = [ + "M059 S01 script registry verifier", + `Generated at: ${report.generatedAt}`, + `Script registry proof surface: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM059S01ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM059S01Proof(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM059S01Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`${COMMAND_NAME} failed: ${failingCodes}\n`); + } + + return { exitCode: report.overallPassed ? 0 : 1, report }; +} + +export function parseM059S01Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +function buildRegistryCoverageCheck(parsedRegistry: ParsedRegistry, trackedFilesResult: Awaited>): Check { + if (!trackedFilesResult.ok) { + return failCheck( + "M059-S01-REGISTRY-COVERAGE", + "tracked_files_unreadable", + trackedFilesResult.error, + ); + } + + if (!parsedRegistry.ok) { + return failCheck("M059-S01-REGISTRY-COVERAGE", parsedRegistry.status_code, parsedRegistry.detail); + } + + const trackedFiles = trackedFilesResult.files; + const rowPaths = new Set(parsedRegistry.rows.map((row) => row.path)); + const missingPaths = trackedFiles.filter((filePath) => !rowPaths.has(filePath)); + const extraPaths = parsedRegistry.rows + .map((row) => row.path) + .filter((filePath) => !trackedFiles.includes(filePath)); + + if (missingPaths.length > 0 || extraPaths.length > 0) { + const detailParts: string[] = []; + if (missingPaths.length > 0) { + detailParts.push(`Missing rows for tracked scripts: ${missingPaths.join(", ")}`); + } + if (extraPaths.length > 0) { + detailParts.push(`Registry rows without tracked files: ${extraPaths.join(", ")}`); + } + return failCheck("M059-S01-REGISTRY-COVERAGE", "registry_rows_missing", detailParts.join(". ")); + } + + return passCheck( + "M059-S01-REGISTRY-COVERAGE", + "registry_coverage_ok", + `Registry covers ${trackedFiles.length} tracked scripts with no stale rows.`, + ); +} + +function buildRegistryDuplicatesCheck(parsedRegistry: ParsedRegistry): Check { + if (!parsedRegistry.ok) { + return failCheck("M059-S01-REGISTRY-DUPLICATES", parsedRegistry.status_code, parsedRegistry.detail); + } + + const seen = new Map(); + for (const row of parsedRegistry.rows) { + const lines = seen.get(row.path) ?? []; + lines.push(row.sourceLine); + seen.set(row.path, lines); + } + + const duplicates = [...seen.entries()] + .filter(([, lines]) => lines.length > 1) + .map(([filePath, lines]) => `${filePath} (lines ${lines.join(", ")})`); + + if (duplicates.length > 0) { + return failCheck("M059-S01-REGISTRY-DUPLICATES", "duplicate_row", duplicates.join("; ")); + } + + return passCheck( + "M059-S01-REGISTRY-DUPLICATES", + "registry_duplicates_ok", + `Registry paths are unique across ${parsedRegistry.rows.length} rows.`, + ); +} + +function buildRegistryUsageTruthCheck( + parsedRegistry: ParsedRegistry, + parsedPackageJson: ReturnType, + workflowContents: Array<{ relativePath: (typeof WORKFLOW_PATHS)[number]; result: ReadResult }>, +): Check { + for (const workflow of workflowContents) { + if (!workflow.result.ok) { + return failCheck( + "M059-S01-REGISTRY-USAGE-TRUTH", + "workflow_file_unreadable", + `${workflow.relativePath}: ${normalizeDetail(workflow.result.error)}`, + ); + } + } + + if (!parsedRegistry.ok) { + if (parsedRegistry.status_code === "registry_missing" || parsedRegistry.status_code === "registry_file_unreadable") { + return failCheck("M059-S01-REGISTRY-USAGE-TRUTH", parsedRegistry.status_code, parsedRegistry.detail); + } + return failCheck("M059-S01-REGISTRY-USAGE-TRUTH", "registry_schema_invalid", parsedRegistry.detail); + } + + if (!parsedPackageJson.ok) { + return failCheck("M059-S01-REGISTRY-USAGE-TRUTH", "package_json_invalid", parsedPackageJson.error); + } + + const scripts = normalizeScriptsMap(parsedPackageJson.value); + if (!scripts.ok) { + return failCheck("M059-S01-REGISTRY-USAGE-TRUTH", "package_json_invalid", scripts.error); + } + + const workflowMap = new Map( + workflowContents + .filter((workflow): workflow is { relativePath: (typeof WORKFLOW_PATHS)[number]; result: { ok: true; content: string } } => workflow.result.ok) + .map((workflow) => [workflow.relativePath, workflow.result.content]), + ); + + const problems: string[] = []; + + for (const row of parsedRegistry.rows) { + if (row.usageRefs.length === 1 && row.usageRefs[0]?.kind === "none") { + const actualPackageRefs = Object.entries(scripts.value) + .filter(([, command]) => command.includes(row.path)) + .map(([name]) => `package:${name}`); + const actualWorkflowRefs = [...workflowMap.entries()] + .filter(([, content]) => content.includes(row.path)) + .map(([workflowPath]) => `workflow:${workflowPath}`); + const actualRefs = [...actualPackageRefs, ...actualWorkflowRefs]; + if (actualRefs.length > 0) { + problems.push(`${row.path} declares usage none but actual references exist: ${actualRefs.join(", ")}`); + } + continue; + } + + for (const usageRef of row.usageRefs) { + if (usageRef.kind === "none") { + problems.push(`${row.path} mixes usage none with real references.`); + continue; + } + + if (usageRef.kind === "package") { + const scriptCommand = scripts.value[usageRef.name]; + if (scriptCommand == null) { + problems.push(`${row.path} references missing package script ${`package:${usageRef.name}`}.`); + continue; + } + if (!scriptCommand.includes(row.path)) { + problems.push(`${row.path} package:${usageRef.name} does not reference ${row.path}; found ${scriptCommand}`); + } + continue; + } + + const workflowContent = workflowMap.get(usageRef.workflowPath); + if (workflowContent == null) { + problems.push(`${row.path} references unknown workflow ${usageRef.workflowPath}.`); + continue; + } + if (!workflowContent.includes(usageRef.command)) { + problems.push(`${row.path} workflow reference invalid: ${usageRef.workflowPath} is missing command ${usageRef.command}.`); + continue; + } + if (!workflowCommandReferencesPath(usageRef.command, row.path, scripts.value)) { + problems.push(`${row.path} workflow reference ${usageRef.workflowPath}#${usageRef.command} does not reference ${row.path}.`); + } + } + } + + if (problems.length > 0) { + return failCheck("M059-S01-REGISTRY-USAGE-TRUTH", "usage_reference_invalid", problems.join(" ")); + } + + return passCheck( + "M059-S01-REGISTRY-USAGE-TRUTH", + "registry_usage_truth_ok", + `Validated package/workflow references and explicit none semantics for ${parsedRegistry.rows.length} registry rows.`, + ); +} + +function buildScopeContractCheck(parsedRegistry: ParsedRegistry, trackedFilesResult: Awaited>): Check { + if (!trackedFilesResult.ok) { + return failCheck("M059-S01-SCOPE-CONTRACT", "tracked_files_unreadable", trackedFilesResult.error); + } + + const trackedShellFiles = trackedFilesResult.files.filter((filePath) => filePath.endsWith(".sh")); + + if (!parsedRegistry.ok) { + if (parsedRegistry.status_code === "registry_missing" || parsedRegistry.status_code === "registry_file_unreadable") { + return failCheck("M059-S01-SCOPE-CONTRACT", parsedRegistry.status_code, parsedRegistry.detail); + } + return failCheck("M059-S01-SCOPE-CONTRACT", "registry_schema_invalid", parsedRegistry.detail); + } + + const rowPaths = new Set(parsedRegistry.rows.map((row) => row.path)); + const missingShellRows = trackedShellFiles.filter((filePath) => !rowPaths.has(filePath)); + + if (missingShellRows.length > 0 && !parsedRegistry.scopeDeclarationPresent) { + return failCheck( + "M059-S01-SCOPE-CONTRACT", + "scope_contract_missing", + `Tracked shell helpers are omitted without an explicit scope declaration: ${missingShellRows.join(", ")}`, + ); + } + + return passCheck( + "M059-S01-SCOPE-CONTRACT", + "scope_contract_ok", + trackedShellFiles.length === 0 + ? "No tracked shell helpers under scripts/." + : `Scope declaration present=${parsedRegistry.scopeDeclarationPresent}; tracked shell helpers=${trackedShellFiles.length}.`, + ); +} + +function buildPackageWiringCheck(parsedPackageJson: ReturnType): Check { + if (!parsedPackageJson.ok) { + return failCheck("M059-S01-PACKAGE-WIRING", "package_json_invalid", parsedPackageJson.error); + } + + const scripts = normalizeScriptsMap(parsedPackageJson.value); + if (!scripts.ok) { + return failCheck("M059-S01-PACKAGE-WIRING", "package_json_invalid", scripts.error); + } + + const actualScript = scripts.value[COMMAND_NAME]; + if (actualScript == null) { + return failCheck( + "M059-S01-PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT}`, + ); + } + + if (actualScript !== EXPECTED_PACKAGE_SCRIPT) { + return failCheck( + "M059-S01-PACKAGE-WIRING", + "package_wiring_incorrect", + `Expected scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT} but found ${actualScript}`, + ); + } + + return passCheck( + "M059-S01-PACKAGE-WIRING", + "package_wiring_ok", + `package.json wires ${COMMAND_NAME} to ${EXPECTED_PACKAGE_SCRIPT}`, + ); +} + +function parseRegistryReadResult(result: ReadResult): ParsedRegistry { + if (!result.ok) { + const detail = normalizeDetail(result.error); + if (/ENOENT/i.test(detail)) { + return { ok: false, status_code: "registry_missing", detail, scopeDeclarationPresent: false }; + } + return { ok: false, status_code: "registry_file_unreadable", detail, scopeDeclarationPresent: false }; + } + + return parseRegistryContent(result.content); +} + +export function parseRegistryContent(content: string): ParsedRegistry { + const lines = content.split(/\r?\n/); + const scopeDeclarationPresent = /scope:/i.test(content); + const dividerIndex = lines.findIndex((line) => /^\|\s*---/.test(line.trim())); + if (dividerIndex <= 0) { + return { + ok: false, + status_code: "registry_schema_invalid", + detail: "scripts/REGISTRY.md must contain a markdown table with a header and divider row.", + scopeDeclarationPresent, + }; + } + + const headerCells = splitTableRow(lines[dividerIndex - 1] ?? ""); + if (headerCells.length !== REGISTRY_HEADER.length || !REGISTRY_HEADER.every((cell, index) => headerCells[index] === cell)) { + return { + ok: false, + status_code: "registry_schema_invalid", + detail: `Registry header must be exactly: ${REGISTRY_HEADER.join(", ")}`, + scopeDeclarationPresent, + }; + } + + const rows: RegistryRow[] = []; + for (let index = dividerIndex + 1; index < lines.length; index += 1) { + const line = lines[index] ?? ""; + const trimmed = line.trim(); + if (!trimmed.startsWith("|")) { + if (rows.length > 0 && /^##\s+/.test(trimmed)) { + break; + } + continue; + } + + const cells = splitTableRow(line); + if (cells.length !== REGISTRY_HEADER.length) { + return { + ok: false, + status_code: "registry_schema_invalid", + detail: `Line ${index + 1} must contain ${REGISTRY_HEADER.length} table cells; found ${cells.length}.`, + scopeDeclarationPresent, + }; + } + + const [filePath, purpose, owner, lifecycle, usageRaw] = cells; + if (!filePath || !purpose || !owner || !lifecycle || !usageRaw) { + return { + ok: false, + status_code: "registry_schema_invalid", + detail: `Line ${index + 1} must provide non-empty path, purpose, owner, lifecycle, and usage values.`, + scopeDeclarationPresent, + }; + } + + if (!ALLOWED_LIFECYCLES.includes(lifecycle as (typeof ALLOWED_LIFECYCLES)[number])) { + return { + ok: false, + status_code: "registry_schema_invalid", + detail: `Line ${index + 1} uses unknown lifecycle ${lifecycle}; allowed values: ${ALLOWED_LIFECYCLES.join(", ")}.`, + scopeDeclarationPresent, + }; + } + + const parsedUsage = parseUsageField(usageRaw, index + 1); + if (!parsedUsage.ok) { + return { + ok: false, + status_code: "registry_schema_invalid", + detail: parsedUsage.detail, + scopeDeclarationPresent, + }; + } + + rows.push({ + path: filePath, + purpose, + owner, + lifecycle, + usageRaw, + usageRefs: parsedUsage.usageRefs, + sourceLine: index + 1, + }); + } + + if (rows.length === 0) { + return { + ok: false, + status_code: "registry_schema_invalid", + detail: "Registry table must contain at least one data row.", + scopeDeclarationPresent, + }; + } + + return { ok: true, rows, scopeDeclarationPresent }; +} + +function parseUsageField(usageRaw: string, lineNumber: number): { ok: true; usageRefs: UsageRef[] } | { ok: false; detail: string } { + const parts = usageRaw.split(",").map((part) => part.trim()).filter(Boolean); + if (parts.length === 0) { + return { ok: false, detail: `Line ${lineNumber} must declare at least one usage value.` }; + } + + if (parts.includes("none") && parts.length > 1) { + return { ok: false, detail: `Line ${lineNumber} may not mix usage none with package/workflow references.` }; + } + + const usageRefs: UsageRef[] = []; + for (const part of parts) { + if (part === "none") { + usageRefs.push({ kind: "none" }); + continue; + } + + if (part.startsWith("package:")) { + const name = part.slice("package:".length).trim(); + if (!name) { + return { ok: false, detail: `Line ${lineNumber} has invalid package usage syntax: ${part}` }; + } + usageRefs.push({ kind: "package", name }); + continue; + } + + if (part.startsWith("workflow:")) { + const payload = part.slice("workflow:".length); + const hashIndex = payload.indexOf("#"); + if (hashIndex <= 0 || hashIndex === payload.length - 1) { + return { ok: false, detail: `Line ${lineNumber} has invalid workflow usage syntax: ${part}` }; + } + const workflowPath = payload.slice(0, hashIndex).trim(); + const command = payload.slice(hashIndex + 1).trim(); + if (!workflowPath || !command) { + return { ok: false, detail: `Line ${lineNumber} has invalid workflow usage syntax: ${part}` }; + } + usageRefs.push({ kind: "workflow", workflowPath, command }); + continue; + } + + return { ok: false, detail: `Line ${lineNumber} has unknown usage syntax: ${part}` }; + } + + return { ok: true, usageRefs }; +} + +function splitTableRow(line: string): string[] { + return line + .trim() + .replace(/^\|/, "") + .replace(/\|$/, "") + .split("|") + .map((cell) => cell.trim()); +} + +function workflowCommandReferencesPath( + command: string, + filePath: string, + packageScripts: Record, +): boolean { + if (command.includes(filePath)) { + return true; + } + + const bunRunMatch = command.match(/\bbun\s+run\s+([a-z0-9:_-]+)/i); + if (!bunRunMatch) { + return false; + } + + const packageScriptName = bunRunMatch[1]; + if (!packageScriptName) { + return false; + } + + const packageCommand = packageScripts[packageScriptName]; + return typeof packageCommand === "string" && packageCommand.includes(filePath); +} + +function parsePackageJsonResult(result: ReadResult): + | { ok: true; value: PackageJsonShape } + | { ok: false; error: unknown } { + if (!result.ok) { + return { ok: false, error: result.error }; + } + + try { + return { ok: true, value: JSON.parse(result.content) as PackageJsonShape }; + } catch (error) { + return { ok: false, error }; + } +} + +function normalizeScriptsMap(packageJson: PackageJsonShape): + | { ok: true; value: Record } + | { ok: false; error: unknown } { + if (typeof packageJson.scripts !== "object" || packageJson.scripts == null || Array.isArray(packageJson.scripts)) { + return { ok: true, value: {} }; + } + + const scripts: Record = {}; + for (const [name, command] of Object.entries(packageJson.scripts)) { + if (typeof command !== "string") { + return { ok: false, error: `package.json scripts.${name} must be a string command.` }; + } + scripts[name] = command; + } + + return { ok: true, value: scripts }; +} + +async function readOptionalTextFile( + readTextFile: (filePath: string) => Promise, + filePath: string, +): Promise { + try { + return { ok: true, content: await readTextFile(filePath) }; + } catch (error) { + return { ok: false, error }; + } +} + +async function readTrackedScriptFiles( + listTrackedScriptFiles: () => Promise, +): Promise<{ ok: true; files: string[] } | { ok: false; error: unknown }> { + try { + const files = (await listTrackedScriptFiles()) + .filter((filePath) => filePath.startsWith("scripts/")) + .filter((filePath) => TRACKED_SCRIPT_EXTENSIONS.some((extension) => filePath.endsWith(extension))) + .sort(); + return { ok: true, files }; + } catch (error) { + return { ok: false, error }; + } +} + +function passCheck(id: M059S01CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M059S01CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +async function defaultListTrackedScriptFiles(): Promise { + const entries = await Array.fromAsync(new Bun.Glob("scripts/**/*").scan({ + cwd: REPO_ROOT, + onlyFiles: true, + absolute: false, + })); + + return entries.sort(); +} + +if (import.meta.main) { + try { + const args = parseM059S01Args(process.argv.slice(2)); + const { exitCode } = await buildM059S01ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${COMMAND_NAME} failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m059-s02.test.ts b/scripts/verify-m059-s02.test.ts new file mode 100644 index 00000000..cea4fffe --- /dev/null +++ b/scripts/verify-m059-s02.test.ts @@ -0,0 +1,287 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport } from "./verify-m059-s02.ts"; +import { + M059_S02_CHECK_IDS, + buildM059S02ProofHarness, + evaluateM059S02Proof, + parseM059S02Args, + renderM059S02Report, +} from "./verify-m059-s02.ts"; + +const EXPECTED_CHECK_IDS = [ + "M059-S02-APPENDIX-COVERAGE", + "M059-S02-RETAINED-TRUTH", + "M059-S02-REMOVAL-TRUTH", + "M059-S02-PACKAGE-WIRING", +] as const; + +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/verify-m059-s02.ts"; +const PASSING_PACKAGE_JSON = JSON.stringify( + { + name: "kodiai", + scripts: { + "verify:m059:s02": EXPECTED_PACKAGE_SCRIPT, + }, + }, + null, + 2, +); + +const PASSING_REGISTRY = `# Script Registry + +This document is the canonical inventory for tracked files under \`scripts/\`. + +Scope: every tracked \`scripts/*.ts\`, \`scripts/*.test.ts\`, and \`scripts/*.sh\` file must appear exactly once in the table below. + +Usage contract: +- Use \`package:\` for package-script entrypoints from \`package.json\`. +- Use \`workflow:#\` for direct workflow commands. +- Use \`none\` only when no package-script or direct workflow command references the file. +- \`.sh\` helpers and wrappers are represented inline as first-class rows; they are not implied by a separate appendix. + +Lifecycle vocabulary: +- \`active\` — current operational or verification surface. +- \`internal\` — repo-local helper, test, or maintenance surface not intended as a primary operator entrypoint. +- \`deprecated\` — retained compatibility surface that should not gain new callers. +- \`sunset\` — retained only for bounded removal or historical verification. + +| path | purpose | owner | lifecycle | usage | +| --- | --- | --- | --- | --- | +| scripts/keep-me.ts | Retained orphan script. | M059 | active | none | +| scripts/verify-m059-s02.ts | S02 verifier. | M059 | active | package:verify:m059:s02 | + +## S02 Orphan Audit + +| path | disposition | rationale | +| --- | --- | --- | +| scripts/keep-me.ts | retained | Keep for explicit one-off operator recovery flow. | +| scripts/remove-me.ts | removed | Removed from the repo after the audit. | +`; + +describe("verify m059 s02 proof harness", () => { + test("exports stable check ids and cli parsing", () => { + expect(M059_S02_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM059S02Args([])).toEqual({ json: false }); + expect(parseM059S02Args(["--json"])).toEqual({ json: true }); + expect(() => parseM059S02Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes when every usage none registry row has a truthful orphan-audit disposition and package wiring matches", async () => { + const report = await evaluateM059S02Proof({ + generatedAt: "2026-04-21T16:00:00.000Z", + listTrackedScriptFiles: async () => ["scripts/keep-me.ts", "scripts/verify-m059-s02.ts"], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("scripts/REGISTRY.md")) return PASSING_REGISTRY; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.command).toBe("verify:m059:s02"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ + id: "M059-S02-APPENDIX-COVERAGE", + passed: true, + status_code: "appendix_coverage_ok", + }), + expect.objectContaining({ + id: "M059-S02-RETAINED-TRUTH", + passed: true, + status_code: "retained_truth_ok", + }), + expect.objectContaining({ + id: "M059-S02-REMOVAL-TRUTH", + passed: true, + status_code: "removal_truth_ok", + }), + expect.objectContaining({ + id: "M059-S02-PACKAGE-WIRING", + passed: true, + status_code: "package_wiring_ok", + }), + ]); + + const rendered = renderM059S02Report(report); + expect(rendered).toContain("M059 S02 orphan audit verifier"); + expect(rendered).toContain("Orphan audit proof surface: PASS"); + expect(rendered).toContain("M059-S02-APPENDIX-COVERAGE PASS"); + }); + + test("flags missing orphan-audit entries for live usage none rows", async () => { + const report = await evaluateM059S02Proof({ + listTrackedScriptFiles: async () => ["scripts/keep-me.ts", "scripts/verify-m059-s02.ts"], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("scripts/REGISTRY.md")) { + return PASSING_REGISTRY.replace( + "| scripts/keep-me.ts | retained | Keep for explicit one-off operator recovery flow. |\n", + "", + ); + } + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.checks[0]).toEqual( + expect.objectContaining({ + id: "M059-S02-APPENDIX-COVERAGE", + passed: false, + status_code: "orphan_audit_missing_row", + }), + ); + expect(report.checks[0]?.detail).toContain("scripts/keep-me.ts"); + }); + + test("flags malformed appendix rows", async () => { + const report = await evaluateM059S02Proof({ + listTrackedScriptFiles: async () => ["scripts/keep-me.ts", "scripts/verify-m059-s02.ts"], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("scripts/REGISTRY.md")) { + return PASSING_REGISTRY.replace( + "| scripts/remove-me.ts | removed | Removed from the repo after the audit. |", + "| scripts/remove-me.ts | retained | |", + ); + } + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.checks[0]).toEqual( + expect.objectContaining({ + id: "M059-S02-APPENDIX-COVERAGE", + passed: false, + status_code: "orphan_audit_malformed", + }), + ); + }); + + test("flags stale deleted rows left in the main registry table", async () => { + const report = await evaluateM059S02Proof({ + listTrackedScriptFiles: async () => ["scripts/keep-me.ts", "scripts/verify-m059-s02.ts"], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("scripts/REGISTRY.md")) { + return PASSING_REGISTRY.replace( + "| scripts/verify-m059-s02.ts | S02 verifier. | M059 | active | package:verify:m059:s02 |", + "| scripts/remove-me.ts | Deleted orphan script. | M059 | sunset | none |\n| scripts/verify-m059-s02.ts | S02 verifier. | M059 | active | package:verify:m059:s02 |", + ); + } + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.checks[2]).toEqual( + expect.objectContaining({ + id: "M059-S02-REMOVAL-TRUTH", + passed: false, + status_code: "orphan_removal_stale_registry_row", + }), + ); + }); + + test("flags removed appendix entries that still resolve to live tracked files", async () => { + const report = await evaluateM059S02Proof({ + listTrackedScriptFiles: async () => ["scripts/keep-me.ts", "scripts/remove-me.ts", "scripts/verify-m059-s02.ts"], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("scripts/REGISTRY.md")) return PASSING_REGISTRY; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.checks[2]).toEqual( + expect.objectContaining({ + id: "M059-S02-REMOVAL-TRUTH", + passed: false, + status_code: "orphan_removed_file_still_exists", + }), + ); + expect(report.checks[2]?.detail).toContain("scripts/remove-me.ts"); + }); + + test("flags appendix entries that lie about retained vs removed state", async () => { + const report = await evaluateM059S02Proof({ + listTrackedScriptFiles: async () => ["scripts/keep-me.ts", "scripts/verify-m059-s02.ts"], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) return PASSING_PACKAGE_JSON; + if (filePath.endsWith("scripts/REGISTRY.md")) { + return PASSING_REGISTRY + .replace( + "| scripts/keep-me.ts | retained | Keep for explicit one-off operator recovery flow. |", + "| scripts/keep-me.ts | removed | Claimed removed even though the file still exists. |", + ) + .replace( + "| scripts/remove-me.ts | removed | Removed from the repo after the audit. |", + "| scripts/remove-me.ts | retained | Claimed retained even though the file is gone. |", + ); + } + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + expect(report.checks[1]).toEqual( + expect.objectContaining({ + id: "M059-S02-RETAINED-TRUTH", + passed: false, + status_code: "orphan_retained_missing_file", + }), + ); + expect(report.checks[2]).toEqual( + expect.objectContaining({ + id: "M059-S02-REMOVAL-TRUTH", + passed: false, + status_code: "orphan_removed_file_still_exists", + }), + ); + }); + + test("flags missing or incorrect package wiring", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM059S02ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + listTrackedScriptFiles: async () => ["scripts/keep-me.ts", "scripts/verify-m059-s02.ts"], + readTextFile: async (filePath: string) => { + if (filePath.endsWith("package.json")) { + return JSON.stringify({ + name: "kodiai", + scripts: { + "verify:m059:s02": "bun scripts/not-the-right-file.ts", + }, + }); + } + if (filePath.endsWith("scripts/REGISTRY.md")) return PASSING_REGISTRY; + throw new Error(`Unexpected path: ${filePath}`); + }, + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + + expect(result.exitCode).toBe(1); + expect(report.checks[3]).toEqual( + expect.objectContaining({ + id: "M059-S02-PACKAGE-WIRING", + passed: false, + status_code: "package_wiring_incorrect", + }), + ); + expect(stderr.join(" ")).toContain("package_wiring_incorrect"); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m059:s02"]).toBe( + "bun scripts/verify-m059-s02.ts", + ); + }); +}); diff --git a/scripts/verify-m059-s02.ts b/scripts/verify-m059-s02.ts new file mode 100644 index 00000000..d80ab5e5 --- /dev/null +++ b/scripts/verify-m059-s02.ts @@ -0,0 +1,548 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { parseRegistryContent, type ParsedRegistry, type RegistryRow } from "./verify-m059-s01.ts"; + +const COMMAND_NAME = "verify:m059:s02" as const; +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/verify-m059-s02.ts"; +const REPO_ROOT = path.resolve(import.meta.dir, ".."); +const PACKAGE_JSON_PATH = path.resolve(REPO_ROOT, "package.json"); +const REGISTRY_PATH = path.resolve(REPO_ROOT, "scripts/REGISTRY.md"); +const APPENDIX_HEADING = "## S02 Orphan Audit"; +const APPENDIX_HEADER = ["path", "disposition", "rationale"] as const; + +export const M059_S02_CHECK_IDS = [ + "M059-S02-APPENDIX-COVERAGE", + "M059-S02-RETAINED-TRUTH", + "M059-S02-REMOVAL-TRUTH", + "M059-S02-PACKAGE-WIRING", +] as const; + +export type M059S02CheckId = (typeof M059_S02_CHECK_IDS)[number]; + +export type Check = { + id: M059S02CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M059S02CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { write: (chunk: string) => boolean | void }; + +type EvaluateOptions = { + generatedAt?: string; + readTextFile?: (filePath: string) => Promise; + listTrackedScriptFiles?: () => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +type ReadResult = + | { ok: true; content: string } + | { ok: false; error: unknown }; + +type PackageJsonShape = { + scripts?: Record | unknown; +}; + +type AppendixDisposition = "retained" | "removed"; + +type AppendixRow = { + path: string; + disposition: AppendixDisposition; + rationale: string; + sourceLine: number; +}; + +type ParsedAppendix = + | { ok: true; rows: AppendixRow[] } + | { ok: false; status_code: string; detail: string; rows: AppendixRow[] }; + +export async function evaluateM059S02Proof( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const readTextFile = options.readTextFile ?? defaultReadTextFile; + const listTrackedScriptFiles = options.listTrackedScriptFiles ?? defaultListTrackedScriptFiles; + + const [packageContent, registryContent, trackedFilesResult] = await Promise.all([ + readOptionalTextFile(readTextFile, PACKAGE_JSON_PATH), + readOptionalTextFile(readTextFile, REGISTRY_PATH), + readTrackedScriptFiles(listTrackedScriptFiles), + ]); + + const parsedRegistry = parseRegistryReadResult(registryContent); + const parsedAppendix = parseAppendixReadResult(registryContent); + const parsedPackageJson = parsePackageJsonResult(packageContent); + + const checks: Check[] = [ + buildAppendixCoverageCheck(parsedRegistry, parsedAppendix), + buildRetainedTruthCheck(parsedAppendix, trackedFilesResult), + buildRemovalTruthCheck(parsedRegistry, parsedAppendix, trackedFilesResult), + buildPackageWiringCheck(parsedPackageJson), + ]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M059_S02_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM059S02Report(report: EvaluationReport): string { + const lines = [ + "M059 S02 orphan audit verifier", + `Generated at: ${report.generatedAt}`, + `Orphan audit proof surface: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM059S02ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM059S02Proof(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM059S02Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`${COMMAND_NAME} failed: ${failingCodes}\n`); + } + + return { exitCode: report.overallPassed ? 0 : 1, report }; +} + +export function parseM059S02Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +function buildAppendixCoverageCheck(parsedRegistry: ParsedRegistry, parsedAppendix: ParsedAppendix): Check { + if (!parsedRegistry.ok) { + return failCheck("M059-S02-APPENDIX-COVERAGE", parsedRegistry.status_code, parsedRegistry.detail); + } + + if (!parsedAppendix.ok) { + return failCheck("M059-S02-APPENDIX-COVERAGE", parsedAppendix.status_code, parsedAppendix.detail); + } + + const orphanRows = parsedRegistry.rows.filter(isUsageNoneRow); + const appendixPaths = new Set(parsedAppendix.rows.map((row) => row.path)); + const missingPaths = orphanRows.map((row) => row.path).filter((filePath) => !appendixPaths.has(filePath)); + + if (missingPaths.length > 0) { + return failCheck( + "M059-S02-APPENDIX-COVERAGE", + "orphan_audit_missing_row", + `Missing orphan-audit entries for usage none rows: ${missingPaths.join(", ")}`, + ); + } + + return passCheck( + "M059-S02-APPENDIX-COVERAGE", + "appendix_coverage_ok", + `Orphan audit covers ${orphanRows.length} usage none registry rows.`, + ); +} + +function buildRetainedTruthCheck( + parsedAppendix: ParsedAppendix, + trackedFilesResult: Awaited>, +): Check { + if (!trackedFilesResult.ok) { + return failCheck("M059-S02-RETAINED-TRUTH", "tracked_files_unreadable", trackedFilesResult.error); + } + + if (!parsedAppendix.ok) { + return failCheck("M059-S02-RETAINED-TRUTH", parsedAppendix.status_code, parsedAppendix.detail); + } + + const trackedFiles = new Set(trackedFilesResult.files); + const missingRetained = parsedAppendix.rows + .filter((row) => row.disposition === "retained") + .map((row) => row.path) + .filter((filePath) => !trackedFiles.has(filePath)); + + if (missingRetained.length > 0) { + return failCheck( + "M059-S02-RETAINED-TRUTH", + "orphan_retained_missing_file", + `Retained orphan-audit entries must point to live tracked files: ${missingRetained.join(", ")}`, + ); + } + + return passCheck( + "M059-S02-RETAINED-TRUTH", + "retained_truth_ok", + `Validated ${parsedAppendix.rows.filter((row) => row.disposition === "retained").length} retained orphan entries.`, + ); +} + +function buildRemovalTruthCheck( + parsedRegistry: ParsedRegistry, + parsedAppendix: ParsedAppendix, + trackedFilesResult: Awaited>, +): Check { + if (!trackedFilesResult.ok) { + return failCheck("M059-S02-REMOVAL-TRUTH", "tracked_files_unreadable", trackedFilesResult.error); + } + + if (!parsedRegistry.ok) { + return failCheck("M059-S02-REMOVAL-TRUTH", parsedRegistry.status_code, parsedRegistry.detail); + } + + if (!parsedAppendix.ok) { + return failCheck("M059-S02-REMOVAL-TRUTH", parsedAppendix.status_code, parsedAppendix.detail); + } + + const trackedFiles = new Set(trackedFilesResult.files); + const removedStillTracked = parsedAppendix.rows + .filter((row) => row.disposition === "removed") + .map((row) => row.path) + .filter((filePath) => trackedFiles.has(filePath)); + + if (removedStillTracked.length > 0) { + return failCheck( + "M059-S02-REMOVAL-TRUTH", + "orphan_removed_file_still_exists", + `Removed orphan-audit entries must not point to live tracked files: ${removedStillTracked.join(", ")}`, + ); + } + + const removedRegistryRows = new Set( + parsedAppendix.rows + .filter((row) => row.disposition === "removed") + .map((row) => row.path), + ); + const staleRows = parsedRegistry.rows + .filter((row) => removedRegistryRows.has(row.path)) + .map((row) => row.path); + + if (staleRows.length > 0) { + return failCheck( + "M059-S02-REMOVAL-TRUTH", + "orphan_removal_stale_registry_row", + `Removed orphan-audit entries must not remain in the main registry table: ${staleRows.join(", ")}`, + ); + } + + return passCheck( + "M059-S02-REMOVAL-TRUTH", + "removal_truth_ok", + `Validated ${parsedAppendix.rows.filter((row) => row.disposition === "removed").length} removed orphan entries.`, + ); +} + +function buildPackageWiringCheck(parsedPackageJson: ReturnType): Check { + if (!parsedPackageJson.ok) { + return failCheck("M059-S02-PACKAGE-WIRING", "package_json_invalid", parsedPackageJson.error); + } + + const scripts = normalizeScriptsMap(parsedPackageJson.value); + if (!scripts.ok) { + return failCheck("M059-S02-PACKAGE-WIRING", "package_json_invalid", scripts.error); + } + + const actualScript = scripts.value[COMMAND_NAME]; + if (actualScript == null) { + return failCheck( + "M059-S02-PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT}`, + ); + } + + if (actualScript !== EXPECTED_PACKAGE_SCRIPT) { + return failCheck( + "M059-S02-PACKAGE-WIRING", + "package_wiring_incorrect", + `Expected scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT} but found ${actualScript}`, + ); + } + + return passCheck( + "M059-S02-PACKAGE-WIRING", + "package_wiring_ok", + `package.json wires ${COMMAND_NAME} to ${EXPECTED_PACKAGE_SCRIPT}`, + ); +} + +function parseRegistryReadResult(result: ReadResult): ParsedRegistry { + if (!result.ok) { + const detail = normalizeDetail(result.error); + if (/ENOENT/i.test(detail)) { + return { ok: false, status_code: "registry_missing", detail, scopeDeclarationPresent: false }; + } + return { ok: false, status_code: "registry_file_unreadable", detail, scopeDeclarationPresent: false }; + } + + return parseRegistryContent(result.content); +} + +function parseAppendixReadResult(result: ReadResult): ParsedAppendix { + if (!result.ok) { + const detail = normalizeDetail(result.error); + if (/ENOENT/i.test(detail)) { + return { ok: false, status_code: "registry_missing", detail, rows: [] }; + } + return { ok: false, status_code: "registry_file_unreadable", detail, rows: [] }; + } + + return parseAppendixContent(result.content); +} + +function parseAppendixContent(content: string): ParsedAppendix { + const lines = content.split(/\r?\n/); + const headingIndex = lines.findIndex((line) => line.trim() === APPENDIX_HEADING); + if (headingIndex < 0) { + return { + ok: false, + status_code: "orphan_audit_missing", + detail: `${APPENDIX_HEADING} heading is required next to the main registry table.`, + rows: [], + }; + } + + let headerIndex = headingIndex + 1; + while (headerIndex < lines.length && !(lines[headerIndex] ?? "").trim()) { + headerIndex += 1; + } + const dividerIndex = headerIndex + 1; + const headerCells = splitTableRow(lines[headerIndex] ?? ""); + if (headerCells.length !== APPENDIX_HEADER.length || !APPENDIX_HEADER.every((cell, index) => headerCells[index] === cell)) { + return { + ok: false, + status_code: "orphan_audit_malformed", + detail: `Orphan audit header must be exactly: ${APPENDIX_HEADER.join(", ")}`, + rows: [], + }; + } + + if (!/^\|\s*---/.test((lines[dividerIndex] ?? "").trim())) { + return { + ok: false, + status_code: "orphan_audit_malformed", + detail: "Orphan audit table must include a divider row.", + rows: [], + }; + } + + const rows: AppendixRow[] = []; + for (let index = dividerIndex + 1; index < lines.length; index += 1) { + const line = lines[index] ?? ""; + const trimmed = line.trim(); + if (!trimmed) { + break; + } + if (!trimmed.startsWith("|")) { + break; + } + + const cells = splitTableRow(line); + if (cells.length !== APPENDIX_HEADER.length) { + return { + ok: false, + status_code: "orphan_audit_malformed", + detail: `Line ${index + 1} must contain ${APPENDIX_HEADER.length} orphan-audit cells; found ${cells.length}.`, + rows, + }; + } + + const [filePath, dispositionRaw, rationale] = cells; + if (!filePath || !dispositionRaw || !rationale) { + return { + ok: false, + status_code: "orphan_audit_malformed", + detail: `Line ${index + 1} must provide non-empty path, disposition, and rationale values.`, + rows, + }; + } + + if (dispositionRaw !== "retained" && dispositionRaw !== "removed") { + return { + ok: false, + status_code: "orphan_audit_malformed", + detail: `Line ${index + 1} uses unknown disposition ${dispositionRaw}; allowed values: retained, removed.`, + rows, + }; + } + + rows.push({ + path: filePath, + disposition: dispositionRaw, + rationale, + sourceLine: index + 1, + }); + } + + return { ok: true, rows }; +} + +function isUsageNoneRow(row: RegistryRow): boolean { + return row.usageRefs.length === 1 && row.usageRefs[0]?.kind === "none"; +} + +function parsePackageJsonResult(result: ReadResult): + | { ok: true; value: PackageJsonShape } + | { ok: false; error: unknown } { + if (!result.ok) { + return { ok: false, error: result.error }; + } + + try { + return { ok: true, value: JSON.parse(result.content) as PackageJsonShape }; + } catch (error) { + return { ok: false, error }; + } +} + +function normalizeScriptsMap(packageJson: PackageJsonShape): + | { ok: true; value: Record } + | { ok: false; error: unknown } { + if (typeof packageJson.scripts !== "object" || packageJson.scripts == null || Array.isArray(packageJson.scripts)) { + return { ok: true, value: {} }; + } + + const scripts: Record = {}; + for (const [name, command] of Object.entries(packageJson.scripts)) { + if (typeof command !== "string") { + return { ok: false, error: `package.json scripts.${name} must be a string command.` }; + } + scripts[name] = command; + } + + return { ok: true, value: scripts }; +} + +async function readOptionalTextFile( + readTextFile: (filePath: string) => Promise, + filePath: string, +): Promise { + try { + return { ok: true, content: await readTextFile(filePath) }; + } catch (error) { + return { ok: false, error }; + } +} + +async function readTrackedScriptFiles( + listTrackedScriptFiles: () => Promise, +): Promise<{ ok: true; files: string[] } | { ok: false; error: unknown }> { + try { + const files = (await listTrackedScriptFiles()) + .filter((filePath) => filePath.startsWith("scripts/")) + .filter((filePath) => filePath.endsWith(".ts") || filePath.endsWith(".sh")) + .sort(); + return { ok: true, files }; + } catch (error) { + return { ok: false, error }; + } +} + +function splitTableRow(line: string): string[] { + return line + .trim() + .replace(/^\|/, "") + .replace(/\|$/, "") + .split("|") + .map((cell) => cell.trim()); +} + +function passCheck(id: M059S02CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M059S02CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +async function defaultListTrackedScriptFiles(): Promise { + const entries = await Array.fromAsync(new Bun.Glob("scripts/**/*").scan({ + cwd: REPO_ROOT, + onlyFiles: true, + absolute: false, + })); + + return entries.sort(); +} + +if (import.meta.main) { + try { + const args = parseM059S02Args(process.argv.slice(2)); + const { exitCode } = await buildM059S02ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${COMMAND_NAME} failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m060-s01.test.ts b/scripts/verify-m060-s01.test.ts new file mode 100644 index 00000000..4b5ad491 --- /dev/null +++ b/scripts/verify-m060-s01.test.ts @@ -0,0 +1,356 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport } from "./verify-m060-s01.ts"; +import { + M060_S01_CHECK_IDS, + buildM060S01ProofHarness, + evaluateM060S01CoverageContract, + parseM060S01Args, + renderM060S01Report, +} from "./verify-m060-s01.ts"; + +const EXPECTED_CHECK_IDS = [ + "M060-S01-PACKAGE-WIRING", + "M060-S01-REGISTRY-WIRING", + "M060-S01-RUNTIME-TARGET-MANIFEST", + "M060-S01-DIRECT-TEST-COVERAGE", + "M060-S01-TYPE-ONLY-EXEMPTIONS", +] as const; + +const PASSING_RUNTIME_TARGETS = [ + "src/knowledge/isolation.ts", + "src/knowledge/wiki-fetch.ts", + "src/knowledge/issue-retrieval.ts", + "src/knowledge/wiki-popularity-config.ts", + "src/knowledge/wiki-linkshere-fetcher.ts", + "src/knowledge/wiki-popularity-scorer.ts", + "src/knowledge/cluster-scheduler.ts", +] as const; + +const PASSING_TYPE_ONLY_EXEMPTIONS = [ + "src/knowledge/canonical-code-types.ts", + "src/knowledge/cluster-types.ts", + "src/knowledge/code-snippet-types.ts", + "src/knowledge/issue-types.ts", + "src/knowledge/review-comment-types.ts", + "src/knowledge/types.ts", + "src/knowledge/wiki-publisher-types.ts", + "src/knowledge/wiki-staleness-types.ts", + "src/knowledge/wiki-types.ts", + "src/knowledge/wiki-update-types.ts", + "src/knowledge/wiki-voice-types.ts", +] as const; + +const PASSING_PACKAGE_JSON = JSON.stringify( + { + name: "kodiai", + scripts: { + "verify:m060:s01": "bun scripts/verify-m060-s01.ts", + }, + }, + null, + 2, +); + +const PASSING_REGISTRY = `# Script Registry + +| path | purpose | owner | lifecycle | usage | +| --- | --- | --- | --- | --- | +| scripts/verify-m060-s01.test.ts | Regression tests for the M060 S01 direct-test coverage verifier. | M060 | internal | none | +| scripts/verify-m060-s01.ts | Verification CLI for the M060 S01 knowledge direct-test coverage contract. | M060 | active | package:verify:m060:s01 | +`; + +function buildTrackedFiles(opts: { + omitTestsFor?: string[]; + includeTestsForExemptions?: string[]; +} = {}): string[] { + const tracked = [ + "package.json", + "scripts/REGISTRY.md", + ...PASSING_RUNTIME_TARGETS, + ...PASSING_RUNTIME_TARGETS.map((target) => target.replace(/\.ts$/u, ".test.ts")), + ...PASSING_TYPE_ONLY_EXEMPTIONS, + ]; + + for (const target of opts.omitTestsFor ?? []) { + const testPath = target.replace(/\.ts$/u, ".test.ts"); + const index = tracked.indexOf(testPath); + if (index !== -1) tracked.splice(index, 1); + } + + for (const target of opts.includeTestsForExemptions ?? []) { + tracked.push(target.replace(/\.ts$/u, ".test.ts")); + } + + return tracked.sort(); +} + +function buildSourceMap(overrides: Record = {}): Record { + const sources: Record = { + "scripts/REGISTRY.md": PASSING_REGISTRY, + "package.json": PASSING_PACKAGE_JSON, + }; + + for (const filePath of PASSING_TYPE_ONLY_EXEMPTIONS) { + sources[filePath] = `export type ${filePath.replace(/[^a-z]/giu, "_")} = { ok: true };\n`; + } + + return { ...sources, ...overrides }; +} + +function buildFileExists(trackedFiles: string[]) { + const fileSet = new Set(trackedFiles); + return async (filePath: string): Promise => fileSet.has(filePath.replace(/\\/g, "/")); +} + +describe("verify m060 s01 coverage contract", () => { + test("exports stable check ids and cli parsing", () => { + expect(M060_S01_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM060S01Args([])).toEqual({ json: false }); + expect(parseM060S01Args(["--json"])).toEqual({ json: true }); + expect(() => parseM060S01Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes for the wired direct-test coverage contract", async () => { + const trackedFiles = buildTrackedFiles(); + const report = await evaluateM060S01CoverageContract({ + generatedAt: "2026-04-21T17:00:00.000Z", + readTextFile: async (filePath: string) => { + const normalized = filePath.replace(/\\/g, "/"); + const mapped = buildSourceMap()[normalized.replace(/^.*?(package\.json|scripts\/REGISTRY\.md|src\/knowledge\/.*)$/u, "$1")]; + if (mapped != null) return mapped; + throw new Error(`Unexpected path: ${filePath}`); + }, + listTrackedFiles: async () => trackedFiles, + fileExists: buildFileExists(trackedFiles), + loadManifest: async () => ({ + runtimeTargets: PASSING_RUNTIME_TARGETS, + typeOnlyExemptions: PASSING_TYPE_ONLY_EXEMPTIONS, + }), + }); + + expect(report.command).toBe("verify:m060:s01"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ id: "M060-S01-PACKAGE-WIRING", passed: true, status_code: "package_wiring_ok" }), + expect.objectContaining({ id: "M060-S01-REGISTRY-WIRING", passed: true, status_code: "registry_rows_ok" }), + expect.objectContaining({ id: "M060-S01-RUNTIME-TARGET-MANIFEST", passed: true, status_code: "runtime_targets_ok" }), + expect.objectContaining({ id: "M060-S01-DIRECT-TEST-COVERAGE", passed: true, status_code: "direct_tests_ok" }), + expect.objectContaining({ id: "M060-S01-TYPE-ONLY-EXEMPTIONS", passed: true, status_code: "type_only_exemptions_ok" }), + ]); + + const rendered = renderM060S01Report(report); + expect(rendered).toContain("Coverage contract: PASS"); + expect(rendered).toContain("M060-S01-DIRECT-TEST-COVERAGE PASS"); + expect(rendered).toContain("M060-S01-TYPE-ONLY-EXEMPTIONS PASS"); + }); + + test("fails with stable status codes for missing direct tests, bad wiring, and runtime-bearing exemptions", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + const trackedFiles = buildTrackedFiles({ omitTestsFor: ["src/knowledge/cluster-scheduler.ts"] }); + + const result = await buildM060S01ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + readTextFile: async (filePath: string) => { + const normalized = filePath.replace(/\\/g, "/"); + const relative = normalized.replace(/^.*?(package\.json|scripts\/REGISTRY\.md|src\/knowledge\/.*)$/u, "$1"); + const map = buildSourceMap({ + "package.json": JSON.stringify({ scripts: {} }), + "scripts/REGISTRY.md": "# Script Registry\n", + "src/knowledge/issue-types.ts": "export const notTypeOnly = 1;\n", + }); + const value = map[relative]; + if (value != null) return value; + throw new Error(`Unexpected path: ${filePath}`); + }, + listTrackedFiles: async () => trackedFiles, + fileExists: buildFileExists(trackedFiles), + loadManifest: async () => ({ + runtimeTargets: PASSING_RUNTIME_TARGETS, + typeOnlyExemptions: PASSING_TYPE_ONLY_EXEMPTIONS, + }), + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ id: "M060-S01-PACKAGE-WIRING", passed: false, status_code: "package_wiring_missing" }), + expect.objectContaining({ id: "M060-S01-REGISTRY-WIRING", passed: false, status_code: "registry_rows_missing" }), + expect.objectContaining({ id: "M060-S01-RUNTIME-TARGET-MANIFEST", passed: true, status_code: "runtime_targets_ok" }), + expect.objectContaining({ id: "M060-S01-DIRECT-TEST-COVERAGE", passed: false, status_code: "direct_tests_missing" }), + expect.objectContaining({ id: "M060-S01-TYPE-ONLY-EXEMPTIONS", passed: false, status_code: "type_only_exemption_invalid" }), + ]); + expect(report.checks[3]?.detail).toContain("cluster-scheduler.ts -> src/knowledge/cluster-scheduler.test.ts"); + expect(report.checks[4]?.detail).toContain("src/knowledge/issue-types.ts: runtime exports detected"); + expect(stderr.join(" ")).toContain("package_wiring_missing"); + expect(stderr.join(" ")).toContain("registry_rows_missing"); + expect(stderr.join(" ")).toContain("direct_tests_missing"); + expect(stderr.join(" ")).toContain("type_only_exemption_invalid"); + }); + + test("surfaces malformed manifest inputs and boundary conditions with stable status codes", async () => { + const trackedFiles = buildTrackedFiles(); + const duplicateManifest = await evaluateM060S01CoverageContract({ + readTextFile: async (filePath: string) => { + const normalized = filePath.replace(/\\/g, "/"); + const relative = normalized.replace(/^.*?(package\.json|scripts\/REGISTRY\.md|src\/knowledge\/.*)$/u, "$1"); + const value = buildSourceMap()[relative]; + if (value != null) return value; + throw new Error(`Unexpected path: ${filePath}`); + }, + listTrackedFiles: async () => trackedFiles, + fileExists: buildFileExists(trackedFiles), + loadManifest: async () => ({ + runtimeTargets: [...PASSING_RUNTIME_TARGETS, "src/knowledge/isolation.ts"], + typeOnlyExemptions: PASSING_TYPE_ONLY_EXEMPTIONS, + }), + }); + + expect(duplicateManifest.checks[2]).toEqual( + expect.objectContaining({ + id: "M060-S01-RUNTIME-TARGET-MANIFEST", + passed: false, + status_code: "runtime_target_duplicate_entries", + }), + ); + + const outsideScopeManifest = await evaluateM060S01CoverageContract({ + readTextFile: async (filePath: string) => { + const normalized = filePath.replace(/\\/g, "/"); + const relative = normalized.replace(/^.*?(package\.json|scripts\/REGISTRY\.md|src\/knowledge\/.*)$/u, "$1"); + const value = buildSourceMap()[relative]; + if (value != null) return value; + throw new Error(`Unexpected path: ${filePath}`); + }, + listTrackedFiles: async () => trackedFiles, + fileExists: buildFileExists(trackedFiles), + loadManifest: async () => ({ + runtimeTargets: PASSING_RUNTIME_TARGETS, + typeOnlyExemptions: ["scripts/verify-m060-s01.ts"], + }), + }); + + expect(outsideScopeManifest.checks[4]).toEqual( + expect.objectContaining({ + id: "M060-S01-TYPE-ONLY-EXEMPTIONS", + passed: false, + status_code: "type_only_exemption_outside_scope", + }), + ); + + const emptyLists = await evaluateM060S01CoverageContract({ + readTextFile: async (filePath: string) => { + const normalized = filePath.replace(/\\/g, "/"); + const relative = normalized.replace(/^.*?(package\.json|scripts\/REGISTRY\.md|src\/knowledge\/.*)$/u, "$1"); + const value = buildSourceMap()[relative]; + if (value != null) return value; + throw new Error(`Unexpected path: ${filePath}`); + }, + listTrackedFiles: async () => trackedFiles, + fileExists: buildFileExists(trackedFiles), + loadManifest: async () => ({ runtimeTargets: [], typeOnlyExemptions: [] }), + }); + + expect(emptyLists.checks[2]).toEqual( + expect.objectContaining({ + id: "M060-S01-RUNTIME-TARGET-MANIFEST", + passed: false, + status_code: "runtime_targets_incomplete", + }), + ); + expect(emptyLists.checks[4]).toEqual( + expect.objectContaining({ + id: "M060-S01-TYPE-ONLY-EXEMPTIONS", + passed: false, + status_code: "type_only_exemptions_incomplete", + }), + ); + }); + + test("flags malformed package json, unreadable tracked-file state, and exemption files that gain direct tests", async () => { + const trackedFiles = buildTrackedFiles(); + const malformedPackage = await evaluateM060S01CoverageContract({ + readTextFile: async (filePath: string) => { + const normalized = filePath.replace(/\\/g, "/"); + const relative = normalized.replace(/^.*?(package\.json|scripts\/REGISTRY\.md|src\/knowledge\/.*)$/u, "$1"); + const map = buildSourceMap({ "package.json": "{ not valid json" }); + const value = map[relative]; + if (value != null) return value; + throw new Error(`Unexpected path: ${filePath}`); + }, + listTrackedFiles: async () => trackedFiles, + fileExists: buildFileExists(trackedFiles), + loadManifest: async () => ({ + runtimeTargets: PASSING_RUNTIME_TARGETS, + typeOnlyExemptions: PASSING_TYPE_ONLY_EXEMPTIONS, + }), + }); + + expect(malformedPackage.checks[0]).toEqual( + expect.objectContaining({ id: "M060-S01-PACKAGE-WIRING", passed: false, status_code: "package_json_invalid" }), + ); + + const unreadableTracked = await evaluateM060S01CoverageContract({ + readTextFile: async (filePath: string) => { + const normalized = filePath.replace(/\\/g, "/"); + const relative = normalized.replace(/^.*?(package\.json|scripts\/REGISTRY\.md|src\/knowledge\/.*)$/u, "$1"); + const value = buildSourceMap()[relative]; + if (value != null) return value; + throw new Error(`Unexpected path: ${filePath}`); + }, + listTrackedFiles: async () => { + throw new Error("git ls-files failed"); + }, + fileExists: async () => false, + loadManifest: async () => ({ + runtimeTargets: PASSING_RUNTIME_TARGETS, + typeOnlyExemptions: PASSING_TYPE_ONLY_EXEMPTIONS, + }), + }); + + expect(unreadableTracked.checks[2]).toEqual( + expect.objectContaining({ id: "M060-S01-RUNTIME-TARGET-MANIFEST", passed: false, status_code: "tracked_files_unreadable" }), + ); + expect(unreadableTracked.checks[3]).toEqual( + expect.objectContaining({ id: "M060-S01-DIRECT-TEST-COVERAGE", passed: false, status_code: "direct_tests_missing" }), + ); + + const exemptionTrackedFiles = buildTrackedFiles({ includeTestsForExemptions: ["src/knowledge/types.ts"] }); + const exemptionHasTest = await evaluateM060S01CoverageContract({ + readTextFile: async (filePath: string) => { + const normalized = filePath.replace(/\\/g, "/"); + const relative = normalized.replace(/^.*?(package\.json|scripts\/REGISTRY\.md|src\/knowledge\/.*)$/u, "$1"); + const value = buildSourceMap()[relative]; + if (value != null) return value; + throw new Error(`Unexpected path: ${filePath}`); + }, + listTrackedFiles: async () => exemptionTrackedFiles, + fileExists: buildFileExists(exemptionTrackedFiles), + loadManifest: async () => ({ + runtimeTargets: PASSING_RUNTIME_TARGETS, + typeOnlyExemptions: PASSING_TYPE_ONLY_EXEMPTIONS, + }), + }); + + expect(exemptionHasTest.checks[4]).toEqual( + expect.objectContaining({ + id: "M060-S01-TYPE-ONLY-EXEMPTIONS", + passed: false, + status_code: "type_only_exemption_has_direct_test", + }), + ); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m060:s01"]).toBe("bun scripts/verify-m060-s01.ts"); + }); +}); diff --git a/scripts/verify-m060-s01.ts b/scripts/verify-m060-s01.ts new file mode 100644 index 00000000..b9146515 --- /dev/null +++ b/scripts/verify-m060-s01.ts @@ -0,0 +1,614 @@ +import { access, readFile } from "node:fs/promises"; +import path from "node:path"; +import { + M060_S01_RUNTIME_TARGETS, + M060_S01_TYPE_ONLY_EXEMPTIONS, +} from "../src/knowledge/test-coverage-exemptions.ts"; + +const COMMAND_NAME = "verify:m060:s01" as const; +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/verify-m060-s01.ts"; +const REGISTRY_PATH = path.resolve(import.meta.dir, "REGISTRY.md"); +const PACKAGE_JSON_PATH = path.resolve(import.meta.dir, "../package.json"); +const REQUIRED_RUNTIME_TARGETS = [...M060_S01_RUNTIME_TARGETS].sort(); +const REQUIRED_TYPE_ONLY_EXEMPTIONS = [...M060_S01_TYPE_ONLY_EXEMPTIONS].sort(); +const EXPECTED_REGISTRY_ROWS = [ + "| scripts/verify-m060-s01.test.ts | Regression tests for the M060 S01 direct-test coverage verifier. | M060 | internal | none |", + "| scripts/verify-m060-s01.ts | Verification CLI for the M060 S01 knowledge direct-test coverage contract. | M060 | active | package:verify:m060:s01 |", +] as const; + +export const M060_S01_CHECK_IDS = [ + "M060-S01-PACKAGE-WIRING", + "M060-S01-REGISTRY-WIRING", + "M060-S01-RUNTIME-TARGET-MANIFEST", + "M060-S01-DIRECT-TEST-COVERAGE", + "M060-S01-TYPE-ONLY-EXEMPTIONS", +] as const; + +export type M060S01CheckId = (typeof M060_S01_CHECK_IDS)[number]; + +export type Check = { + id: M060S01CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M060S01CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type ManifestShape = { + runtimeTargets?: readonly string[]; + typeOnlyExemptions?: readonly string[]; +}; + +type EvaluateOptions = { + generatedAt?: string; + readTextFile?: (filePath: string) => Promise; + listTrackedFiles?: () => Promise; + loadManifest?: () => Promise; + fileExists?: (filePath: string) => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +export async function evaluateM060S01CoverageContract( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const readTextFile = options.readTextFile ?? defaultReadTextFile; + const listTrackedFiles = options.listTrackedFiles ?? defaultListTrackedFiles; + const loadManifest = options.loadManifest ?? defaultLoadManifest; + const fileExists = options.fileExists ?? defaultFileExists; + + const trackedFilesState = await readTrackedFileState(listTrackedFiles); + const manifestState = await readManifestState(loadManifest); + + const packageCheck = await buildPackageCheck(readTextFile); + const registryCheck = await buildRegistryCheck(readTextFile); + const runtimeManifestCheck = buildRuntimeTargetManifestCheck(manifestState, trackedFilesState); + const directTestCoverageCheck = await buildDirectTestCoverageCheck(manifestState, fileExists); + const typeOnlyExemptionsCheck = await buildTypeOnlyExemptionsCheck({ + manifestState, + trackedFilesState, + readTextFile, + fileExists, + }); + + const checks = [ + packageCheck, + registryCheck, + runtimeManifestCheck, + directTestCoverageCheck, + typeOnlyExemptionsCheck, + ]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M060_S01_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM060S01Report(report: EvaluationReport): string { + const lines = [ + "M060 S01 direct-test coverage verifier", + `Generated at: ${report.generatedAt}`, + `Coverage contract: ${report.overallPassed ? "PASS" : "FAIL"}`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM060S01ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM060S01CoverageContract(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM060S01Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`${COMMAND_NAME} failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM060S01Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +type FileSetResult = + | { ok: true; entries: string[] } + | { ok: false; error: unknown }; + +type ManifestState = + | { ok: true; runtimeTargets: string[]; typeOnlyExemptions: string[] } + | { ok: false; error: unknown }; + +async function buildPackageCheck( + readTextFile: (filePath: string) => Promise, +): Promise { + let packageText: string; + try { + packageText = await readTextFile(PACKAGE_JSON_PATH); + } catch (error) { + return failCheck("M060-S01-PACKAGE-WIRING", "package_file_unreadable", error); + } + + let parsed: { scripts?: Record }; + try { + parsed = JSON.parse(packageText) as { scripts?: Record }; + } catch (error) { + return failCheck("M060-S01-PACKAGE-WIRING", "package_json_invalid", error); + } + + const actualScript = parsed.scripts?.[COMMAND_NAME]; + if (actualScript == null) { + return failCheck( + "M060-S01-PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT}`, + ); + } + + if (actualScript !== EXPECTED_PACKAGE_SCRIPT) { + return failCheck( + "M060-S01-PACKAGE-WIRING", + "package_wiring_incorrect", + `Expected scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT} but found ${actualScript}`, + ); + } + + return passCheck( + "M060-S01-PACKAGE-WIRING", + "package_wiring_ok", + `package.json wires ${COMMAND_NAME} to ${EXPECTED_PACKAGE_SCRIPT}`, + ); +} + +async function buildRegistryCheck( + readTextFile: (filePath: string) => Promise, +): Promise { + let registryText: string; + try { + registryText = await readTextFile(REGISTRY_PATH); + } catch (error) { + return failCheck("M060-S01-REGISTRY-WIRING", "registry_file_unreadable", error); + } + + const missingRows = EXPECTED_REGISTRY_ROWS.filter((row) => !registryText.includes(row)); + if (missingRows.length > 0) { + return failCheck( + "M060-S01-REGISTRY-WIRING", + "registry_rows_missing", + `scripts/REGISTRY.md must include canonical rows for ${missingRows.map(extractRegistryPathFromRow).join(", ")}`, + ); + } + + return passCheck( + "M060-S01-REGISTRY-WIRING", + "registry_rows_ok", + `scripts/REGISTRY.md registers ${EXPECTED_REGISTRY_ROWS.map(extractRegistryPathFromRow).join(" and ")}.`, + ); +} + +function buildRuntimeTargetManifestCheck( + manifestState: ManifestState, + trackedFilesState: FileSetResult, +): Check { + if (!manifestState.ok) { + return failCheck("M060-S01-RUNTIME-TARGET-MANIFEST", "manifest_unreadable", manifestState.error); + } + + const validationError = validatePathList(manifestState.runtimeTargets, "runtime target"); + if (validationError) { + return failCheck("M060-S01-RUNTIME-TARGET-MANIFEST", validationError.code, validationError.detail); + } + + const missingRequired = REQUIRED_RUNTIME_TARGETS.filter( + (target) => !manifestState.runtimeTargets.includes(target), + ); + if (missingRequired.length > 0) { + return failCheck( + "M060-S01-RUNTIME-TARGET-MANIFEST", + "runtime_targets_incomplete", + `Manifest is missing required runtime targets: ${missingRequired.join(", ")}`, + ); + } + + const unexpectedTargets = manifestState.runtimeTargets.filter( + (target) => !REQUIRED_RUNTIME_TARGETS.includes(target), + ); + if (unexpectedTargets.length > 0) { + return failCheck( + "M060-S01-RUNTIME-TARGET-MANIFEST", + "runtime_targets_unexpected", + `Manifest includes unexpected S01 runtime targets: ${unexpectedTargets.join(", ")}`, + ); + } + + if (!trackedFilesState.ok) { + return failCheck( + "M060-S01-RUNTIME-TARGET-MANIFEST", + "tracked_files_unreadable", + trackedFilesState.error, + ); + } + + const trackedFileSet = new Set(trackedFilesState.entries); + const missingTracked = manifestState.runtimeTargets.filter((target) => !trackedFileSet.has(target)); + if (missingTracked.length > 0) { + return failCheck( + "M060-S01-RUNTIME-TARGET-MANIFEST", + "runtime_target_untracked", + `Manifest runtime targets must be tracked files: ${missingTracked.join(", ")}`, + ); + } + + return passCheck( + "M060-S01-RUNTIME-TARGET-MANIFEST", + "runtime_targets_ok", + `Manifest lists ${manifestState.runtimeTargets.length} required S01 runtime targets.`, + ); +} + +async function buildDirectTestCoverageCheck( + manifestState: ManifestState, + fileExists: (filePath: string) => Promise, +): Promise { + if (!manifestState.ok) { + return failCheck("M060-S01-DIRECT-TEST-COVERAGE", "manifest_unreadable", manifestState.error); + } + + const missingDirectTests: string[] = []; + + for (const target of manifestState.runtimeTargets) { + const testPath = target.replace(/\.ts$/u, ".test.ts"); + if (!(await fileExists(testPath))) { + missingDirectTests.push(`${target} -> ${testPath}`); + } + } + + if (missingDirectTests.length > 0) { + return failCheck( + "M060-S01-DIRECT-TEST-COVERAGE", + "direct_tests_missing", + `Runtime targets missing direct same-name tests: ${missingDirectTests.join(", ")}`, + ); + } + + return passCheck( + "M060-S01-DIRECT-TEST-COVERAGE", + "direct_tests_ok", + `All ${manifestState.runtimeTargets.length} runtime targets have direct same-name tests.`, + ); +} + +async function buildTypeOnlyExemptionsCheck({ + manifestState, + trackedFilesState, + readTextFile, + fileExists, +}: { + manifestState: ManifestState; + trackedFilesState: FileSetResult; + readTextFile: (filePath: string) => Promise; + fileExists: (filePath: string) => Promise; +}): Promise { + if (!manifestState.ok) { + return failCheck("M060-S01-TYPE-ONLY-EXEMPTIONS", "manifest_unreadable", manifestState.error); + } + + const validationError = validatePathList(manifestState.typeOnlyExemptions, "type-only exemption"); + if (validationError) { + return failCheck("M060-S01-TYPE-ONLY-EXEMPTIONS", validationError.code, validationError.detail); + } + + const missingRequired = REQUIRED_TYPE_ONLY_EXEMPTIONS.filter( + (exemption) => !manifestState.typeOnlyExemptions.includes(exemption), + ); + if (missingRequired.length > 0) { + return failCheck( + "M060-S01-TYPE-ONLY-EXEMPTIONS", + "type_only_exemptions_incomplete", + `Manifest is missing required type-only exemptions: ${missingRequired.join(", ")}`, + ); + } + + const unexpectedExemptions = manifestState.typeOnlyExemptions.filter( + (exemption) => !REQUIRED_TYPE_ONLY_EXEMPTIONS.includes(exemption), + ); + if (unexpectedExemptions.length > 0) { + return failCheck( + "M060-S01-TYPE-ONLY-EXEMPTIONS", + "type_only_exemptions_unexpected", + `Manifest includes unexpected type-only exemptions: ${unexpectedExemptions.join(", ")}`, + ); + } + + if (!trackedFilesState.ok) { + return failCheck( + "M060-S01-TYPE-ONLY-EXEMPTIONS", + "tracked_files_unreadable", + trackedFilesState.error, + ); + } + + const trackedFileSet = new Set(trackedFilesState.entries); + const missingTracked = manifestState.typeOnlyExemptions.filter((target) => !trackedFileSet.has(target)); + if (missingTracked.length > 0) { + return failCheck( + "M060-S01-TYPE-ONLY-EXEMPTIONS", + "type_only_exemption_untracked", + `Manifest exemptions must be tracked files: ${missingTracked.join(", ")}`, + ); + } + + const directTestConflicts: Array<{ target: string; testPath: string }> = []; + for (const target of manifestState.typeOnlyExemptions) { + const testPath = target.replace(/\.ts$/u, ".test.ts"); + if (await fileExists(testPath)) { + directTestConflicts.push({ target, testPath }); + } + } + if (directTestConflicts.length > 0) { + return failCheck( + "M060-S01-TYPE-ONLY-EXEMPTIONS", + "type_only_exemption_has_direct_test", + `Type-only exemptions should not point at files that already have direct tests: ${directTestConflicts.map(({ target, testPath }) => `${target} -> ${testPath}`).join(", ")}`, + ); + } + + const classificationFailures: string[] = []; + for (const exemption of manifestState.typeOnlyExemptions) { + let text: string; + try { + text = await readTextFile(path.resolve(import.meta.dir, "..", exemption)); + } catch (error) { + classificationFailures.push(`${exemption}: unreadable: ${normalizeDetail(error)}`); + continue; + } + + const classification = classifyExports(text); + if (classification.hasRuntime) { + classificationFailures.push(`${exemption}: runtime exports detected`); + continue; + } + if (!classification.hasTypeOnly) { + classificationFailures.push(`${exemption}: no explicit type-only exports detected`); + } + } + + if (classificationFailures.length > 0) { + return failCheck( + "M060-S01-TYPE-ONLY-EXEMPTIONS", + "type_only_exemption_invalid", + classificationFailures.join(" | "), + ); + } + + return passCheck( + "M060-S01-TYPE-ONLY-EXEMPTIONS", + "type_only_exemptions_ok", + `Validated ${manifestState.typeOnlyExemptions.length} explicit type-only exemptions.`, + ); +} + +function validatePathList( + items: string[], + label: string, +): { code: string; detail: string } | null { + const duplicates = findDuplicates(items); + if (duplicates.length > 0) { + return { + code: `${label.replace(/[^a-z0-9]+/giu, "_")}_duplicate_entries`, + detail: `Manifest contains duplicate ${label} entries: ${duplicates.join(", ")}`, + }; + } + + const outsideScope = items.filter((item) => !item.startsWith("src/knowledge/") || !item.endsWith(".ts")); + if (outsideScope.length > 0) { + return { + code: `${label.replace(/[^a-z0-9]+/giu, "_")}_outside_scope`, + detail: `${label} entries must stay under src/knowledge/*.ts: ${outsideScope.join(", ")}`, + }; + } + + return null; +} + +function classifyExports(sourceText: string): { hasRuntime: boolean; hasTypeOnly: boolean } { + const stripped = stripComments(sourceText); + + const hasRuntime = + /(^|\n)\s*export\s+(?:default\s+)?(?:async\s+)?function\b/u.test(stripped) || + /(^|\n)\s*export\s+(?:const|let|var|class|enum)\b/u.test(stripped) || + /(^|\n)\s*export\s*\{(?!\s*type\b)[^}]+\}/u.test(stripped) || + /(^|\n)\s*export\s*\*\s*from\s*["']/u.test(stripped) || + /(^|\n)\s*export\s+default\b/u.test(stripped); + + const hasTypeOnly = + /(^|\n)\s*export\s+type\b/u.test(stripped) || + /(^|\n)\s*export\s+interface\b/u.test(stripped) || + /(^|\n)\s*export\s*\{\s*type\b/u.test(stripped); + + return { hasRuntime, hasTypeOnly }; +} + +function stripComments(text: string): string { + return text + .replace(/\/\*[\s\S]*?\*\//gu, "") + .replace(/(^|[^:])\/\/.*$/gmu, "$1"); +} + +function extractRegistryPathFromRow(row: string): string { + return row.split("|")[1]?.trim() ?? row; +} + +function findDuplicates(items: readonly string[]): string[] { + const counts = new Map(); + for (const item of items) { + counts.set(item, (counts.get(item) ?? 0) + 1); + } + return [...counts.entries()] + .filter(([, count]) => count > 1) + .map(([item]) => item) + .sort(); +} + +function passCheck(id: M060S01CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M060S01CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +async function readTrackedFileState( + listTrackedFiles: () => Promise, +): Promise { + try { + return { ok: true, entries: (await listTrackedFiles()).sort() }; + } catch (error) { + return { ok: false, error }; + } +} + +async function readManifestState(loadManifest: () => Promise): Promise { + try { + const loaded = await loadManifest(); + return { + ok: true, + runtimeTargets: [...(loaded.runtimeTargets ?? [])].sort(), + typeOnlyExemptions: [...(loaded.typeOnlyExemptions ?? [])].sort(), + }; + } catch (error) { + return { ok: false, error }; + } +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +async function defaultListTrackedFiles(): Promise { + const proc = Bun.spawn(["git", "ls-files", "--", "src/knowledge", "scripts", "package.json"], { + cwd: path.resolve(import.meta.dir, ".."), + stdout: "pipe", + stderr: "pipe", + }); + + const [stdoutText, stderrText, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + + if (exitCode !== 0) { + throw new Error(stderrText.trim() || `git ls-files exited with code ${exitCode}`); + } + + return stdoutText + .split(/\r?\n/u) + .map((entry) => entry.trim()) + .filter(Boolean) + .sort(); +} + +async function defaultFileExists(filePath: string): Promise { + try { + await access(path.resolve(import.meta.dir, "..", filePath)); + return true; + } catch { + return false; + } +} + +async function defaultLoadManifest(): Promise { + return { + runtimeTargets: [...M060_S01_RUNTIME_TARGETS], + typeOnlyExemptions: [...M060_S01_TYPE_ONLY_EXEMPTIONS], + }; +} + +if (import.meta.main) { + try { + const args = parseM060S01Args(process.argv.slice(2)); + const { exitCode } = await buildM060S01ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${COMMAND_NAME} failed: ${message}\n`); + process.exit(1); + } +} diff --git a/scripts/verify-m060-s02.test.ts b/scripts/verify-m060-s02.test.ts new file mode 100644 index 00000000..c7d003b1 --- /dev/null +++ b/scripts/verify-m060-s02.test.ts @@ -0,0 +1,211 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { EvaluationReport } from "./verify-m060-s02.ts"; +import { + M060_S02_CHECK_IDS, + buildM060S02ProofHarness, + evaluateM060S02BoundaryContract, + parseM060S02Args, + renderM060S02Report, +} from "./verify-m060-s02.ts"; + +const EXPECTED_CHECK_IDS = [ + "M060-S02-PACKAGE-WIRING", + "M060-S02-REGISTRY-WIRING", + "M060-S02-DECISION-RENDER", + "M060-S02-SUMMARY-BOUNDARY", +] as const; + +const PASSING_PACKAGE_JSON = JSON.stringify( + { + name: "kodiai", + scripts: { + "verify:m060:s02": "bun scripts/verify-m060-s02.ts", + }, + }, + null, + 2, +); + +const PASSING_REGISTRY = `# Script Registry + +| path | purpose | owner | lifecycle | usage | +| --- | --- | --- | --- | --- | +| scripts/verify-m060-s02.test.ts | Regression tests for the M060 S02 ownership-boundary verifier. | M060 | internal | none | +| scripts/verify-m060-s02.ts | Verification CLI for the M060 S02 M060-vs-M027 ownership-boundary contract. | M060 | active | package:verify:m060:s02 | +`; + +const PASSING_DECISIONS = `# Decisions Register + +| # | When | Scope | Decision | Choice | Rationale | Revisable? | Made By | +|---|------|-------|----------|--------|-----------|------------|---------| +| D171 | M060/S02 planning | testing | How M060/S02 should define the ownership boundary between M060 and M027 knowledge verification | Define the boundary by proof class, not file exclusivity: M060 owns direct same-name unit tests plus explicit type-only exemptions for selected \`src/knowledge\` modules, while M027 owns persisted-corpus audit, repair/status, and live retriever acceptance proofs. S02 should document that split in tracked narrative artifacts and enforce it with a deterministic repo-local verifier instead of changing M027 runtime verifier behavior. | Several \`src/knowledge\` modules legitimately participate in both milestones for different reasons, so a file-only split would be false and brittle. A proof-class boundary matches existing verifier behavior, prevents duplicate or misleading claims, and gives future slices one machine-checkable contract for where unit proof ends and corpus-integrity proof begins. | Yes | agent | +`; + +const PASSING_SUMMARY = `# M027 Summary + +## M060/M027 ownership boundary + +- **M060/M027 ownership boundary** — canonical proof-class split for overlapping \`src/knowledge\` surfaces. +- **M060 owns direct same-name unit tests** for the runtime targets listed in \`src/knowledge/test-coverage-exemptions.ts\`, plus the explicit type-only exemptions in that same manifest. The canonical unit-test contract source is \`bun run verify:m060:s01\`. +- **M027 owns persisted-corpus audit, repair/status, and live retriever acceptance proof** across stored corpora. The canonical corpus-level proof family is \`bun run verify:m027:s01\`, \`bun run verify:m027:s02\`, \`bun run verify:m027:s03\`, and \`bun run verify:m027:s04\`. +- File overlap is allowed when the proof class differs. The same source file can participate in M060 unit-test ownership and M027 corpus-integrity acceptance proof without conflict. +- \`issue_comments\` remains truthful current scope: audited and repairable under M027, but outside the live retriever and therefore not evidence of retriever coverage. +- \`D171\` in \`.gsd/DECISIONS.md\` is the canonical architectural rationale for this boundary and should remain the verifier-readable decision surface. +`; + +function createReadTextFile(overrides: Record = {}) { + const map: Record = { + package: PASSING_PACKAGE_JSON, + registry: PASSING_REGISTRY, + decisions: PASSING_DECISIONS, + summary: PASSING_SUMMARY, + ...overrides, + }; + + return async (filePath: string): Promise => { + const normalized = filePath.replace(/\\/g, "/"); + if (normalized.endsWith("/package.json")) { + const value = map.package; + if (value instanceof Error) throw value; + return value; + } + if (normalized.endsWith("/scripts/REGISTRY.md")) { + const value = map.registry; + if (value instanceof Error) throw value; + return value; + } + if (normalized.endsWith("/.gsd/DECISIONS.md")) { + const value = map.decisions; + if (value instanceof Error) throw value; + return value; + } + if (normalized.endsWith("/.gsd/milestones/M027/M027-SUMMARY.md")) { + const value = map.summary; + if (value instanceof Error) throw value; + return value; + } + + throw new Error(`Unexpected path: ${filePath}`); + }; +} + +describe("verify m060 s02 ownership boundary contract", () => { + test("exports stable check ids and cli parsing", () => { + expect(M060_S02_CHECK_IDS).toEqual(EXPECTED_CHECK_IDS); + expect(parseM060S02Args([])).toEqual({ json: false }); + expect(parseM060S02Args(["--json"])).toEqual({ json: true }); + expect(() => parseM060S02Args(["--wat"])).toThrow(/invalid_cli_args/i); + }); + + test("passes for the wired ownership-boundary contract", async () => { + const report = await evaluateM060S02BoundaryContract({ + generatedAt: "2026-04-21T18:30:00.000Z", + readTextFile: createReadTextFile(), + }); + + expect(report.command).toBe("verify:m060:s02"); + expect(report.check_ids).toEqual(EXPECTED_CHECK_IDS); + expect(report.overallPassed).toBe(true); + expect(report.checks).toEqual([ + expect.objectContaining({ id: "M060-S02-PACKAGE-WIRING", passed: true, status_code: "package_wiring_ok" }), + expect.objectContaining({ id: "M060-S02-REGISTRY-WIRING", passed: true, status_code: "registry_rows_ok" }), + expect.objectContaining({ id: "M060-S02-DECISION-RENDER", passed: true, status_code: "decision_render_ok" }), + expect.objectContaining({ id: "M060-S02-SUMMARY-BOUNDARY", passed: true, status_code: "boundary_markers_ok" }), + ]); + + const rendered = renderM060S02Report(report); + expect(rendered).toContain("Boundary contract: PASS"); + expect(rendered).toContain("Manifest anchor:"); + expect(rendered).toContain("M060-S02-SUMMARY-BOUNDARY PASS"); + }); + + test("fails with stable status codes for missing wiring and missing canonical markers", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + + const result = await buildM060S02ProofHarness({ + json: true, + stdout: { write: (chunk: string) => void stdout.push(chunk) }, + stderr: { write: (chunk: string) => void stderr.push(chunk) }, + readTextFile: createReadTextFile({ + package: JSON.stringify({ scripts: {} }), + registry: "# Script Registry\n", + decisions: PASSING_DECISIONS.replace("deterministic repo-local verifier", "repo-local check"), + summary: PASSING_SUMMARY.replace("File overlap is allowed when the proof class differs.", "No overlap guidance here."), + }), + }); + + const report = JSON.parse(stdout.join("")) as EvaluationReport; + expect(result.exitCode).toBe(1); + expect(report.overallPassed).toBe(false); + expect(report.checks).toEqual([ + expect.objectContaining({ id: "M060-S02-PACKAGE-WIRING", passed: false, status_code: "package_wiring_missing" }), + expect.objectContaining({ id: "M060-S02-REGISTRY-WIRING", passed: false, status_code: "registry_rows_missing" }), + expect.objectContaining({ id: "M060-S02-DECISION-RENDER", passed: false, status_code: "decision_marker_missing" }), + expect.objectContaining({ id: "M060-S02-SUMMARY-BOUNDARY", passed: false, status_code: "boundary_marker_missing" }), + ]); + expect(report.checks[2]?.detail).toContain("deterministic repo-local verifier"); + expect(report.checks[3]?.detail).toContain("File overlap is allowed when the proof class differs."); + expect(stderr.join(" ")).toContain("package_wiring_missing"); + expect(stderr.join(" ")).toContain("registry_rows_missing"); + expect(stderr.join(" ")).toContain("decision_marker_missing"); + expect(stderr.join(" ")).toContain("boundary_marker_missing"); + }); + + test("surfaces unreadable and malformed dependency states with stable status codes", async () => { + const packageInvalid = await evaluateM060S02BoundaryContract({ + readTextFile: createReadTextFile({ package: "{ not valid json" }), + }); + expect(packageInvalid.checks[0]).toEqual( + expect.objectContaining({ id: "M060-S02-PACKAGE-WIRING", passed: false, status_code: "package_json_invalid" }), + ); + + const summaryUnreadable = await evaluateM060S02BoundaryContract({ + readTextFile: createReadTextFile({ summary: new Error("enoent") }), + }); + expect(summaryUnreadable.checks[3]).toEqual( + expect.objectContaining({ id: "M060-S02-SUMMARY-BOUNDARY", passed: false, status_code: "summary_file_unreadable" }), + ); + expect(summaryUnreadable.checks[3]?.detail).toContain(".gsd/milestones/M027/M027-SUMMARY.md"); + + const decisionUnreadable = await evaluateM060S02BoundaryContract({ + readTextFile: createReadTextFile({ decisions: new Error("permission denied") }), + }); + expect(decisionUnreadable.checks[2]).toEqual( + expect.objectContaining({ id: "M060-S02-DECISION-RENDER", passed: false, status_code: "decision_file_unreadable" }), + ); + expect(decisionUnreadable.checks[2]?.detail).toContain(".gsd/DECISIONS.md"); + }); + + test("flags missing rendered D171 row and missing M027 proof command markers", async () => { + const missingDecision = await evaluateM060S02BoundaryContract({ + readTextFile: createReadTextFile({ + decisions: PASSING_DECISIONS.replace("| D171 |", "| D999 |"), + }), + }); + expect(missingDecision.checks[2]).toEqual( + expect.objectContaining({ id: "M060-S02-DECISION-RENDER", passed: false, status_code: "decision_render_missing" }), + ); + + const missingVerifierFamily = await evaluateM060S02BoundaryContract({ + readTextFile: createReadTextFile({ + summary: PASSING_SUMMARY + .replace(", \`bun run verify:m027:s03\`, and \`bun run verify:m027:s04\`.", ".") + .replace("outside the live retriever and therefore not evidence of retriever coverage.", "not part of the retriever scope."), + }), + }); + expect(missingVerifierFamily.checks[3]).toEqual( + expect.objectContaining({ id: "M060-S02-SUMMARY-BOUNDARY", passed: false, status_code: "boundary_marker_missing" }), + ); + expect(missingVerifierFamily.checks[3]?.detail).toContain("outside the live retriever"); + }); + + test("wires the canonical package script", () => { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.["verify:m060:s02"]).toBe("bun scripts/verify-m060-s02.ts"); + }); +}); diff --git a/scripts/verify-m060-s02.ts b/scripts/verify-m060-s02.ts new file mode 100644 index 00000000..d88eecb2 --- /dev/null +++ b/scripts/verify-m060-s02.ts @@ -0,0 +1,367 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { + M060_S01_RUNTIME_TARGETS, + M060_S01_TYPE_ONLY_EXEMPTIONS, +} from "../src/knowledge/test-coverage-exemptions.ts"; + +const COMMAND_NAME = "verify:m060:s02" as const; +const EXPECTED_PACKAGE_SCRIPT = "bun scripts/verify-m060-s02.ts"; +const REGISTRY_PATH = path.resolve(import.meta.dir, "REGISTRY.md"); +const PACKAGE_JSON_PATH = path.resolve(import.meta.dir, "../package.json"); +const DECISIONS_PATH = path.resolve(import.meta.dir, "../.gsd/DECISIONS.md"); +const M027_SUMMARY_PATH = path.resolve( + import.meta.dir, + "../.gsd/milestones/M027/M027-SUMMARY.md", +); +const EXPECTED_REGISTRY_ROWS = [ + "| scripts/verify-m060-s02.test.ts | Regression tests for the M060 S02 ownership-boundary verifier. | M060 | internal | none |", + "| scripts/verify-m060-s02.ts | Verification CLI for the M060 S02 M060-vs-M027 ownership-boundary contract. | M060 | active | package:verify:m060:s02 |", +] as const; +const EXPECTED_DECISION_MARKERS = [ + "| D171 |", + "Define the boundary by proof class, not file exclusivity", + "M060 owns direct same-name unit tests plus explicit type-only exemptions", + "M027 owns persisted-corpus audit, repair/status, and live retriever acceptance proofs", + "deterministic repo-local verifier", +] as const; +const EXPECTED_SUMMARY_MARKERS = [ + "## M060/M027 ownership boundary", + "src/knowledge/test-coverage-exemptions.ts", + "bun run verify:m060:s01", + "M060 owns direct same-name unit tests", + "M027 owns persisted-corpus audit, repair/status, and live retriever acceptance proof", + "File overlap is allowed when the proof class differs.", + "issue_comments", + "audited and repairable under M027", + "outside the live retriever", + "D171", + ".gsd/DECISIONS.md", +] as const; + +export const M060_S02_CHECK_IDS = [ + "M060-S02-PACKAGE-WIRING", + "M060-S02-REGISTRY-WIRING", + "M060-S02-DECISION-RENDER", + "M060-S02-SUMMARY-BOUNDARY", +] as const; + +export type M060S02CheckId = (typeof M060_S02_CHECK_IDS)[number]; + +export type Check = { + id: M060S02CheckId; + passed: boolean; + skipped: boolean; + status_code: string; + detail?: string; +}; + +export type EvaluationReport = { + command: typeof COMMAND_NAME; + generatedAt: string; + check_ids: readonly M060S02CheckId[]; + overallPassed: boolean; + checks: Check[]; +}; + +type StdWriter = { + write: (chunk: string) => boolean | void; +}; + +type EvaluateOptions = { + generatedAt?: string; + readTextFile?: (filePath: string) => Promise; +}; + +type BuildOptions = EvaluateOptions & { + json?: boolean; + stdout?: StdWriter; + stderr?: StdWriter; +}; + +export async function evaluateM060S02BoundaryContract( + options: EvaluateOptions = {}, +): Promise { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const readTextFile = options.readTextFile ?? defaultReadTextFile; + + const packageCheck = await buildPackageCheck(readTextFile); + const registryCheck = await buildRegistryCheck(readTextFile); + const decisionCheck = await buildDecisionRenderCheck(readTextFile); + const summaryCheck = await buildSummaryBoundaryCheck(readTextFile); + + const checks = [packageCheck, registryCheck, decisionCheck, summaryCheck]; + + return { + command: COMMAND_NAME, + generatedAt, + check_ids: M060_S02_CHECK_IDS, + overallPassed: checks.every((check) => check.passed || check.skipped), + checks, + }; +} + +export function renderM060S02Report(report: EvaluationReport): string { + const lines = [ + "M060 S02 ownership-boundary verifier", + `Generated at: ${report.generatedAt}`, + `Boundary contract: ${report.overallPassed ? "PASS" : "FAIL"}`, + `Manifest anchor: ${M060_S01_RUNTIME_TARGETS.length} runtime targets, ${M060_S01_TYPE_ONLY_EXEMPTIONS.length} type-only exemptions from src/knowledge/test-coverage-exemptions.ts`, + "Checks:", + ]; + + for (const check of report.checks) { + const verdict = check.skipped ? "SKIP" : check.passed ? "PASS" : "FAIL"; + lines.push( + `- ${check.id} ${verdict} status_code=${check.status_code}${check.detail ? ` ${check.detail}` : ""}`, + ); + } + + return `${lines.join("\n")}\n`; +} + +export async function buildM060S02ProofHarness( + options: BuildOptions = {}, +): Promise<{ exitCode: number; report: EvaluationReport }> { + const stdout = options.stdout ?? process.stdout; + const stderr = options.stderr ?? process.stderr; + const report = await evaluateM060S02BoundaryContract(options); + + if (options.json) { + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + stdout.write(renderM060S02Report(report)); + } + + if (!report.overallPassed) { + const failingCodes = report.checks + .filter((check) => !check.passed && !check.skipped) + .map((check) => `${check.id}:${check.status_code}`) + .join(", "); + stderr.write(`${COMMAND_NAME} failed: ${failingCodes}\n`); + } + + return { + exitCode: report.overallPassed ? 0 : 1, + report, + }; +} + +export function parseM060S02Args(args: readonly string[]): { json: boolean } { + let json = false; + + for (const arg of args) { + if (arg === "--json") { + json = true; + continue; + } + + throw new Error(`invalid_cli_args: Unknown argument: ${arg}`); + } + + return { json }; +} + +async function buildPackageCheck( + readTextFile: (filePath: string) => Promise, +): Promise { + let packageText: string; + try { + packageText = await readTextFile(PACKAGE_JSON_PATH); + } catch (error) { + return failCheck("M060-S02-PACKAGE-WIRING", "package_file_unreadable", error); + } + + let parsed: { scripts?: Record }; + try { + parsed = JSON.parse(packageText) as { scripts?: Record }; + } catch (error) { + return failCheck("M060-S02-PACKAGE-WIRING", "package_json_invalid", error); + } + + const actualScript = parsed.scripts?.[COMMAND_NAME]; + if (actualScript == null) { + return failCheck( + "M060-S02-PACKAGE-WIRING", + "package_wiring_missing", + `package.json must define scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT}`, + ); + } + + if (actualScript !== EXPECTED_PACKAGE_SCRIPT) { + return failCheck( + "M060-S02-PACKAGE-WIRING", + "package_wiring_incorrect", + `Expected scripts.${COMMAND_NAME}=${EXPECTED_PACKAGE_SCRIPT} but found ${actualScript}`, + ); + } + + return passCheck( + "M060-S02-PACKAGE-WIRING", + "package_wiring_ok", + `package.json wires ${COMMAND_NAME} to ${EXPECTED_PACKAGE_SCRIPT}`, + ); +} + +async function buildRegistryCheck( + readTextFile: (filePath: string) => Promise, +): Promise { + let registryText: string; + try { + registryText = await readTextFile(REGISTRY_PATH); + } catch (error) { + return failCheck("M060-S02-REGISTRY-WIRING", "registry_file_unreadable", error); + } + + const missingRows = EXPECTED_REGISTRY_ROWS.filter((row) => !registryText.includes(row)); + if (missingRows.length > 0) { + return failCheck( + "M060-S02-REGISTRY-WIRING", + "registry_rows_missing", + `scripts/REGISTRY.md must include canonical rows for ${missingRows.map(extractRegistryPathFromRow).join(", ")}`, + ); + } + + return passCheck( + "M060-S02-REGISTRY-WIRING", + "registry_rows_ok", + `scripts/REGISTRY.md registers ${EXPECTED_REGISTRY_ROWS.map(extractRegistryPathFromRow).join(" and ")}.`, + ); +} + +async function buildDecisionRenderCheck( + readTextFile: (filePath: string) => Promise, +): Promise { + let decisionsText: string; + try { + decisionsText = await readTextFile(DECISIONS_PATH); + } catch (error) { + return failCheck( + "M060-S02-DECISION-RENDER", + "decision_file_unreadable", + `${formatPathForDetail(DECISIONS_PATH)}: ${normalizeDetail(error)}`, + ); + } + + if (!decisionsText.includes("| D171 |")) { + return failCheck( + "M060-S02-DECISION-RENDER", + "decision_render_missing", + `${formatPathForDetail(DECISIONS_PATH)} must include rendered decision D171.`, + ); + } + + const missingMarkers = EXPECTED_DECISION_MARKERS.filter((marker) => !decisionsText.includes(marker)); + if (missingMarkers.length > 0) { + return failCheck( + "M060-S02-DECISION-RENDER", + "decision_marker_missing", + `Rendered D171 is missing canonical markers: ${missingMarkers.join(" | ")}`, + ); + } + + return passCheck( + "M060-S02-DECISION-RENDER", + "decision_render_ok", + "Rendered D171 preserves the proof-class boundary rationale.", + ); +} + +async function buildSummaryBoundaryCheck( + readTextFile: (filePath: string) => Promise, +): Promise { + let summaryText: string; + try { + summaryText = await readTextFile(M027_SUMMARY_PATH); + } catch (error) { + return failCheck( + "M060-S02-SUMMARY-BOUNDARY", + "summary_file_unreadable", + `${formatPathForDetail(M027_SUMMARY_PATH)}: ${normalizeDetail(error)}`, + ); + } + + const missingMarkers = EXPECTED_SUMMARY_MARKERS.filter((marker) => !summaryText.includes(marker)); + if (missingMarkers.length > 0) { + return failCheck( + "M060-S02-SUMMARY-BOUNDARY", + "boundary_marker_missing", + `Canonical ownership summary is missing markers: ${missingMarkers.join(" | ")}`, + ); + } + + for (const verifier of [ + "bun run verify:m027:s01", + "bun run verify:m027:s02", + "bun run verify:m027:s03", + "bun run verify:m027:s04", + ]) { + if (!summaryText.includes(verifier)) { + return failCheck( + "M060-S02-SUMMARY-BOUNDARY", + "boundary_marker_missing", + `Canonical ownership summary must include corpus-level proof command ${verifier}`, + ); + } + } + + return passCheck( + "M060-S02-SUMMARY-BOUNDARY", + "boundary_markers_ok", + `Canonical ownership summary references the shared manifest anchor (${M060_S01_RUNTIME_TARGETS.length} runtime targets / ${M060_S01_TYPE_ONLY_EXEMPTIONS.length} type-only exemptions) and corpus-level proof family without claiming file exclusivity.`, + ); +} + +function extractRegistryPathFromRow(row: string): string { + return row.split("|")[1]?.trim() ?? row; +} + +function passCheck(id: M060S02CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: true, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function failCheck(id: M060S02CheckId, status_code: string, detail?: unknown): Check { + return { + id, + passed: false, + skipped: false, + status_code, + detail: detail == null ? undefined : normalizeDetail(detail), + }; +} + +function normalizeDetail(detail: unknown): string { + if (detail instanceof Error) { + return detail.message; + } + if (typeof detail === "string") { + return detail; + } + return String(detail); +} + +function formatPathForDetail(filePath: string): string { + return path.relative(path.resolve(import.meta.dir, ".."), filePath).replace(/\\/gu, "/"); +} + +async function defaultReadTextFile(filePath: string): Promise { + return readFile(filePath, "utf8"); +} + +if (import.meta.main) { + try { + const args = parseM060S02Args(process.argv.slice(2)); + const { exitCode } = await buildM060S02ProofHarness(args); + process.exit(exitCode); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${COMMAND_NAME} failed: ${message}\n`); + process.exit(1); + } +} diff --git a/src/config.test.ts b/src/config.test.ts index a7006bef..662382a9 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -22,8 +22,6 @@ beforeEach(() => { } savedEnv.MCP_INTERNAL_BASE_URL = process.env.MCP_INTERNAL_BASE_URL; delete process.env.MCP_INTERNAL_BASE_URL; - savedEnv.SLACK_WEBHOOK_RELAY_SOURCES = process.env.SLACK_WEBHOOK_RELAY_SOURCES; - delete process.env.SLACK_WEBHOOK_RELAY_SOURCES; }); afterEach(() => { @@ -40,44 +38,5 @@ describe("loadConfig", () => { test("defaults MCP internal base URL to the internal ACA app host without the external route suffix or port", async () => { const config = await loadConfig(); expect(config.mcpInternalBaseUrl).toBe("http://ca-kodiai"); - expect(config.slackWebhookRelaySources).toEqual([]); - }); - - test("parses Slack webhook relay sources from app config", async () => { - process.env.SLACK_WEBHOOK_RELAY_SOURCES = JSON.stringify([ - { - id: "buildkite", - targetChannel: "C_BUILD_ALERTS", - auth: { - type: "header_secret", - headerName: "x-relay-secret", - secret: "super-secret", - }, - filter: { - eventTypes: ["build.failed"], - textIncludes: ["failed"], - textExcludes: ["flaky"], - }, - }, - ]); - - const config = await loadConfig(); - - expect(config.slackWebhookRelaySources).toEqual([ - { - id: "buildkite", - targetChannel: "C_BUILD_ALERTS", - auth: { - type: "header_secret", - headerName: "x-relay-secret", - secret: "super-secret", - }, - filter: { - eventTypes: ["build.failed"], - textIncludes: ["failed"], - textExcludes: ["flaky"], - }, - }, - ]); }); }); diff --git a/src/config.ts b/src/config.ts index ebb123dc..8840027e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,8 +1,4 @@ import { z } from "zod"; -import { - parseWebhookRelaySourcesEnv, - webhookRelaySourceSchema, -} from "./slack/webhook-relay-config.ts"; const configSchema = z.object({ githubAppId: z.string().min(1, "GITHUB_APP_ID is required"), @@ -14,7 +10,6 @@ const configSchema = z.object({ slackKodiaiChannelId: z.string().min(1, "SLACK_KODIAI_CHANNEL_ID is required"), slackDefaultRepo: z.string().default("xbmc/xbmc"), slackAssistantModel: z.string().default("claude-3-5-haiku-latest"), - slackWebhookRelaySources: z.array(webhookRelaySourceSchema).default([]), port: z.coerce.number().default(3000), logLevel: z.string().default("info"), botAllowList: z @@ -92,16 +87,6 @@ export async function loadConfig(): Promise { process.exit(1); } - let slackWebhookRelaySources; - try { - slackWebhookRelaySources = parseWebhookRelaySourcesEnv(process.env.SLACK_WEBHOOK_RELAY_SOURCES); - } catch (err) { - console.error( - `FATAL: ${err instanceof Error ? err.message : String(err)}`, - ); - process.exit(1); - } - const result = configSchema.safeParse({ githubAppId: process.env.GITHUB_APP_ID, githubPrivateKey: privateKey, @@ -112,7 +97,6 @@ export async function loadConfig(): Promise { slackKodiaiChannelId: process.env.SLACK_KODIAI_CHANNEL_ID, slackDefaultRepo: process.env.SLACK_DEFAULT_REPO, slackAssistantModel: process.env.SLACK_ASSISTANT_MODEL, - slackWebhookRelaySources, port: process.env.PORT, logLevel: process.env.LOG_LEVEL, botAllowList: process.env.BOT_ALLOW_LIST, diff --git a/src/contributor/calibration-change-contract.test.ts b/src/contributor/calibration-change-contract.test.ts index b778d7a6..f623c126 100644 --- a/src/contributor/calibration-change-contract.test.ts +++ b/src/contributor/calibration-change-contract.test.ts @@ -67,25 +67,21 @@ type ContractModule = { ) => Error & { code: string }; }; -const importModule = new Function( - "specifier", - "return import(specifier)", -) as (specifier: string) => Promise; async function loadSnapshotModule(): Promise { - return (await importModule("./xbmc-fixture-snapshot.ts").catch( + return (await import("./xbmc-fixture-snapshot.ts").catch( () => null, )) as SnapshotModule | null; } async function loadCalibrationModule(): Promise { - return (await importModule("./calibration-evaluator.ts").catch( + return (await import("./calibration-evaluator.ts").catch( () => null, )) as CalibrationModule | null; } async function loadContractModule(): Promise { - return (await importModule("./calibration-change-contract.ts").catch( + return (await import("./calibration-change-contract.ts").catch( () => null, )) as ContractModule | null; } diff --git a/src/contributor/calibration-evaluator.test.ts b/src/contributor/calibration-evaluator.test.ts index d4096c66..1f774271 100644 --- a/src/contributor/calibration-evaluator.test.ts +++ b/src/contributor/calibration-evaluator.test.ts @@ -15,19 +15,15 @@ type CalibrationModule = { ) => any; }; -const importModule = new Function( - "specifier", - "return import(specifier)", -) as (specifier: string) => Promise; async function loadSnapshotModule(): Promise { - return (await importModule("./xbmc-fixture-snapshot.ts").catch( + return (await import("./xbmc-fixture-snapshot.ts").catch( () => null, )) as SnapshotModule | null; } async function loadCalibrationModule(): Promise { - return (await importModule("./calibration-evaluator.ts").catch( + return (await import("./calibration-evaluator.ts").catch( () => null, )) as CalibrationModule | null; } diff --git a/src/contributor/profile-store.test.ts b/src/contributor/profile-store.test.ts index b9dd9d6d..4b0bc221 100644 --- a/src/contributor/profile-store.test.ts +++ b/src/contributor/profile-store.test.ts @@ -17,9 +17,7 @@ import { createContributorProfileStore } from "./profile-store.ts"; import type { ContributorProfileStore } from "./types.ts"; import type { Sql } from "../db/client.ts"; -const DATABASE_URL = - process.env.TEST_DATABASE_URL ?? - "postgresql://kodiai:kodiai@localhost:5432/kodiai"; +const TEST_DB_URL = process.env.TEST_DATABASE_URL; const mockLogger = { info: () => {}, @@ -39,21 +37,20 @@ async function truncateAll(): Promise { await sql`TRUNCATE contributor_expertise, contributor_profiles CASCADE`; } -beforeAll(async () => { - sql = postgres(DATABASE_URL, { - max: 5, - idle_timeout: 20, - connect_timeout: 10, +describe.skipIf(!TEST_DB_URL)("ContributorProfileStore", () => { + beforeAll(async () => { + sql = postgres(TEST_DB_URL!, { + max: 5, + idle_timeout: 20, + connect_timeout: 10, + }); + await runMigrations(sql); + store = createContributorProfileStore({ sql, logger: mockLogger }); }); - await runMigrations(sql); - store = createContributorProfileStore({ sql, logger: mockLogger }); -}); -afterAll(async () => { - await sql.end(); -}); - -describe("ContributorProfileStore", () => { + afterAll(async () => { + await sql.end(); + }); beforeEach(async () => { await truncateAll(); }); diff --git a/src/contributor/xbmc-fixture-refresh.test.ts b/src/contributor/xbmc-fixture-refresh.test.ts index e1ab3698..0701ac84 100644 --- a/src/contributor/xbmc-fixture-refresh.test.ts +++ b/src/contributor/xbmc-fixture-refresh.test.ts @@ -7,15 +7,11 @@ type RefreshModule = { refreshXbmcFixtureSnapshot?: (options?: Record) => Promise; }; -const importModule = new Function( - "specifier", - "return import(specifier)", -) as (specifier: string) => Promise; const tempDirs: string[] = []; async function loadRefreshModule(): Promise { - return (await importModule("./xbmc-fixture-refresh.ts").catch( + return (await import("./xbmc-fixture-refresh.ts").catch( () => null, )) as RefreshModule | null; } diff --git a/src/contributor/xbmc-fixture-refresh.ts b/src/contributor/xbmc-fixture-refresh.ts index a65468ac..68632892 100644 --- a/src/contributor/xbmc-fixture-refresh.ts +++ b/src/contributor/xbmc-fixture-refresh.ts @@ -1014,7 +1014,6 @@ function buildGitHubAppConfig(repository: string, privateKey: string) { slackKodiaiChannelId: "unused", slackDefaultRepo: repository, slackAssistantModel: "unused", - slackWebhookRelaySources: [], port: 0, logLevel: "info", botAllowList: [], diff --git a/src/contributor/xbmc-fixture-snapshot.test.ts b/src/contributor/xbmc-fixture-snapshot.test.ts index b796c7eb..b9e091f7 100644 --- a/src/contributor/xbmc-fixture-snapshot.test.ts +++ b/src/contributor/xbmc-fixture-snapshot.test.ts @@ -12,15 +12,11 @@ type SnapshotModule = { inspectXbmcFixtureSnapshot?: (value: unknown) => any; }; -const importModule = new Function( - "specifier", - "return import(specifier)", -) as (specifier: string) => Promise; const tempDirs: string[] = []; async function loadSnapshotModule(): Promise { - return (await importModule("./xbmc-fixture-snapshot.ts").catch( + return (await import("./xbmc-fixture-snapshot.ts").catch( () => null, )) as SnapshotModule | null; } diff --git a/src/db/migrate.ts b/src/db/migrate.ts index 616f440b..491265d8 100644 --- a/src/db/migrate.ts +++ b/src/db/migrate.ts @@ -4,6 +4,10 @@ import type { Sql } from "./client.ts"; const MIGRATIONS_DIR = join(import.meta.dir, "migrations"); +// This module is an operator-facing CLI surface. Direct console output is +// intentional here so migration/apply/rollback progress is visible in local +// runs and CI logs; eslint.config.mjs documents the file-level exception. + /** * Apply all pending migrations in order. * diff --git a/src/db/migrations/012-wiki-staleness-run-state.down.sql b/src/db/migrations/012-wiki-staleness-run-state.down.sql new file mode 100644 index 00000000..60960595 --- /dev/null +++ b/src/db/migrations/012-wiki-staleness-run-state.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS wiki_staleness_run_state; diff --git a/src/db/migrations/013-review-clusters.down.sql b/src/db/migrations/013-review-clusters.down.sql new file mode 100644 index 00000000..65fe1c84 --- /dev/null +++ b/src/db/migrations/013-review-clusters.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS review_cluster_assignments; +DROP TABLE IF EXISTS cluster_run_state; +DROP TABLE IF EXISTS review_clusters; diff --git a/src/db/migrations/016-issue-triage-state.down.sql b/src/db/migrations/016-issue-triage-state.down.sql new file mode 100644 index 00000000..9c8ef950 --- /dev/null +++ b/src/db/migrations/016-issue-triage-state.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS issue_triage_state; diff --git a/src/db/migrations/025-wiki-style-cache.down.sql b/src/db/migrations/025-wiki-style-cache.down.sql new file mode 100644 index 00000000..5284449c --- /dev/null +++ b/src/db/migrations/025-wiki-style-cache.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS wiki_style_cache; diff --git a/src/db/migrations/026-guardrail-audit.down.sql b/src/db/migrations/026-guardrail-audit.down.sql new file mode 100644 index 00000000..0872f51e --- /dev/null +++ b/src/db/migrations/026-guardrail-audit.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS guardrail_audit; diff --git a/src/db/migrations/030-reserved.down.sql b/src/db/migrations/030-reserved.down.sql new file mode 100644 index 00000000..a89452f8 --- /dev/null +++ b/src/db/migrations/030-reserved.down.sql @@ -0,0 +1,7 @@ +-- 030-reserved.down.sql +-- Reserved migration slot rollback. +-- +-- The paired down migration is intentionally schema-neutral because the reserved +-- slot must not mutate schema state during rollback proofs or late application. +-- Keep this file as a no-op. +SELECT 1; diff --git a/src/db/migrations/030-reserved.sql b/src/db/migrations/030-reserved.sql new file mode 100644 index 00000000..5a6201bb --- /dev/null +++ b/src/db/migrations/030-reserved.sql @@ -0,0 +1,10 @@ +-- 030-reserved.sql +-- Reserved migration slot. +-- +-- Migration 030 was left unused historically between 029 and 031. We reserve it +-- explicitly so the sequence is no longer ambiguous and later rollback proofs can +-- reason about a contiguous migration timeline. +-- +-- This file must remain schema-neutral even if applied late to an existing +-- database. Do not add DDL here. +SELECT 1; diff --git a/src/db/migrations/033-canonical-code-corpus.down.sql b/src/db/migrations/033-canonical-code-corpus.down.sql new file mode 100644 index 00000000..0974b686 --- /dev/null +++ b/src/db/migrations/033-canonical-code-corpus.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS canonical_corpus_backfill_state; +DROP TABLE IF EXISTS canonical_code_chunks; diff --git a/src/db/migrations/034-review-graph.down.sql b/src/db/migrations/034-review-graph.down.sql new file mode 100644 index 00000000..4973b244 --- /dev/null +++ b/src/db/migrations/034-review-graph.down.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS review_graph_edges; +DROP TABLE IF EXISTS review_graph_nodes; +DROP TABLE IF EXISTS review_graph_files; +DROP TABLE IF EXISTS review_graph_builds; diff --git a/src/db/migrations/035-generated-rules.down.sql b/src/db/migrations/035-generated-rules.down.sql new file mode 100644 index 00000000..f97eeeb4 --- /dev/null +++ b/src/db/migrations/035-generated-rules.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS generated_rules; diff --git a/src/db/migrations/036-suggestion-cluster-models.down.sql b/src/db/migrations/036-suggestion-cluster-models.down.sql new file mode 100644 index 00000000..566da872 --- /dev/null +++ b/src/db/migrations/036-suggestion-cluster-models.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS suggestion_cluster_models; diff --git a/src/execution/executor.test.ts b/src/execution/executor.test.ts index 15460e39..01f24131 100644 --- a/src/execution/executor.test.ts +++ b/src/execution/executor.test.ts @@ -1,6 +1,6 @@ import { $ } from "bun"; import { test, expect, afterEach, mock, beforeEach } from "bun:test"; -import { mkdtemp, rm, readFile, writeFile, mkdir } from "node:fs/promises"; +import { mkdtemp, rm, readFile, writeFile, mkdir, lstat, symlink, stat } from "node:fs/promises"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { buildSecurityClaudeMd, createExecutor, prepareAgentWorkspace } from "./executor.ts"; @@ -97,7 +97,6 @@ function makeConfig(overrides?: Partial): AppConfig { slackKodiaiChannelId: "C123", slackDefaultRepo: "xbmc/xbmc", slackAssistantModel: "claude-3-5-haiku-latest", - slackWebhookRelaySources: [], port: 3000, logLevel: "info", botAllowList: [], @@ -472,14 +471,9 @@ function createTestableExecutor(deps: { !isWriteMode && context.prNumber !== undefined && taskType !== "review.full"; - const gitDiffInspectionAvailable = context.gitDiffInspectionAvailable !== false; const baseTools = isReadOnlyPrMention - ? gitDiffInspectionAvailable - ? ["Read", "Grep", "Bash(git diff:*)", "Bash(git status:*)"] - : ["Read", "Grep", "Bash(git status:*)"] - : gitDiffInspectionAvailable - ? ["Read", "Grep", "Glob", "Bash(git diff:*)", "Bash(git log:*)", "Bash(git show:*)", "Bash(git status:*)"] - : ["Read", "Grep", "Glob", "Bash(git status:*)"]; + ? ["Read", "Grep", "Bash(git diff:*)", "Bash(git status:*)"] + : ["Read", "Grep", "Glob", "Bash(git diff:*)", "Bash(git log:*)", "Bash(git show:*)", "Bash(git status:*)"]; const writeTools = isWriteMode ? ["Edit", "Write", "MultiEdit"] : []; const mcpTools = buildAllowedMcpTools(Object.keys(mcpServers)); const allowedTools = [...baseTools, ...writeTools, ...mcpTools]; @@ -1183,63 +1177,6 @@ test("ACA dispatch: explicit review mention stages a review bundle transport wit }); }); -test("ACA dispatch: degraded review lane strips git diff and history tools", async () => { - tmpDir = await mkdtemp(join(tmpdir(), "kodiai-executor-test-")); - const sourceRepoDir = await mkdtemp(join(tmpdir(), "kodiai-source-repo-")); - await mkdir(join(sourceRepoDir, "src"), { recursive: true }); - await writeFile(join(sourceRepoDir, "src", "feature.ts"), "export const feature = true;\n"); - await writeFile(join(sourceRepoDir, ".kodiai.yml"), "review:\n enabled: true\n"); - - await $`git -C ${sourceRepoDir} init`.quiet(); - await $`git -C ${sourceRepoDir} config user.email test@example.com`.quiet(); - await $`git -C ${sourceRepoDir} config user.name "Test User"`.quiet(); - await $`git -C ${sourceRepoDir} remote add origin https://github.com/xbmc/xbmc.git`.quiet(); - await $`git -C ${sourceRepoDir} add .`.quiet(); - await $`git -C ${sourceRepoDir} commit -m base`.quiet(); - await $`git -C ${sourceRepoDir} branch -M main`.quiet(); - await $`git -C ${sourceRepoDir} checkout -b pr-mention`.quiet(); - await writeFile(join(sourceRepoDir, "src", "feature.ts"), "export const feature = 'updated';\n"); - await $`git -C ${sourceRepoDir} commit -am feature`.quiet(); - await $`git -C ${sourceRepoDir} update-ref refs/remotes/origin/main $(git -C ${sourceRepoDir} rev-parse main)`.quiet(); - - const config = makeConfig(); - const logger = makeLogger(); - const registry = createMcpJobRegistry(); - - const executor = createTestableExecutor({ - githubApp: makeGithubApp(), - logger, - config, - mcpJobRegistry: registry, - launchFn: async () => ({ executionName: "exec-degraded-review-lane" }), - pollFn: async () => ({ status: "succeeded", durationMs: 1000 }), - readResultFn: async () => makeJobResult(), - createWorkspaceDirFn: async () => tmpDir!, - }); - - await executor.execute( - makeContext(sourceRepoDir, { - eventType: "pull_request.opened", - prNumber: 80, - taskType: "review.full", - gitDiffInspectionAvailable: false, - }), - ); - - const rawAgentConfig = await readFile(join(tmpDir!, "agent-config.json"), "utf-8"); - const agentConfig = JSON.parse(rawAgentConfig) as { - allowedTools: string[]; - taskType: string; - }; - - expect(agentConfig.taskType).toBe("review.full"); - expect(agentConfig.allowedTools).toContain("Glob"); - expect(agentConfig.allowedTools).toContain("Bash(git status:*)"); - expect(agentConfig.allowedTools).not.toContain("Bash(git diff:*)"); - expect(agentConfig.allowedTools).not.toContain("Bash(git log:*)"); - expect(agentConfig.allowedTools).not.toContain("Bash(git show:*)"); -}); - test("ACA dispatch: conversational PR mention keeps reduced toolset", async () => { tmpDir = await mkdtemp(join(tmpdir(), "kodiai-executor-test-")); const config = makeConfig(); @@ -1408,3 +1345,198 @@ test("ACA dispatch: malformed remote timing payload falls back to locally measur }, ]); }); + +test("prepareAgentWorkspace copies the repo and writes agent-config with repoCwd", async () => { + const sourceRepoDir = await mkdtemp(join(tmpdir(), "kodiai-source-repo-")); + const workspaceDir = await mkdtemp(join(tmpdir(), "kodiai-agent-workspace-")); + const cleanupDirs = [sourceRepoDir, workspaceDir]; + + try { + await mkdir(join(sourceRepoDir, "src"), { recursive: true }); + await writeFile(join(sourceRepoDir, "src", "feature.ts"), "export const feature = true;\n"); + await writeFile(join(sourceRepoDir, ".kodiai.yml"), "review:\n enabled: true\n"); + + const { repoCwd } = await prepareAgentWorkspace({ + sourceRepoDir, + workspaceDir, + prompt: "Review this PR", + model: "claude-sonnet-4-5-20250929", + maxTurns: 25, + allowedTools: ["Read", "Grep", "Glob"], + taskType: "review.full", + mcpServerNames: ["github_comment"], + }); + + expect(repoCwd).toBe(join(workspaceDir, "repo")); + const repoDir = repoCwd!; + expect(await readFile(join(repoDir, "src", "feature.ts"), "utf-8")).toContain("feature = true"); + expect(await readFile(join(repoDir, ".kodiai.yml"), "utf-8")).toContain("review:"); + expect(await readFile(join(workspaceDir, "prompt.txt"), "utf-8")).toBe("Review this PR"); + + const rawAgentConfig = await readFile(join(workspaceDir, "agent-config.json"), "utf-8"); + const agentConfig = JSON.parse(rawAgentConfig) as { + prompt: string; + model: string; + maxTurns: number; + allowedTools: string[]; + taskType: string; + repoCwd?: string; + mcpServerNames?: string[]; + }; + + expect(agentConfig.prompt).toBe("Review this PR"); + expect(agentConfig.model).toBe("claude-sonnet-4-5-20250929"); + expect(agentConfig.maxTurns).toBe(25); + expect(agentConfig.allowedTools).toEqual(["Read", "Grep", "Glob"]); + expect(agentConfig.taskType).toBe("review.full"); + expect(agentConfig.repoCwd).toBe(repoCwd); + expect(agentConfig.mcpServerNames).toEqual(["github_comment"]); + } finally { + await Promise.all(cleanupDirs.map((dir) => rm(dir, { recursive: true, force: true }))); + } +}); + +test("prepareAgentWorkspace writes a review bundle transport for repos with tracked symlinks", async () => { + const sourceRepoDir = await mkdtemp(join(tmpdir(), "kodiai-source-symlink-repo-")); + const workspaceDir = await mkdtemp(join(tmpdir(), "kodiai-agent-symlink-workspace-")); + const cleanupDirs = [sourceRepoDir, workspaceDir]; + + try { + await mkdir(join(sourceRepoDir, "system", "settings"), { recursive: true }); + await writeFile(join(sourceRepoDir, ".kodiai.yml"), "review:\n enabled: true\n"); + await writeFile(join(sourceRepoDir, "system", "settings", "linux.xml"), "\n"); + + await $`git -C ${sourceRepoDir} init`.quiet(); + await $`git -C ${sourceRepoDir} config user.email t@example.com`.quiet(); + await $`git -C ${sourceRepoDir} config user.name T`.quiet(); + await $`git -C ${sourceRepoDir} remote add origin https://github.com/xbmc/xbmc.git`.quiet(); + await symlink("linux.xml", join(sourceRepoDir, "system", "settings", "freebsd.xml")); + await $`git -C ${sourceRepoDir} add .`.quiet(); + await $`git -C ${sourceRepoDir} commit -m init`.quiet(); + await $`git -C ${sourceRepoDir} branch -M main`.quiet(); + await $`git -C ${sourceRepoDir} checkout -b pr-mention`.quiet(); + await writeFile(join(sourceRepoDir, "feature.ts"), "export const enabled = true;\n"); + await $`git -C ${sourceRepoDir} add .`.quiet(); + await $`git -C ${sourceRepoDir} commit -m feature`.quiet(); + await $`git -C ${sourceRepoDir} update-ref refs/remotes/origin/main $(git -C ${sourceRepoDir} rev-parse main)`.quiet(); + + const result = await prepareAgentWorkspace({ + sourceRepoDir, + workspaceDir, + prompt: "Review this PR", + model: "claude-sonnet-4-5-20250929", + maxTurns: 25, + allowedTools: ["Read", "Grep", "Glob"], + taskType: "review.full", + mcpServerNames: ["github_comment"], + }) as { repoCwd?: string; repoBundlePath?: string }; + + expect(result.repoCwd).toBeUndefined(); + expect(result.repoBundlePath).toBe(join(workspaceDir, "repo.bundle")); + expect((await lstat(join(workspaceDir, "repo.bundle"))).isFile()).toBe(true); + await expect(stat(join(workspaceDir, "repo"))).rejects.toThrow(); + + const rawAgentConfig = await readFile(join(workspaceDir, "agent-config.json"), "utf-8"); + const agentConfig = JSON.parse(rawAgentConfig) as { + repoCwd?: string; + repoTransport?: { + kind?: string; + bundlePath?: string; + headRef?: string; + baseRef?: string; + originUrl?: string; + }; + }; + + expect(agentConfig.repoCwd).toBeUndefined(); + expect(agentConfig.repoTransport).toEqual({ + kind: "review-bundle", + bundlePath: join(workspaceDir, "repo.bundle"), + headRef: "pr-mention", + baseRef: "main", + originUrl: "https://github.com/xbmc/xbmc.git", + }); + + const cloneCheckDir = await mkdtemp(join(tmpdir(), "kodiai-bundle-clone-check-")); + cleanupDirs.push(cloneCheckDir); + await $`git clone -b pr-mention ${join(workspaceDir, "repo.bundle")} ${cloneCheckDir}`.quiet(); + expect((await lstat(join(cloneCheckDir, "system", "settings", "freebsd.xml"))).isSymbolicLink()).toBe(true); + expect((await $`git -C ${cloneCheckDir} status --porcelain`.quiet()).text()).toBe(""); + expect((await $`git -C ${cloneCheckDir} diff origin/main...HEAD --stat`.quiet()).text()).toContain("feature.ts"); + } finally { + await Promise.all(cleanupDirs.map((dir) => rm(dir, { recursive: true, force: true }))); + } +}); + +test("prepareAgentWorkspace unshallows PR workspaces before writing a review bundle transport", async () => { + const tempRoot = await mkdtemp(join(tmpdir(), "kodiai-shallow-bundle-")); + const bareRepoDir = join(tempRoot, "origin.git"); + const seedRepoDir = join(tempRoot, "seed"); + const shallowRepoDir = join(tempRoot, "shallow"); + const workspaceDir = await mkdtemp(join(tmpdir(), "kodiai-agent-shallow-workspace-")); + const cloneCheckDir = await mkdtemp(join(tmpdir(), "kodiai-bundle-clone-check-")); + const cleanupDirs = [tempRoot, workspaceDir, cloneCheckDir]; + + try { + await $`git init --bare ${bareRepoDir}`.quiet(); + await $`git clone file://${bareRepoDir} ${seedRepoDir}`.quiet(); + await $`git -C ${seedRepoDir} config user.email t@example.com`.quiet(); + await $`git -C ${seedRepoDir} config user.name T`.quiet(); + await writeFile(join(seedRepoDir, "feature.txt"), "one\n"); + await $`git -C ${seedRepoDir} add feature.txt`.quiet(); + await $`git -C ${seedRepoDir} commit -m one`.quiet(); + await $`git -C ${seedRepoDir} branch -M master`.quiet(); + await $`git -C ${seedRepoDir} push origin master`.quiet(); + await writeFile(join(seedRepoDir, "feature.txt"), "one\ntwo\n"); + await $`git -C ${seedRepoDir} commit -am two`.quiet(); + await $`git -C ${seedRepoDir} push origin master`.quiet(); + await $`git -C ${seedRepoDir} checkout -b pr-mention`.quiet(); + await writeFile(join(seedRepoDir, "feature.txt"), "one\ntwo\npr\n"); + await $`git -C ${seedRepoDir} commit -am pr`.quiet(); + await $`git -C ${seedRepoDir} push origin pr-mention`.quiet(); + + await $`git clone --depth=1 --single-branch --branch master file://${bareRepoDir} ${shallowRepoDir}`.quiet(); + await $`git -C ${shallowRepoDir} fetch file://${bareRepoDir} pr-mention:pr-mention`.quiet(); + await $`git -C ${shallowRepoDir} checkout pr-mention`.quiet(); + await $`git -C ${shallowRepoDir} fetch origin master:refs/remotes/origin/master --depth=1`.quiet(); + expect((await $`git -C ${shallowRepoDir} rev-parse --is-shallow-repository`.quiet().text()).trim()).toBe("true"); + + const result = await prepareAgentWorkspace({ + sourceRepoDir: shallowRepoDir, + workspaceDir, + prompt: "Review this PR", + model: "claude-sonnet-4-5-20250929", + maxTurns: 25, + allowedTools: ["Read", "Grep", "Glob"], + taskType: "review.full", + mcpServerNames: ["github_comment"], + }) as { repoCwd?: string; repoBundlePath?: string }; + + expect(result.repoCwd).toBeUndefined(); + expect(result.repoBundlePath).toBe(join(workspaceDir, "repo.bundle")); + + const rawAgentConfig = await readFile(join(workspaceDir, "agent-config.json"), "utf-8"); + const agentConfig = JSON.parse(rawAgentConfig) as { + repoTransport?: { + kind?: string; + bundlePath?: string; + headRef?: string; + baseRef?: string; + originUrl?: string; + }; + }; + + expect(agentConfig.repoTransport).toEqual({ + kind: "review-bundle", + bundlePath: join(workspaceDir, "repo.bundle"), + headRef: "pr-mention", + baseRef: "master", + originUrl: "file://" + bareRepoDir, + }); + + await $`git clone -b pr-mention ${join(workspaceDir, "repo.bundle")} ${cloneCheckDir}`.quiet(); + expect((await $`git -C ${cloneCheckDir} diff origin/master...HEAD --stat`.quiet().text())).toContain("feature.txt"); + } finally { + await Promise.all(cleanupDirs.map((dir) => rm(dir, { recursive: true, force: true }))); + } +}); diff --git a/src/execution/executor.ts b/src/execution/executor.ts index 43588ea4..869601dc 100644 --- a/src/execution/executor.ts +++ b/src/execution/executor.ts @@ -430,36 +430,22 @@ export function createExecutor(deps: { !isWriteMode && context.prNumber !== undefined && taskType !== "review.full"; - const gitDiffInspectionAvailable = context.gitDiffInspectionAvailable !== false; const baseTools = isReadOnlyPrMention - ? gitDiffInspectionAvailable - ? [ - "Read", - "Grep", - "Bash(git diff:*)", - "Bash(git status:*)", - ] - : [ - "Read", - "Grep", - "Bash(git status:*)", - ] - : gitDiffInspectionAvailable - ? [ - "Read", - "Grep", - "Glob", - "Bash(git diff:*)", - "Bash(git log:*)", - "Bash(git show:*)", - "Bash(git status:*)", - ] - : [ - "Read", - "Grep", - "Glob", - "Bash(git status:*)", - ]; + ? [ + "Read", + "Grep", + "Bash(git diff:*)", + "Bash(git status:*)", + ] + : [ + "Read", + "Grep", + "Glob", + "Bash(git diff:*)", + "Bash(git log:*)", + "Bash(git show:*)", + "Bash(git status:*)", + ]; const writeTools = isWriteMode ? ["Edit", "Write", "MultiEdit"] : []; // Compute server names from the same deps (no instance construction yet) diff --git a/src/execution/review-prompt.test.ts b/src/execution/review-prompt.test.ts index d6f7be38..f26af2b3 100644 --- a/src/execution/review-prompt.test.ts +++ b/src/execution/review-prompt.test.ts @@ -1843,19 +1843,6 @@ describe("epistemic section placement in buildReviewPrompt", () => { }); }); -describe("degraded diff guidance in buildReviewPrompt", () => { - test("does not instruct merge-base git commands when local diff inspection is unavailable", () => { - const prompt = buildReviewPrompt(baseContext({ - gitDiffInspectionAvailable: false, - })); - - expect(prompt).toContain("Full local merge-base diff access is unavailable in this run."); - expect(prompt).toContain("Do not spend turns trying to reconstruct the diff with git history commands."); - expect(prompt).not.toContain("To see the full diff: Bash(git diff origin/main...HEAD)"); - expect(prompt).not.toContain("To see changed files with stats: Bash(git log origin/main..HEAD --stat)"); - }); -}); - // --------------------------------------------------------------------------- // Phase 115: Dep-bump rewrites, footnote citations, conventional commit // --------------------------------------------------------------------------- diff --git a/src/execution/review-prompt.ts b/src/execution/review-prompt.ts index 70039c96..7e6249a1 100644 --- a/src/execution/review-prompt.ts +++ b/src/execution/review-prompt.ts @@ -1695,7 +1695,6 @@ export function buildReviewPrompt(context: { structuralImpact?: StructuralImpactPayload | null; reviewBoundedness?: ReviewBoundednessContract | null; publishToolNames?: string[]; - gitDiffInspectionAvailable?: boolean; }): string { const lines: string[] = []; const scaleNotes: string[] = []; @@ -1856,22 +1855,11 @@ export function buildReviewPrompt(context: { "", "## Reading the code", "", + `To see the full diff: Bash(git diff origin/${context.baseBranch}...HEAD)`, + `To see changed files with stats: Bash(git log origin/${context.baseBranch}..HEAD --stat)`, + "Read the diff carefully before posting any comments.", ); - if (context.gitDiffInspectionAvailable === false) { - lines.push( - "Full local merge-base diff access is unavailable in this run.", - "Do not spend turns trying to reconstruct the diff with git history commands.", - "Use the changed file list above plus Read/Grep/Glob on those files to complete the review.", - ); - } else { - lines.push( - `To see the full diff: Bash(git diff origin/${context.baseBranch}...HEAD)`, - `To see changed files with stats: Bash(git log origin/${context.baseBranch}..HEAD --stat)`, - "Read the diff carefully before posting any comments.", - ); - } - // --- Review instructions --- lines.push( "", diff --git a/src/execution/types.ts b/src/execution/types.ts index 1d7a6dc3..093d28c4 100644 --- a/src/execution/types.ts +++ b/src/execution/types.ts @@ -70,13 +70,6 @@ export type ExecutionContext = { /** Enable review checkpoint MCP tool (save_review_checkpoint). */ enableCheckpointTool?: boolean; - /** - * False when the local workspace cannot support merge-base diff inspection. - * In that mode the agent should stay on changed files surfaced by the handler - * instead of spending turns on git diff/log/show reconstruction. - */ - gitDiffInspectionAvailable?: boolean; - /** Optional overrides for MCP tool surfaces (used for retry flows). */ enableInlineTools?: boolean; enableCommentTools?: boolean; diff --git a/src/handlers/ci-failure.test.ts b/src/handlers/ci-failure.test.ts new file mode 100644 index 00000000..b5a69947 --- /dev/null +++ b/src/handlers/ci-failure.test.ts @@ -0,0 +1,678 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; +import { readFileSync } from "node:fs"; +import type { CheckSuiteCompletedEvent } from "@octokit/webhooks-types"; +import type { Logger } from "pino"; +import { createCIFailureHandler } from "./ci-failure.ts"; +import { createQueueRunMetadata, getEmptyActiveJobs } from "../jobs/queue.test-helpers.ts"; +import { buildCIAnalysisMarker } from "../lib/ci-failure-formatter.ts"; +import type { GitHubApp } from "../auth/github-app.ts"; +import type { Sql } from "../db/client.ts"; +import type { JobQueue, JobQueueContext } from "../jobs/types.ts"; +import type { EventHandler, EventRouter, WebhookEvent } from "../webhook/types.ts"; + +type LogCall = { bindings: Record; message: string }; +type RegisteredHandler = { key: string; handler: EventHandler }; +type CheckRun = { name: string; conclusion: string | null; status: string }; +type CheckRunsPage = { data: Array }; +type QueueCall = { installationId: number; context?: JobQueueContext }; +type Comment = { id: number; body?: string | null }; +type FlakinessRow = { check_name: string; conclusion: string }; + +type HarnessOptions = { + headChecksByRef?: Record; + headChecksErrorByRef?: Record; + baseCommitsByRef?: Record>; + baseCommitsErrorByRef?: Record; + flakinessRows?: FlakinessRow[]; + commentsByPage?: Comment[][]; + listCommentsError?: Error; +}; + +function createSharedLogger() { + const debugCalls: LogCall[] = []; + const warnCalls: LogCall[] = []; + + const logger = { + debug: (bindings: Record, message: string) => { + debugCalls.push({ bindings, message }); + }, + warn: (bindings: Record, message: string) => { + warnCalls.push({ bindings, message }); + }, + info: () => {}, + error: () => {}, + child: () => logger, + } as unknown as Logger; + + return { logger, debugCalls, warnCalls }; +} + +function createCapturedRouter(): EventRouter & { captured: RegisteredHandler[] } { + const captured: RegisteredHandler[] = []; + return { + captured, + register(eventKey: string, handler: EventHandler) { + captured.push({ key: eventKey, handler }); + }, + dispatch: async () => {}, + }; +} + +function toAsyncIterable(pages: T[]): AsyncIterable { + return { + async *[Symbol.asyncIterator]() { + for (const page of pages) { + yield page; + } + }, + }; +} + +function createSqlMock(options: Pick = {}) { + const sql = (async ( + stringsOrValues: TemplateStringsArray | unknown[], + ...values: unknown[] + ) => { + if (Array.isArray(stringsOrValues) && !("raw" in stringsOrValues)) { + return { rows: stringsOrValues, columns: values }; + } + + const text = String.raw( + { raw: (stringsOrValues as TemplateStringsArray).raw }, + ...values.map(() => "?"), + ); + + if (text.includes("SELECT check_name, conclusion")) { + return options.flakinessRows ?? []; + } + + return []; + }) as unknown as Sql; + + return sql; +} + +function loadFixture(): CheckSuiteCompletedEvent { + return JSON.parse( + readFileSync("fixtures/webhooks/check_suite/completed-basic.json", "utf8"), + ) as unknown as CheckSuiteCompletedEvent; +} + +function clonePayload(): CheckSuiteCompletedEvent { + return structuredClone(loadFixture()); +} + +function makeEvent(payload: CheckSuiteCompletedEvent): WebhookEvent { + return { + id: "delivery-1", + name: "check_suite", + installationId: payload.installation?.id ?? 42, + payload: payload as unknown as Record, + }; +} + +function createHarness(options: HarnessOptions = {}) { + const router = createCapturedRouter(); + const { logger, debugCalls, warnCalls } = createSharedLogger(); + const queueCalls: QueueCall[] = []; + + const listForRef = mock(async () => ({ data: [] })); + const iterator = mock((_method: unknown, params: { ref: string }) => { + const error = options.headChecksErrorByRef?.[params.ref]; + if (error) { + return { + async *[Symbol.asyncIterator]() { + throw error; + }, + } as AsyncIterable; + } + + const pages = options.headChecksByRef?.[params.ref] ?? []; + return toAsyncIterable(pages); + }); + const listCommits = mock(async (params: { sha: string }) => { + const error = options.baseCommitsErrorByRef?.[params.sha]; + if (error) { + throw error; + } + + return { + data: options.baseCommitsByRef?.[params.sha] ?? [], + }; + }); + const listComments = mock(async (params: { page: number }) => { + if (options.listCommentsError) { + throw options.listCommentsError; + } + return { data: options.commentsByPage?.[params.page - 1] ?? [] }; + }); + const createComment = mock(async () => ({ data: { id: 999 } })); + const updateComment = mock(async () => ({ data: { id: 999 } })); + + const octokit = { + paginate: { iterator }, + rest: { + checks: { listForRef }, + repos: { listCommits }, + issues: { + listComments, + createComment, + updateComment, + }, + }, + }; + + const githubApp = { + getInstallationOctokit: async () => octokit as never, + getAppSlug: () => "kodiai", + initialize: async () => {}, + checkConnectivity: async () => true, + getInstallationToken: async () => "token", + getRepoInstallationContext: async () => null, + } as unknown as GitHubApp; + + const jobQueue: JobQueue = { + enqueue: async (installationId, run, context) => { + queueCalls.push({ installationId, context }); + return run( + createQueueRunMetadata({ + lane: context?.lane ?? "sync", + key: context?.key ?? "test-key", + }), + ); + }, + getQueueSize: () => 0, + getPendingCount: () => 0, + getActiveJobs: getEmptyActiveJobs, + }; + + createCIFailureHandler({ + eventRouter: router, + jobQueue, + githubApp, + sql: createSqlMock(options), + logger, + }); + + return { + router, + debugCalls, + warnCalls, + queueCalls, + octokit, + }; +} + +function extractPostedBody(harness: ReturnType): string { + const createCalls = harness.octokit.rest.issues.createComment.mock.calls as unknown as Array<[{ body: string }]>; + const createCall = createCalls[0]; + if (createCall) { + return createCall[0].body; + } + + const updateCalls = harness.octokit.rest.issues.updateComment.mock.calls as unknown as Array<[{ body: string }]>; + const updateCall = updateCalls[0]; + if (updateCall) { + return updateCall[0].body; + } + + throw new Error("Expected a CI comment to be created or updated"); +} + +describe("createCIFailureHandler", () => { + let fixture: CheckSuiteCompletedEvent; + + beforeEach(() => { + fixture = clonePayload(); + }); + + it("registers the check_suite.completed handler", () => { + const harness = createHarness(); + + expect(harness.router.captured).toHaveLength(1); + expect(harness.router.captured[0]?.key).toBe("check_suite.completed"); + }); + + it("sorts multiple PR numbers into deterministic queue metadata", async () => { + const payload = clonePayload(); + payload.check_suite.pull_requests = [ + { ...payload.check_suite.pull_requests[0]!, number: 22 }, + { ...payload.check_suite.pull_requests[0]!, number: 5 }, + { ...payload.check_suite.pull_requests[0]!, number: 17 }, + ]; + + const harness = createHarness(); + await harness.router.captured[0]!.handler(makeEvent(payload)); + + expect(harness.queueCalls).toHaveLength(1); + expect(harness.queueCalls[0]?.context).toMatchObject({ + jobType: "ci-failure-analysis", + lane: "sync", + key: "octo-org/widget#5,17,22", + prNumber: 22, + eventName: "check_suite", + action: "completed", + }); + }); + + it("returns early when repository owner metadata is missing", async () => { + const payload = clonePayload(); + payload.repository.owner.login = undefined as never; + + const harness = createHarness(); + await harness.router.captured[0]!.handler(makeEvent(payload)); + + expect(harness.queueCalls).toHaveLength(0); + expect(harness.debugCalls).toHaveLength(0); + }); + + it("skips when the check suite has no pull requests", async () => { + const payload = clonePayload(); + payload.check_suite.pull_requests = []; + + const harness = createHarness(); + await harness.router.captured[0]!.handler(makeEvent(payload)); + + expect(harness.queueCalls).toHaveLength(0); + expect(harness.debugCalls).toContainEqual({ + bindings: { + deliveryId: "delivery-1", + headSha: payload.check_suite.head_sha, + }, + message: "No PRs in check_suite (fork?)", + }); + }); + + it("skips annotation when the head check runs have no failures", async () => { + const payload = clonePayload(); + const harness = createHarness({ + headChecksByRef: { + [payload.check_suite.head_sha]: [ + { + data: [ + { name: "build", conclusion: "success", status: "completed" }, + { name: "lint", conclusion: "neutral", status: "completed" }, + ], + }, + ], + }, + }); + + await harness.router.captured[0]!.handler(makeEvent(payload)); + + expect(harness.debugCalls).toContainEqual({ + bindings: { + deliveryId: "delivery-1", + prNumber: 17, + headSha: payload.check_suite.head_sha, + }, + message: "All checks pass, skipping CI annotation", + }); + expect(harness.octokit.rest.repos.listCommits).not.toHaveBeenCalled(); + expect(harness.octokit.rest.issues.createComment).not.toHaveBeenCalled(); + }); + + it("skips annotation when the PR payload has no base ref", async () => { + const payload = clonePayload(); + payload.check_suite.pull_requests = [ + { + ...payload.check_suite.pull_requests[0]!, + base: undefined as never, + }, + ]; + + const harness = createHarness({ + headChecksByRef: { + [payload.check_suite.head_sha]: [ + { + data: [ + { name: "build", conclusion: "failure", status: "completed" }, + ], + }, + ], + }, + }); + + await harness.router.captured[0]!.handler(makeEvent(payload)); + + expect(harness.debugCalls).toContainEqual({ + bindings: { + deliveryId: "delivery-1", + prNumber: 17, + }, + message: "No base ref available, skipping CI annotation", + }); + expect(harness.octokit.rest.repos.listCommits).not.toHaveBeenCalled(); + }); + + it("skips annotation when base commits produce no check data", async () => { + const payload = clonePayload(); + const harness = createHarness({ + headChecksByRef: { + [payload.check_suite.head_sha]: [ + { + data: [ + { name: "build", conclusion: "failure", status: "completed" }, + ], + }, + ], + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa: [{ data: [] }], + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb: [{ data: [] }], + }, + baseCommitsByRef: { + main: [ + { sha: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, + { sha: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }, + ], + }, + }); + + await harness.router.captured[0]!.handler(makeEvent(payload)); + + expect(harness.debugCalls).toContainEqual({ + bindings: { + deliveryId: "delivery-1", + prNumber: 17, + }, + message: "No base-branch check data, skipping CI annotation", + }); + expect(harness.octokit.rest.issues.createComment).not.toHaveBeenCalled(); + expect(harness.warnCalls).toHaveLength(0); + }); + + it("creates a comment whose body shows unrelated classification from matching base failures", async () => { + const payload = clonePayload(); + const marker = buildCIAnalysisMarker("octo-org", "widget", 17); + const harness = createHarness({ + headChecksByRef: { + [payload.check_suite.head_sha]: [ + { + data: [ + { name: "build", conclusion: "failure", status: "completed" }, + ], + }, + ], + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa: [ + { + data: [ + { name: "build", conclusion: "failure", status: "completed" }, + ], + }, + ], + }, + baseCommitsByRef: { + main: [{ sha: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }], + }, + commentsByPage: [[{ id: 41, body: "existing comment without hidden marker" }]], + }); + + await harness.router.captured[0]!.handler(makeEvent(payload)); + + expect(harness.octokit.rest.issues.updateComment).not.toHaveBeenCalled(); + expect(harness.octokit.rest.issues.createComment).toHaveBeenCalledTimes(1); + + const body = extractPostedBody(harness); + expect(body).toContain(marker); + expect(body.split(marker)).toHaveLength(2); + expect(body).toContain("**All 1 failure appear unrelated to this PR**"); + expect(body).toContain("- :white_check_mark: **build** [high confidence] — Also fails on aaaaaaa"); + expect(harness.debugCalls).toContainEqual({ + bindings: { + deliveryId: "delivery-1", + prNumber: 17, + }, + message: "Created new CI analysis comment", + }); + }); + + it("updates the existing marker comment when flakiness overrides the classification", async () => { + const payload = clonePayload(); + const marker = buildCIAnalysisMarker("octo-org", "widget", 17); + const harness = createHarness({ + headChecksByRef: { + [payload.check_suite.head_sha]: [ + { + data: [ + { name: "build", conclusion: "failure", status: "completed" }, + ], + }, + ], + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa: [ + { + data: [ + { name: "build", conclusion: "success", status: "completed" }, + ], + }, + ], + }, + baseCommitsByRef: { + main: [{ sha: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }], + }, + flakinessRows: [ + { check_name: "build", conclusion: "failure" }, + { check_name: "build", conclusion: "failure" }, + { check_name: "build", conclusion: "failure" }, + { check_name: "build", conclusion: "failure" }, + { check_name: "build", conclusion: "failure" }, + { check_name: "build", conclusion: "failure" }, + { check_name: "build", conclusion: "failure" }, + { check_name: "build", conclusion: "success" }, + { check_name: "build", conclusion: "success" }, + { check_name: "build", conclusion: "success" }, + { check_name: "build", conclusion: "success" }, + { check_name: "build", conclusion: "success" }, + { check_name: "build", conclusion: "success" }, + { check_name: "build", conclusion: "success" }, + { check_name: "build", conclusion: "success" }, + { check_name: "build", conclusion: "success" }, + { check_name: "build", conclusion: "success" }, + { check_name: "build", conclusion: "success" }, + { check_name: "build", conclusion: "success" }, + { check_name: "build", conclusion: "success" }, + ], + commentsByPage: [[{ id: 52, body: `${marker}\nold body` }]], + }); + + await harness.router.captured[0]!.handler(makeEvent(payload)); + + expect(harness.octokit.rest.issues.createComment).not.toHaveBeenCalled(); + expect(harness.octokit.rest.issues.updateComment).toHaveBeenCalledTimes(1); + expect(harness.octokit.rest.issues.updateComment).toHaveBeenCalledWith({ + owner: "octo-org", + repo: "widget", + comment_id: 52, + body: expect.any(String), + }); + + const body = extractPostedBody(harness); + expect(body).toContain(marker); + expect(body.split(marker)).toHaveLength(2); + expect(body).toContain("**All 1 failure appear unrelated to this PR**"); + expect(body).toContain("- :warning: **build** [medium confidence] — Historically flaky"); + expect(body).toContain("Failed 35% of last 20 runs"); + expect(harness.debugCalls).toContainEqual({ + bindings: { + deliveryId: "delivery-1", + prNumber: 17, + commentId: 52, + }, + message: "Updated existing CI analysis comment", + }); + }); + + it("warns and returns when head check listing hits a 403 permission error", async () => { + const payload = clonePayload(); + const forbiddenError = Object.assign(new Error("forbidden"), { status: 403 }); + const harness = createHarness({ + headChecksErrorByRef: { + [payload.check_suite.head_sha]: forbiddenError, + }, + }); + + await expect(harness.router.captured[0]!.handler(makeEvent(payload))).resolves.toBeUndefined(); + + expect(harness.warnCalls).toContainEqual({ + bindings: { + deliveryId: "delivery-1", + owner: "octo-org", + repo: "widget", + }, + message: "checks:read permission may be missing", + }); + expect(harness.debugCalls).toHaveLength(0); + expect(harness.octokit.rest.repos.listCommits).not.toHaveBeenCalled(); + expect(harness.octokit.rest.issues.createComment).not.toHaveBeenCalled(); + }); + + it("logs and skips annotation when fetching base commits fails", async () => { + const payload = clonePayload(); + const harness = createHarness({ + headChecksByRef: { + [payload.check_suite.head_sha]: [ + { + data: [ + { name: "build", conclusion: "failure", status: "completed" }, + ], + }, + ], + }, + baseCommitsErrorByRef: { + main: new Error("base branch unavailable"), + }, + }); + + await expect(harness.router.captured[0]!.handler(makeEvent(payload))).resolves.toBeUndefined(); + + expect(harness.debugCalls).toContainEqual({ + bindings: { + deliveryId: "delivery-1", + baseRef: "main", + }, + message: "Failed to fetch base branch commits, skipping CI annotation", + }); + expect(harness.warnCalls).toHaveLength(0); + expect(harness.octokit.rest.issues.createComment).not.toHaveBeenCalled(); + }); + + it("degrades individual base-commit check fetch failures to empty results", async () => { + const payload = clonePayload(); + const harness = createHarness({ + headChecksByRef: { + [payload.check_suite.head_sha]: [ + { + data: [ + { name: "build", conclusion: "failure", status: "completed" }, + ], + }, + ], + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb: [ + { + data: [ + { name: "build", conclusion: "failure", status: "completed" }, + ], + }, + ], + }, + headChecksErrorByRef: { + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa: new Error("base checks unavailable"), + }, + baseCommitsByRef: { + main: [ + { sha: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, + { sha: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }, + ], + }, + commentsByPage: [[]], + }); + + await expect(harness.router.captured[0]!.handler(makeEvent(payload))).resolves.toBeUndefined(); + + expect(harness.debugCalls).toContainEqual({ + bindings: { + deliveryId: "delivery-1", + sha: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + message: "Failed to fetch checks for base commit", + }); + expect(harness.warnCalls).toHaveLength(0); + expect(harness.octokit.rest.issues.createComment).toHaveBeenCalledTimes(1); + + const body = extractPostedBody(harness); + expect(body).toContain("**All 1 failure appear unrelated to this PR**"); + expect(body).toContain("Also fails on bbbbbbb"); + }); + + it("warns fail-open and avoids escaping the queued job on unexpected GitHub errors", async () => { + const payload = clonePayload(); + const harness = createHarness({ + headChecksErrorByRef: { + [payload.check_suite.head_sha]: new Error("network exploded"), + }, + }); + + await expect(harness.router.captured[0]!.handler(makeEvent(payload))).resolves.toBeUndefined(); + + expect(harness.warnCalls).toHaveLength(1); + expect(harness.warnCalls[0]?.message).toBe("CI failure analysis error (fail-open)"); + expect(harness.warnCalls[0]?.bindings).toMatchObject({ + deliveryId: "delivery-1", + }); + expect(harness.warnCalls[0]?.bindings.err).toBeDefined(); + expect(harness.octokit.rest.repos.listCommits).not.toHaveBeenCalled(); + expect(harness.octokit.rest.issues.createComment).not.toHaveBeenCalled(); + }); + + it("creates a new comment with possibly-pr-related evidence when comment scanning fails open", async () => { + const payload = clonePayload(); + const marker = buildCIAnalysisMarker("octo-org", "widget", 17); + const harness = createHarness({ + headChecksByRef: { + [payload.check_suite.head_sha]: [ + { + data: [ + { name: "build", conclusion: "failure", status: "completed" }, + ], + }, + ], + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa: [ + { + data: [ + { name: "build", conclusion: "success", status: "completed" }, + ], + }, + ], + }, + baseCommitsByRef: { + main: [{ sha: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }], + }, + listCommentsError: new Error("comments unavailable"), + flakinessRows: [], + }); + + await harness.router.captured[0]!.handler(makeEvent(payload)); + + expect(harness.octokit.rest.issues.listComments).toHaveBeenCalledTimes(1); + expect(harness.octokit.rest.issues.updateComment).not.toHaveBeenCalled(); + expect(harness.octokit.rest.issues.createComment).toHaveBeenCalledTimes(1); + + const body = extractPostedBody(harness); + expect(body).toContain(marker); + expect(body.split(marker)).toHaveLength(2); + expect(body).toContain("**0 of 1 failure appear unrelated to this PR**"); + expect(body).toContain("- :x: **build** [low confidence] — Passes on base branch"); + expect(harness.debugCalls).toContainEqual({ + bindings: { + deliveryId: "delivery-1", + prNumber: 17, + }, + message: "Failed to scan for existing CI comment, will create new", + }); + expect(harness.debugCalls).toContainEqual({ + bindings: { + deliveryId: "delivery-1", + prNumber: 17, + }, + message: "Created new CI analysis comment", + }); + }); +}); diff --git a/src/handlers/mention.test.ts b/src/handlers/mention.test.ts index 2b94e62a..c2088da4 100644 --- a/src/handlers/mention.test.ts +++ b/src/handlers/mention.test.ts @@ -4308,6 +4308,482 @@ describe("createMentionHandler write intent gating", () => { await workspaceFixture.cleanup(); }); + test("issue write-mode routes single-file fork output to a gist reply", async () => { + const handlers = new Map Promise>(); + const workspaceFixture = await createWorkspaceFixture( + "mention:\n enabled: true\nwrite:\n enabled: true\n", + ); + + const issueReplies: string[] = []; + const gistCalls: Array> = []; + const forkCalls: string[] = []; + const { logger, infoCalls } = createMockLogger(); + + const eventRouter: EventRouter = { + register: (eventKey, handler) => { + handlers.set(eventKey, handler); + }, + dispatch: async () => undefined, + }; + + const jobQueue: JobQueue = { + enqueue: async (_installationId: number, fn: (metadata: JobQueueRunMetadata) => Promise) => fn(createQueueRunMetadata()), + getQueueSize: () => 0, + getPendingCount: () => 0, + getActiveJobs: getEmptyActiveJobs, + }; + + const workspaceManager: WorkspaceManager = { + create: async (_installationId: number, options: CloneOptions) => { + await $`git -C ${workspaceFixture.dir} checkout ${options.ref}`.quiet(); + return { dir: workspaceFixture.dir, cleanup: async () => undefined }; + }, + cleanupStale: async () => 0, + }; + + const octokit = { + rest: { + reactions: { + createForPullRequestReviewComment: async () => ({ data: {} }), + createForIssueComment: async () => ({ data: {} }), + }, + issues: { + listComments: async () => ({ data: [] }), + createComment: async (params: { body: string }) => { + issueReplies.push(params.body); + return { data: {} }; + }, + }, + pulls: { + list: async () => ({ data: [] }), + get: async () => ({ data: {} }), + create: async () => { + throw new Error("should not create PR when gist routing wins"); + }, + createReplyForReviewComment: async () => ({ data: {} }), + }, + }, + }; + + createMentionHandler({ + eventRouter, + jobQueue, + workspaceManager, + githubApp: { + getAppSlug: () => "kodiai", + getInstallationOctokit: async () => octokit as never, + } as unknown as GitHubApp, + executor: { + execute: async (ctx: { workspace: { dir: string } }) => { + await Bun.write(join(ctx.workspace.dir, "README.md"), "base\nissue gist path\n"); + return { + conclusion: "success", + published: false, + costUsd: 0, + numTurns: 1, + durationMs: 1, + sessionId: "session-mention-gist", + }; + }, + } as never, + forkManager: { + enabled: true, + ensureFork: async () => { + forkCalls.push("ensureFork"); + return { forkOwner: "xbmc-bot", forkRepo: "repo" }; + }, + syncFork: async () => { + forkCalls.push("syncFork"); + }, + getBotPat: () => "bot-pat", + } as never, + gistPublisher: { + enabled: true, + createPatchGist: async (input: Record) => { + gistCalls.push(input); + return { htmlUrl: "https://gist.github.com/kodiai/gist-issue-1", id: "gist-issue-1" }; + }, + } as never, + telemetryStore: noopTelemetryStore, + logger, + }); + + const handler = handlers.get("issue_comment.created"); + expect(handler).toBeDefined(); + + await handler!( + buildIssueCommentMentionEvent({ + issueNumber: 80, + commentBody: "@kodiai apply: update the README wording", + }), + ); + + expect(forkCalls).toEqual(["ensureFork", "syncFork"]); + expect(gistCalls).toHaveLength(1); + expect(gistCalls[0]).toMatchObject({ + owner: "acme", + repo: "repo", + summary: "update the README wording", + }); + expect(String(gistCalls[0]?.patch)).toContain("issue gist path"); + expect(issueReplies).toHaveLength(1); + expect(issueReplies[0]).toContain("Patch gist: https://gist.github.com/kodiai/gist-issue-1"); + expect(issueReplies[0]).toContain("curl -sL https://gist.github.com/kodiai/gist-issue-1.patch | git apply"); + expect(issueReplies[0]).toContain("Files changed: README.md"); + expect( + infoCalls.some((entry) => entry.message === "Evidence bundle" && entry.bindings.outcome === "created-gist"), + ).toBeTrue(); + + await workspaceFixture.cleanup(); + }); + + test("issue write-mode falls back from fork PR failure to a gist reply", async () => { + const handlers = new Map Promise>(); + const workspaceFixture = await createWorkspaceFixture( + "mention:\n enabled: true\nwrite:\n enabled: true\n", + ); + await $`git -C ${workspaceFixture.dir} checkout main`.quiet(); + await Bun.write(join(workspaceFixture.dir, "CHANGELOG.md"), "seed changelog\n"); + await Bun.write(join(workspaceFixture.dir, "NOTES.md"), "seed notes\n"); + await Bun.write(join(workspaceFixture.dir, "TODO.md"), "seed todo\n"); + await $`git -C ${workspaceFixture.dir} add CHANGELOG.md NOTES.md TODO.md`.quiet(); + await $`git -C ${workspaceFixture.dir} commit -m "seed tracked files for fork fallback"`.quiet(); + await $`git -C ${workspaceFixture.dir} push origin main`.quiet(); + + const issueReplies: string[] = []; + const gistCalls: Array> = []; + const { logger, warnCalls, infoCalls } = createMockLogger(); + + const eventRouter: EventRouter = { + register: (eventKey, handler) => { + handlers.set(eventKey, handler); + }, + dispatch: async () => undefined, + }; + + const jobQueue: JobQueue = { + enqueue: async (_installationId: number, fn: (metadata: JobQueueRunMetadata) => Promise) => fn(createQueueRunMetadata()), + getQueueSize: () => 0, + getPendingCount: () => 0, + getActiveJobs: getEmptyActiveJobs, + }; + + const workspaceManager: WorkspaceManager = { + create: async (_installationId: number, options: CloneOptions) => { + await $`git -C ${workspaceFixture.dir} checkout ${options.ref}`.quiet(); + return { dir: workspaceFixture.dir, cleanup: async () => undefined }; + }, + cleanupStale: async () => 0, + }; + + const octokit = { + rest: { + reactions: { + createForPullRequestReviewComment: async () => ({ data: {} }), + createForIssueComment: async () => ({ data: {} }), + }, + issues: { + listComments: async () => ({ data: [] }), + createComment: async (params: { body: string }) => { + issueReplies.push(params.body); + return { data: {} }; + }, + }, + pulls: { + list: async () => ({ data: [] }), + get: async () => ({ data: {} }), + create: async () => { + throw new Error("should not reach PR creation after origin fork assertion fails"); + }, + createReplyForReviewComment: async () => ({ data: {} }), + }, + }, + }; + + createMentionHandler({ + eventRouter, + jobQueue, + workspaceManager, + githubApp: { + getAppSlug: () => "kodiai", + getInstallationOctokit: async () => octokit as never, + } as unknown as GitHubApp, + executor: { + execute: async (ctx: { workspace: { dir: string } }) => { + await Bun.write(join(ctx.workspace.dir, "README.md"), "base\nfallback gist readme\n"); + await Bun.write(join(ctx.workspace.dir, "CHANGELOG.md"), "fallback gist changelog\n"); + await Bun.write(join(ctx.workspace.dir, "NOTES.md"), "fallback gist notes\n"); + await Bun.write(join(ctx.workspace.dir, "TODO.md"), "fallback gist todo\n"); + return { + conclusion: "success", + published: false, + costUsd: 0, + numTurns: 1, + durationMs: 1, + sessionId: "session-mention-fallback-gist", + }; + }, + } as never, + forkManager: { + enabled: true, + ensureFork: async () => ({ forkOwner: "xbmc-bot", forkRepo: "repo" }), + syncFork: async () => undefined, + getBotPat: () => "bot-pat", + } as never, + gistPublisher: { + enabled: true, + createPatchGist: async (input: Record) => { + gistCalls.push(input); + return { htmlUrl: "https://gist.github.com/kodiai/fallback-issue-1", id: "fallback-issue-1" }; + }, + } as never, + telemetryStore: noopTelemetryStore, + logger, + }); + + const handler = handlers.get("issue_comment.created"); + expect(handler).toBeDefined(); + + await handler!( + buildIssueCommentMentionEvent({ + issueNumber: 81, + commentBody: "@kodiai apply: update the repo files", + }), + ); + + expect(gistCalls).toHaveLength(1); + expect(String(gistCalls[0]?.patch)).toContain("fallback gist readme"); + expect(issueReplies).toHaveLength(1); + expect(issueReplies[0]).toContain("Could not create a PR from the fork, but here is the patch as a gist:"); + expect(issueReplies[0]).toContain("Patch gist: https://gist.github.com/kodiai/fallback-issue-1"); + expect( + warnCalls.some((entry) => entry.message === "Fork-based PR creation failed; falling back to gist"), + ).toBeTrue(); + expect( + infoCalls.some((entry) => entry.message === "Evidence bundle" && entry.bindings.outcome === "fallback-gist"), + ).toBeTrue(); + + await workspaceFixture.cleanup(); + }); + + test("issue write-mode without fork helpers falls back to legacy PR behavior", async () => { + const handlers = new Map Promise>(); + const workspaceFixture = await createWorkspaceFixture( + "mention:\n enabled: true\nwrite:\n enabled: true\n", + ); + + const issueReplies: string[] = []; + let createdPrHead: string | undefined; + const { logger, warnCalls } = createMockLogger(); + + const eventRouter: EventRouter = { + register: (eventKey, handler) => { + handlers.set(eventKey, handler); + }, + dispatch: async () => undefined, + }; + + const jobQueue: JobQueue = { + enqueue: async (_installationId: number, fn: (metadata: JobQueueRunMetadata) => Promise) => fn(createQueueRunMetadata()), + getQueueSize: () => 0, + getPendingCount: () => 0, + getActiveJobs: getEmptyActiveJobs, + }; + + const workspaceManager: WorkspaceManager = { + create: async (_installationId: number, options: CloneOptions) => { + await $`git -C ${workspaceFixture.dir} checkout ${options.ref}`.quiet(); + return { dir: workspaceFixture.dir, cleanup: async () => undefined }; + }, + cleanupStale: async () => 0, + }; + + const octokit = { + rest: { + reactions: { + createForPullRequestReviewComment: async () => ({ data: {} }), + createForIssueComment: async () => ({ data: {} }), + }, + issues: { + listComments: async () => ({ data: [] }), + createComment: async (params: { body: string }) => { + issueReplies.push(params.body); + return { data: {} }; + }, + }, + pulls: { + list: async () => ({ data: [] }), + get: async () => ({ data: {} }), + create: async (params: { head: string }) => { + createdPrHead = params.head; + return { data: { html_url: "https://example.com/pr/legacy-issue-path" } }; + }, + createReplyForReviewComment: async () => ({ data: {} }), + }, + }, + }; + + createMentionHandler({ + eventRouter, + jobQueue, + workspaceManager, + githubApp: { + getAppSlug: () => "kodiai", + getInstallationOctokit: async () => octokit as never, + } as unknown as GitHubApp, + executor: { + execute: async (ctx: { workspace: { dir: string } }) => { + await Bun.write(join(ctx.workspace.dir, "README.md"), "base\nlegacy path update\n"); + return { + conclusion: "success", + published: false, + costUsd: 0, + numTurns: 1, + durationMs: 1, + sessionId: "session-mention-legacy-path", + }; + }, + } as never, + telemetryStore: noopTelemetryStore, + logger, + }); + + const handler = handlers.get("issue_comment.created"); + expect(handler).toBeDefined(); + + await handler!( + buildIssueCommentMentionEvent({ + issueNumber: 82, + commentBody: "@kodiai apply: update the README wording", + }), + ); + + expect(createdPrHead).toBeDefined(); + expect(createdPrHead).not.toContain(":"); + expect(issueReplies).toHaveLength(1); + expect(issueReplies[0]).toContain("status: success"); + expect(issueReplies[0]).toContain("pr_url: https://example.com/pr/legacy-issue-path"); + expect( + warnCalls.some((entry) => entry.message === "Write-mode active without BOT_USER_PAT; using legacy direct-push behavior"), + ).toBeTrue(); + + await workspaceFixture.cleanup(); + }); + + test("issue gist replies are blocked by the canonical outgoing secret scan before publish", async () => { + const handlers = new Map Promise>(); + const workspaceFixture = await createWorkspaceFixture( + "mention:\n enabled: true\nwrite:\n enabled: true\n", + ); + + const blockedToken = `ghp_${"A".repeat(36)}`; + const issueReplies: string[] = []; + const { logger, warnCalls } = createMockLogger(); + + const eventRouter: EventRouter = { + register: (eventKey, handler) => { + handlers.set(eventKey, handler); + }, + dispatch: async () => undefined, + }; + + const jobQueue: JobQueue = { + enqueue: async (_installationId: number, fn: (metadata: JobQueueRunMetadata) => Promise) => fn(createQueueRunMetadata()), + getQueueSize: () => 0, + getPendingCount: () => 0, + getActiveJobs: getEmptyActiveJobs, + }; + + const workspaceManager: WorkspaceManager = { + create: async (_installationId: number, options: CloneOptions) => { + await $`git -C ${workspaceFixture.dir} checkout ${options.ref}`.quiet(); + return { dir: workspaceFixture.dir, cleanup: async () => undefined }; + }, + cleanupStale: async () => 0, + }; + + const octokit = { + rest: { + reactions: { + createForPullRequestReviewComment: async () => ({ data: {} }), + createForIssueComment: async () => ({ data: {} }), + }, + issues: { + listComments: async () => ({ data: [] }), + createComment: async (params: { body: string }) => { + issueReplies.push(params.body); + return { data: {} }; + }, + }, + pulls: { + list: async () => ({ data: [] }), + get: async () => ({ data: {} }), + create: async () => { + throw new Error("should stay on gist path"); + }, + createReplyForReviewComment: async () => ({ data: {} }), + }, + }, + }; + + createMentionHandler({ + eventRouter, + jobQueue, + workspaceManager, + githubApp: { + getAppSlug: () => "kodiai", + getInstallationOctokit: async () => octokit as never, + } as unknown as GitHubApp, + executor: { + execute: async (ctx: { workspace: { dir: string } }) => { + await Bun.write(join(ctx.workspace.dir, "README.md"), "base\nsecret scrub gist path\n"); + return { + conclusion: "success", + published: false, + costUsd: 0, + numTurns: 1, + durationMs: 1, + sessionId: "session-mention-secret-scan", + }; + }, + } as never, + forkManager: { + enabled: true, + ensureFork: async () => ({ forkOwner: "xbmc-bot", forkRepo: "repo" }), + syncFork: async () => undefined, + getBotPat: () => "bot-pat", + } as never, + gistPublisher: { + enabled: true, + createPatchGist: async () => ({ + htmlUrl: `https://gist.github.com/kodiai/${blockedToken}`, + id: "gist-secret-block", + }), + } as never, + telemetryStore: noopTelemetryStore, + logger, + }); + + const handler = handlers.get("issue_comment.created"); + expect(handler).toBeDefined(); + + await handler!( + buildIssueCommentMentionEvent({ + issueNumber: 83, + commentBody: "@kodiai apply: update the README wording", + }), + ); + + expect(issueReplies).toEqual(["[Response blocked by security policy]"]); + expect(issueReplies.join("\n")).not.toContain(blockedToken); + expect(warnCalls).toContainEqual({ + bindings: { matchedPattern: "github-pat" }, + message: "Outgoing secret scan blocked mention reply publish", + }); + + await workspaceFixture.cleanup(); + }); + test("success reply without machine-checkable markers is invalid (negative regression)", () => { // This test proves that a success reply lacking deterministic status markers // would fail contract assertions. If the envelope builder ever regresses to diff --git a/src/handlers/mention.ts b/src/handlers/mention.ts index f706ff5e..17f0c13e 100644 --- a/src/handlers/mention.ts +++ b/src/handlers/mention.ts @@ -55,7 +55,7 @@ import { postOrUpdateErrorComment, } from "../lib/errors.ts"; import { wrapInDetails } from "../lib/formatting.ts"; -import { sanitizeOutgoingMentions } from "../lib/sanitizer.ts"; +import { sanitizeOutgoingMentions, scanOutgoingForSecrets } from "../lib/sanitizer.ts"; import { validateIssue, generateGuidanceComment, generateLabelRecommendation, generateGenericNudge } from "../triage/triage-agent.ts"; import { runGuardrailPipeline } from "../lib/guardrail/pipeline.ts"; import { createGuardrailAuditStore } from "../lib/guardrail/audit-store.ts"; @@ -1256,6 +1256,15 @@ export function createMentionHandler(deps: { } } + const scanResult = scanOutgoingForSecrets(sanitizedBody); + if (scanResult.blocked) { + logger.warn( + { matchedPattern: scanResult.matchedPattern }, + "Outgoing secret scan blocked mention reply publish", + ); + sanitizedBody = "[Response blocked by security policy]"; + } + // Prefer replying in-thread for inline review comment mentions. if (mention.surface === "pr_review_comment" && mention.prNumber !== undefined) { try { diff --git a/src/handlers/review.test.ts b/src/handlers/review.test.ts index fd018bca..e931e278 100644 --- a/src/handlers/review.test.ts +++ b/src/handlers/review.test.ts @@ -1909,8 +1909,6 @@ describe("createReviewHandler review_requested idempotency", () => { cleanupStale: async () => 0, }; - let nextIssueCommentId = 70; - const issueComments: Array<{ id: number; body: string }> = []; const octokit = { rest: { pulls: { @@ -1921,7 +1919,6 @@ describe("createReviewHandler review_requested idempotency", () => { body: review.body ?? null, })), }), - listCommits: async () => ({ data: [] }), createReview: async ({ body }: { body?: string }) => { approveCount++; createdReviews.push({ body: body ?? null }); @@ -1932,24 +1929,7 @@ describe("createReviewHandler review_requested idempotency", () => { createForIssue: async () => ({ data: {} }), }, issues: { - listComments: async () => ({ - data: issueComments.map((comment) => ({ id: comment.id, body: comment.body })), - }), - createComment: async ({ body }: { body: string }) => { - const comment = { id: nextIssueCommentId++, body }; - issueComments.push(comment); - return { data: comment }; - }, - updateComment: async ({ comment_id, body }: { comment_id: number; body: string }) => { - const existing = issueComments.find((comment) => comment.id === comment_id); - if (existing) { - existing.body = body; - } - return { data: {} }; - }, - }, - search: { - issuesAndPullRequests: async () => ({ data: { total_count: 4 } }), + listComments: async () => ({ data: [] }), }, }, }; @@ -1966,7 +1946,7 @@ describe("createReviewHandler review_requested idempotency", () => { getInstallationToken: async () => "token", } as unknown as GitHubApp, executor: { - execute: async () => { + execute: async (context: { reviewOutputKey?: string }) => { executeCount++; return { conclusion: "success", @@ -1995,7 +1975,6 @@ describe("createReviewHandler review_requested idempotency", () => { expect(executeCount).toBe(1); expect(approveCount).toBe(1); - expect(issueComments).toHaveLength(1); const expectedReviewOutputKey = buildReviewOutputKey({ installationId: 42, @@ -2008,146 +1987,18 @@ describe("createReviewHandler review_requested idempotency", () => { }); const marker = buildReviewOutputMarker(expectedReviewOutputKey); - const finalIssueBody = issueComments[0]!.body; - - expect(createdReviews[0]?.body).toBe(marker); - expect(finalIssueBody).toContain("kodiai response"); - expect(finalIssueBody).toContain("Decision: APPROVE"); - expect(finalIssueBody).toContain("Issues: none"); - expect(finalIssueBody).toContain("- Review prompt covered 1 changed file."); - expect(finalIssueBody).not.toContain("Merge Confidence:"); - expect(finalIssueBody).toContain("Review Details"); - expect(finalIssueBody).toContain(marker); - expect(finalIssueBody).not.toContain("captured before publication completed"); - expect(extractReviewOutputKey(finalIssueBody)).toBe(expectedReviewOutputKey); - }); - - test("skips duplicate clean approvals when only the approval review marker survived", async () => { - const handlers = new Map Promise>(); - const workspaceFixture = await createWorkspaceFixture({ autoApprove: true }); - const { logger, entries } = createCaptureLogger(); - - const createdReviews: Array<{ body?: string | null }> = []; - let executeCount = 0; - let approveCount = 0; - let createCommentCalls = 0; - - const eventRouter: EventRouter = { - register: (eventKey, handler) => { - handlers.set(eventKey, handler); - }, - dispatch: async () => undefined, - }; - - const jobQueue: JobQueue = { - enqueue: async (_installationId: number, fn: (metadata: JobQueueRunMetadata) => Promise) => fn(createQueueRunMetadata()), - getQueueSize: () => 0, - getPendingCount: () => 0, - getActiveJobs: getEmptyActiveJobs, - }; - - const workspaceManager: WorkspaceManager = { - create: async () => ({ - dir: workspaceFixture.dir, - cleanup: async () => undefined, - }), - cleanupStale: async () => 0, - }; - - const octokit = { - rest: { - pulls: { - listReviewComments: async () => ({ data: [] }), - listReviews: async () => ({ - data: createdReviews.map((review, index) => ({ - id: index + 1, - body: review.body ?? null, - })), - }), - listCommits: async () => ({ data: [] }), - createReview: async ({ body }: { body?: string }) => { - approveCount++; - createdReviews.push({ body: body ?? null }); - return { data: {} }; - }, - }, - reactions: { - createForIssue: async () => ({ data: {} }), - }, - issues: { - listComments: async () => ({ data: [] }), - createComment: async () => { - createCommentCalls += 1; - throw new Error("issue comment publication failed"); - }, - }, - }, - }; - createReviewHandler({ - eventRouter, - jobQueue, - workspaceManager, - githubApp: { - getAppSlug: () => "kodiai", - getInstallationOctokit: async () => octokit as never, - initialize: async () => undefined, - checkConnectivity: async () => true, - getInstallationToken: async () => "token", - } as unknown as GitHubApp, - executor: { - execute: async () => { - executeCount++; - return { - conclusion: "success", - costUsd: 0, - numTurns: 1, - durationMs: 1, - sessionId: "session-clean-review-marker-only", - }; - }, - } as never, - telemetryStore: noopTelemetryStore, - logger: logger as never, - }); - - const handler = handlers.get("pull_request.review_requested"); - expect(handler).toBeDefined(); - - const event = buildReviewRequestedEvent({ - requested_reviewer: { login: "kodiai[bot]" }, - }); - - await handler!(event); - await handler!(event); - - await workspaceFixture.cleanup(); - - const expectedReviewOutputKey = buildReviewOutputKey({ - installationId: 42, - owner: "acme", - repo: "repo", - prNumber: 101, - action: "review_requested", - deliveryId: "delivery-123", - headSha: "abcdef1234567890", - }); - - expect(executeCount).toBe(1); - expect(approveCount).toBe(1); - expect(createCommentCalls).toBeGreaterThanOrEqual(1); - expect(createdReviews[0]?.body).toBe(buildReviewOutputMarker(expectedReviewOutputKey)); - expect( - entries.some((entry) => - entry.data?.gate === "review-output-idempotency" - && entry.data?.gateResult === "skipped" - && entry.data?.skipReason === "already-published" - && entry.data?.existingLocation === "review" - ), - ).toBe(true); + expect(createdReviews[0]?.body ?? "").toContain("Decision: APPROVE"); + expect(createdReviews[0]?.body ?? "").toContain("Issues: none"); + expect(createdReviews[0]?.body ?? "").toContain("Evidence:"); + expect(createdReviews[0]?.body ?? "").toContain("- Review prompt covered 1 changed file."); + expect(createdReviews[0]?.body ?? "").not.toContain("Merge Confidence:"); + expect(extractReviewOutputKey(createdReviews[0]?.body)).toBe(expectedReviewOutputKey); + // Ensure marker format stays stable inside the visible approval body. + expect(createdReviews[0]?.body ?? "").toContain(marker); }); - test("auto-approve includes dep-bump merge confidence inside the shared approval comment", async () => { + test("auto-approve includes dep-bump merge confidence inside the shared approval body", async () => { const handlers = new Map Promise>(); const workspaceFixture = await createWorkspaceFixture({ autoApprove: true }); @@ -2164,8 +2015,6 @@ describe("createReviewHandler review_requested idempotency", () => { let approveCount = 0; const createdReviews: Array<{ body?: string }> = []; - let nextIssueCommentId = 170; - const issueComments: Array<{ id: number; body: string }> = []; const previousFetch = globalThis.fetch; globalThis.fetch = mock(async () => new Response("Not Found", { status: 404 })) as unknown as typeof globalThis.fetch; @@ -2207,21 +2056,7 @@ describe("createReviewHandler review_requested idempotency", () => { createForIssue: async () => ({ data: {} }), }, issues: { - listComments: async () => ({ - data: issueComments.map((comment) => ({ id: comment.id, body: comment.body })), - }), - createComment: async ({ body }: { body: string }) => { - const comment = { id: nextIssueCommentId++, body }; - issueComments.push(comment); - return { data: comment }; - }, - updateComment: async ({ comment_id, body }: { comment_id: number; body: string }) => { - const existing = issueComments.find((comment) => comment.id === comment_id); - if (existing) { - existing.body = body; - } - return { data: {} }; - }, + listComments: async () => ({ data: [] }), }, securityAdvisories: { listGlobalAdvisories: async () => ({ data: [] }), @@ -2229,9 +2064,6 @@ describe("createReviewHandler review_requested idempotency", () => { repos: { listReleases: async () => ({ data: [] }), }, - search: { - issuesAndPullRequests: async () => ({ data: { total_count: 4 } }), - }, }, }; @@ -2296,25 +2128,12 @@ describe("createReviewHandler review_requested idempotency", () => { } expect(approveCount).toBe(1); - expect(issueComments).toHaveLength(1); - expect(createdReviews[0]?.body).toBe( - buildReviewOutputMarker( - buildReviewOutputKey({ - installationId: 42, - owner: "acme", - repo: "repo", - prNumber: 101, - action: "review_requested", - deliveryId: "delivery-123", - headSha: "abcdef1234567890", - }), - ), - ); - expect(issueComments[0]!.body).toContain("Decision: APPROVE"); - expect(issueComments[0]!.body).toContain("Issues: none"); - expect(issueComments[0]!.body).toContain("- Review prompt covered 2 changed files."); - expect(issueComments[0]!.body).toContain("Merge Confidence: High"); - expect(extractReviewOutputKey(issueComments[0]!.body)).toBe( + expect(createdReviews[0]?.body ?? "").toContain("Decision: APPROVE"); + expect(createdReviews[0]?.body ?? "").toContain("Issues: none"); + expect(createdReviews[0]?.body ?? "").toContain("Evidence:"); + expect(createdReviews[0]?.body ?? "").toContain("- Review prompt covered 2 changed files."); + expect(createdReviews[0]?.body ?? "").toContain("Merge Confidence: High"); + expect(extractReviewOutputKey(createdReviews[0]?.body)).toBe( buildReviewOutputKey({ installationId: 42, owner: "acme", @@ -4582,7 +4401,6 @@ describe("createReviewHandler diff collection resilience", () => { let executeCount = 0; let capturedPrompt = ""; - let capturedGitDiffInspectionAvailable: boolean | undefined; const eventRouter: EventRouter = { register: (eventKey, handler) => { @@ -4630,10 +4448,9 @@ describe("createReviewHandler diff collection resilience", () => { getInstallationOctokit: async () => octokit as never, } as unknown as GitHubApp, executor: { - execute: async (context: { prompt: string; gitDiffInspectionAvailable?: boolean }) => { + execute: async (context: { prompt: string }) => { executeCount++; capturedPrompt = context.prompt; - capturedGitDiffInspectionAvailable = context.gitDiffInspectionAvailable; return { conclusion: "success", published: false, @@ -4670,10 +4487,6 @@ describe("createReviewHandler diff collection resilience", () => { expect(executeCount).toBe(1); expect(capturedPrompt).toContain("README.md"); - expect(capturedPrompt).toContain("Full local merge-base diff access is unavailable in this run."); - expect(capturedPrompt).not.toContain("To see the full diff: Bash(git diff origin/main...HEAD)"); - expect(capturedPrompt).not.toContain("To see changed files with stats: Bash(git log origin/main..HEAD --stat)"); - expect(capturedGitDiffInspectionAvailable).toBe(false); } finally { await workspaceFixture.cleanup(); } @@ -5143,11 +4956,7 @@ describe("createReviewHandler finding extraction", () => { const deletedCommentIds: number[] = []; let listCommentsCalls = 0; let createCommentCalls = 0; - let updateCommentCalls = 0; let detailsCommentBody: string | undefined; - let initialDetailsCommentBody: string | undefined; - let nextIssueCommentId = 500; - const issueComments: Array<{ id: number; body: string }> = []; const eventRouter: EventRouter = { register: (eventKey, handler) => { @@ -5188,7 +4997,6 @@ describe("createReviewHandler finding extraction", () => { headSha: "abcdef1234567890", }); const marker = buildReviewOutputMarker(reviewOutputKey); - const reviewDetailsMarker = ``; const octokit = { rest: { @@ -5251,28 +5059,14 @@ describe("createReviewHandler finding extraction", () => { issues: { listComments: async () => { listCommentsCalls += 1; - return { - data: issueComments.map((comment) => ({ id: comment.id, body: comment.body })), - }; + return { data: [] }; }, createComment: async (params: { body: string }) => { createCommentCalls += 1; - const comment = { id: nextIssueCommentId++, body: params.body }; - issueComments.push(comment); - initialDetailsCommentBody = params.body; detailsCommentBody = params.body; - await new Promise((resolve) => setTimeout(resolve, 25)); - return { data: comment }; - }, - updateComment: async ({ comment_id, body }: { comment_id: number; body: string }) => { - updateCommentCalls += 1; - const existing = issueComments.find((comment) => comment.id === comment_id); - if (existing) { - existing.body = body; - } - detailsCommentBody = body; return { data: {} }; }, + updateComment: async () => ({ data: {} }), }, reactions: { createForIssue: async () => ({ data: {} }), @@ -5321,10 +5115,7 @@ describe("createReviewHandler finding extraction", () => { expect(deletedCommentIds).toEqual([41, 42]); expect(listCommentsCalls).toBeGreaterThanOrEqual(1); expect(createCommentCalls).toBe(1); - expect(updateCommentCalls).toBe(1); - expect(initialDetailsCommentBody).toBeDefined(); expect(detailsCommentBody).toContain("Review Details"); - expect(detailsCommentBody).toContain(reviewDetailsMarker); expect(detailsCommentBody).toContain("Files reviewed:"); expect(detailsCommentBody).toMatch(/Lines changed: \+\d+ -\d+/); expect(detailsCommentBody).toMatch(/Findings: \d+ critical, \d+ major, \d+ medium, \d+ minor/); @@ -5337,17 +5128,11 @@ describe("createReviewHandler finding extraction", () => { expect(detailsCommentBody).toContain("executor handoff: 50ms"); expect(detailsCommentBody).toContain("remote runtime: 500ms"); expect(detailsCommentBody).toContain("publication:"); - expect(detailsCommentBody).not.toContain("captured before publication completed"); - expect(detailsCommentBody).not.toContain("degraded:"); + expect(detailsCommentBody).toContain("degraded:"); expect(detailsCommentBody).not.toContain("Suppressions applied:"); expect(detailsCommentBody).not.toContain("Estimated review time saved:"); expect(detailsCommentBody).not.toContain("Low Confidence Findings"); - const initialCompletedAt = initialDetailsCommentBody?.match(/- Review completed: (.+)/)?.[1]; - const finalCompletedAt = detailsCommentBody?.match(/- Review completed: (.+)/)?.[1]; - expect(initialCompletedAt).toBeDefined(); - expect(finalCompletedAt).toBe(initialCompletedAt); - const detailsAttemptLog = entries.find((entry) => entry.data?.gate === "review-details-output" && entry.data?.gateResult === "attempt" ); @@ -9911,106 +9696,6 @@ describe("createReviewHandler timeout resilience", () => { }); }); -describe("createReviewHandler execution failure publication", () => { - test("posts a visible turn-budget fallback when review execution ends with error_max_turns", async () => { - const handlers = new Map Promise>(); - const workspaceFixture = await createWorkspaceFixture(); - - const createdCommentBodies: string[] = []; - - const eventRouter: EventRouter = { - register: (eventKey, handler) => { - handlers.set(eventKey, handler); - }, - dispatch: async () => undefined, - }; - - const jobQueue: JobQueue = { - enqueue: async (_installationId: number, fn: (metadata: JobQueueRunMetadata) => Promise) => fn(createQueueRunMetadata()), - getQueueSize: () => 0, - getPendingCount: () => 0, - getActiveJobs: getEmptyActiveJobs, - }; - - const workspaceManager: WorkspaceManager = { - create: async () => ({ - dir: workspaceFixture.dir, - cleanup: async () => undefined, - }), - cleanupStale: async () => 0, - }; - - const octokit = { - rest: { - pulls: { - listReviewComments: async () => ({ data: [] }), - listReviews: async () => ({ data: [] }), - listCommits: async () => ({ data: [] }), - }, - issues: { - listComments: async () => ({ data: [] }), - createComment: async (params: { body: string }) => { - createdCommentBodies.push(params.body); - return { data: { id: 901 } }; - }, - updateComment: async () => ({ data: {} }), - }, - reactions: { - createForIssue: async () => ({ data: {} }), - }, - }, - }; - - createReviewHandler({ - eventRouter, - jobQueue, - workspaceManager, - githubApp: { - getAppSlug: () => "kodiai", - getInstallationOctokit: async () => octokit as never, - } as unknown as GitHubApp, - executor: { - execute: async () => ({ - conclusion: "failure", - published: false, - failureSubtype: "error_max_turns", - stopReason: "tool_use", - costUsd: 0, - numTurns: 26, - durationMs: 1, - sessionId: "session-review-max-turns", - model: "test-model", - inputTokens: 0, - outputTokens: 0, - cacheReadTokens: 0, - cacheCreationTokens: 0, - errorMessage: undefined, - }), - } as never, - telemetryStore: noopTelemetryStore, - logger: createNoopLogger(), - }); - - const handler = handlers.get("pull_request.review_requested"); - expect(handler).toBeDefined(); - - try { - await handler!( - buildReviewRequestedEvent({ - requested_reviewer: { login: "kodiai[bot]" }, - }), - ); - - expect(createdCommentBodies).toHaveLength(1); - expect(createdCommentBodies[0]).toContain("Kodiai ran out of steps while reviewing this PR"); - expect(createdCommentBodies[0]).toContain("Stop reason: tool_use"); - expect(createdCommentBodies[0]).toContain("Failure subtype: error_max_turns"); - } finally { - await workspaceFixture.cleanup(); - } - }); -}); - describe("createReviewHandler author-tier search cache integration", () => { async function runAuthorTierScenario(params: { eventIds: [string, string]; @@ -12078,161 +11763,6 @@ describe("createReviewHandler Review Details phase timing publication", () => { await workspaceFixture.cleanup(); }); - test("rechecks publish rights before the finalized Review Details timing refresh", async () => { - const handlers = new Map Promise>(); - const workspaceFixture = await createWorkspaceFixture(); - const { logger, entries } = createCaptureLogger(); - - let issueCommentListCalls = 0; - let updateCommentCalls = 0; - const updatedBodies: string[] = []; - const completedAttemptIds: string[] = []; - - const reviewOutputKey = buildReviewOutputKey({ - installationId: 42, - owner: "acme", - repo: "repo", - prNumber: 101, - action: "review_requested", - deliveryId: "delivery-123", - headSha: "abcdef1234567890", - }); - - const summaryBody = [ - "
", - "Review summary", - "", - "No inline findings were published.", - "", - "
", - "", - buildReviewOutputMarker(reviewOutputKey), - ].join("\n"); - - const eventRouter: EventRouter = { - register: (eventKey, handler) => { - handlers.set(eventKey, handler); - }, - dispatch: async () => undefined, - }; - - const jobQueue: JobQueue = { - enqueue: async ( - _installationId: number, - fn: (metadata: JobQueueRunMetadata) => Promise, - ) => fn(createQueueRunMetadata()), - getQueueSize: () => 0, - getPendingCount: () => 0, - getActiveJobs: getEmptyActiveJobs, - }; - - const workspaceManager: WorkspaceManager = { - create: async () => ({ - dir: workspaceFixture.dir, - cleanup: async () => undefined, - }), - cleanupStale: async () => 0, - }; - - const octokit = { - rest: { - pulls: { - listReviewComments: async () => ({ data: [] }), - listReviews: async () => ({ data: [] }), - listCommits: async () => ({ data: [] }), - }, - issues: { - listComments: async () => { - issueCommentListCalls += 1; - return issueCommentListCalls === 1 - ? { data: [] } - : { data: [{ id: 77, body: updatedBodies.at(-1) ?? summaryBody }] }; - }, - createComment: async () => { - throw new Error("standalone fallback should not run"); - }, - updateComment: async (params: { body: string }) => { - updateCommentCalls += 1; - updatedBodies.push(params.body); - return { data: {} }; - }, - }, - reactions: { - createForIssue: async () => ({ data: {} }), - }, - search: { - issuesAndPullRequests: async () => ({ data: { total_count: 4 } }), - }, - }, - }; - - createReviewHandler({ - eventRouter, - jobQueue, - workspaceManager, - githubApp: { - getAppSlug: () => "kodiai", - getInstallationOctokit: async () => octokit as never, - } as unknown as GitHubApp, - executor: { - execute: async () => ({ - conclusion: "success", - published: true, - costUsd: 0, - numTurns: 1, - durationMs: 1, - sessionId: "session-review-details-finalized-recheck", - executorPhaseTimings: [ - { name: "executor handoff", status: "completed", durationMs: 50 }, - { name: "remote runtime", status: "completed", durationMs: 500 }, - ], - }), - } as never, - telemetryStore: noopTelemetryStore, - reviewWorkCoordinator: { - claim: (claim: Record) => ({ - attemptId: "attempt-review-details-finalized-recheck-1", - familyKey: claim.familyKey as string, - source: claim.source as "automatic-review", - lane: claim.lane as "review", - deliveryId: claim.deliveryId as string, - phase: claim.phase as "claimed", - claimedAtMs: 100, - lastProgressAtMs: 100, - }), - canPublish: () => updateCommentCalls === 0, - setPhase: () => null, - getSnapshot: () => null, - release: () => undefined, - complete: (attemptId: string) => { - completedAttemptIds.push(attemptId); - }, - } as never, - logger: logger as never, - }); - - const handler = handlers.get("pull_request.review_requested"); - expect(handler).toBeDefined(); - - await handler!( - buildReviewRequestedEvent({ - requested_reviewer: { login: "kodiai[bot]" }, - }), - ); - - expect(issueCommentListCalls).toBeGreaterThanOrEqual(1); - expect(updateCommentCalls).toBe(1); - expect(updatedBodies[0]).toContain("captured before publication completed"); - expect( - entries.some((entry) => - entry.message === "Skipping finalized Review Details timing update because publish rights were superseded" - ), - ).toBeTrue(); - expect(completedAttemptIds).toEqual(["attempt-review-details-finalized-recheck-1"]); - - await workspaceFixture.cleanup(); - }); - test("falls back to standalone Review Details timings when append cannot find the summary comment", async () => { const handlers = new Map Promise>(); const workspaceFixture = await createWorkspaceFixture(); diff --git a/src/handlers/review.ts b/src/handlers/review.ts index a2a2477c..f24d28fd 100644 --- a/src/handlers/review.ts +++ b/src/handlers/review.ts @@ -478,7 +478,7 @@ async function upsertReviewDetailsComment(params: { body: string; botHandles: string[]; recheckCanPublish?: () => boolean; -}): Promise { +}): Promise { const { octokit, owner, repo, prNumber, reviewOutputKey, body, botHandles } = params; const marker = buildReviewDetailsMarker(reviewOutputKey); const sanitizedBody = sanitizeOutgoingMentions(body, botHandles); @@ -497,7 +497,7 @@ async function upsertReviewDetailsComment(params: { ); if (params.recheckCanPublish && !params.recheckCanPublish()) { - return false; + return; } if (existingComment) { @@ -507,7 +507,7 @@ async function upsertReviewDetailsComment(params: { comment_id: existingComment.id, body: sanitizedBody, }); - return true; + return; } await octokit.rest.issues.createComment({ @@ -516,7 +516,6 @@ async function upsertReviewDetailsComment(params: { issue_number: prNumber, body: sanitizedBody, }); - return true; } async function appendReviewDetailsToSummary(params: { @@ -530,7 +529,7 @@ async function appendReviewDetailsToSummary(params: { requireDegradationDisclosure: boolean; reviewBoundedness?: ReviewBoundednessContract | null; recheckCanPublish?: () => boolean; -}): Promise { +}): Promise { const { octokit, owner, repo, prNumber, reviewOutputKey, botHandles } = params; let updatedReviewDetails = params.reviewDetailsBlock; const marker = buildReviewOutputMarker(reviewOutputKey); @@ -579,42 +578,24 @@ async function appendReviewDetailsToSummary(params: { ); } - const reviewDetailsMarker = buildReviewDetailsMarker(reviewOutputKey); - const reviewDetailsHeader = "
\nReview Details"; - const existingDetailsStart = summaryBody.indexOf(reviewDetailsHeader); - const existingDetailsMarkerIndex = summaryBody.indexOf( - reviewDetailsMarker, - Math.max(existingDetailsStart, 0), - ); - - // Insert or replace the Review Details block INSIDE the summary's
- // block (before the last
tag) so it renders as a nested - // collapsible. The review output marker that follows the summary's
- // stays outside both blocks. + // Insert review details block INSIDE the summary's
block (before + // the last
tag) so it renders as a nested collapsible. The review + // output marker () that follows the + // summary's stays outside both blocks. + const closingTag = ''; + const lastCloseIdx = summaryBody.lastIndexOf(closingTag); let updatedBody: string; - if ( - existingDetailsStart !== -1 && - existingDetailsMarkerIndex !== -1 && - existingDetailsStart < existingDetailsMarkerIndex - ) { - const before = summaryBody.slice(0, existingDetailsStart).replace(/\s*$/, ""); - const after = summaryBody.slice(existingDetailsMarkerIndex + reviewDetailsMarker.length).replace(/^\s*/, ""); - updatedBody = `${before}\n\n${updatedReviewDetails}\n\n${after}`; + if (lastCloseIdx === -1) { + // Fallback: append as before if structure is unexpected + updatedBody = `${summaryBody}\n\n${updatedReviewDetails}`; } else { - const closingTag = ''; - const lastCloseIdx = summaryBody.lastIndexOf(closingTag); - if (lastCloseIdx === -1) { - // Fallback: append as before if structure is unexpected - updatedBody = `${summaryBody}\n\n${updatedReviewDetails}`; - } else { - const before = summaryBody.slice(0, lastCloseIdx); - const after = summaryBody.slice(lastCloseIdx); - updatedBody = `${before}\n\n${updatedReviewDetails}\n${after}`; - } + const before = summaryBody.slice(0, lastCloseIdx); + const after = summaryBody.slice(lastCloseIdx); + updatedBody = `${before}\n\n${updatedReviewDetails}\n${after}`; } if (params.recheckCanPublish && !params.recheckCanPublish()) { - return false; + return; } await octokit.rest.issues.updateComment({ @@ -623,7 +604,6 @@ async function appendReviewDetailsToSummary(params: { comment_id: summaryComment.id, body: sanitizeOutgoingMentions(updatedBody, botHandles), }); - return true; } async function resolveAuthorTier(params: { @@ -3106,7 +3086,6 @@ export function createReviewHandler(deps: { } setReviewWorkPhase("prompt-build"); - const gitDiffInspectionAvailable = diffContext.strategy !== "github-file-list-fallback"; // Build review prompt const reviewPrompt = buildReviewPrompt({ owner: apiOwner, @@ -3195,7 +3174,6 @@ export function createReviewHandler(deps: { graphBlastRadius: graphBlastRadius ?? undefined, structuralImpact: structuralImpactForReview, reviewBoundedness, - gitDiffInspectionAvailable, }); reviewPhaseTimings.set( "retrieval/context assembly", @@ -3229,7 +3207,6 @@ export function createReviewHandler(deps: { dynamicTimeoutSeconds: config.timeout.dynamicScaling !== false ? timeoutEstimate.dynamicTimeoutSeconds : undefined, - gitDiffInspectionAvailable, }); executorResult = result; executorPhaseTimings = result.executorPhaseTimings ?? buildExecutorUnavailablePhases( @@ -3782,7 +3759,6 @@ export function createReviewHandler(deps: { (diffAnalysis?.metrics.totalLinesAdded ?? 0) + (diffAnalysis?.metrics.totalLinesRemoved ?? 0); - const reviewCompletedAt = new Date().toISOString(); const buildReviewDetailsBody = (timeoutProgress?: TimeoutReviewDetailsProgress): string => { const reviewDetailsBody = formatReviewDetailsSummary({ reviewOutputKey, @@ -3811,7 +3787,6 @@ export function createReviewHandler(deps: { totalPhaseStartAt, }), timeoutProgress, - completedAt: reviewCompletedAt, }); const suppressedSection = formatSuppressedFindingsSection(filterResult.filtered); @@ -3820,94 +3795,6 @@ export function createReviewHandler(deps: { : reviewDetailsBody; }; - let cleanApprovalSummaryPublished = false; - - if (config.review.autoApprove && result.conclusion === "success" && !(result.published ?? false)) { - try { - const octokit = await githubApp.getInstallationOctokit(event.installationId); - const appSlug = githubApp.getAppSlug(); - - // Double-check via a scan for the review output marker. This provides - // defense-in-depth if the executor didn't report published=true. - const idempotencyCheck = await ensureReviewOutputNotPublished({ - octokit, - owner: apiOwner, - repo: apiRepo, - prNumber: pr.number, - reviewOutputKey, - }); - - if (!idempotencyCheck.shouldPublish) { - logger.info( - { - prNumber: pr.number, - gate: "auto-approve", - gateResult: "skipped", - skipReason: "output-marker-present", - existingLocation: idempotencyCheck.existingLocation, - }, - "Skipping auto-approval because review output marker was published", - ); - } else if (!canPublishVisibleOutput("auto-approval")) { - // publish-rights helper already logged the skip reason - } else { - setReviewWorkPhase("publish"); - const approvalEvidence = [ - `Review prompt covered ${promptFiles.length} changed file${promptFiles.length === 1 ? "" : "s"}.`, - ]; - const approvalConfidence = depBumpContext?.mergeConfidence - ? renderApprovalConfidence(depBumpContext.mergeConfidence) - : null; - - await octokit.rest.pulls.createReview({ - owner: apiOwner, - repo: apiRepo, - pull_number: pr.number, - event: "APPROVE", - body: buildReviewOutputMarker(reviewOutputKey), - }); - await octokit.rest.issues.createComment({ - owner: apiOwner, - repo: apiRepo, - issue_number: pr.number, - body: sanitizeOutgoingMentions( - buildApprovedReviewBody({ - reviewOutputKey, - evidence: approvalEvidence, - approvalConfidence, - }), - [appSlug, "claude"], - ), - }); - cleanApprovalSummaryPublished = true; - - logger.info( - { - evidenceType: "review", - outcome: "submitted-approval", - deliveryId: event.id, - installationId: event.installationId, - owner: apiOwner, - repoName: apiRepo, - repo: `${apiOwner}/${apiRepo}`, - prNumber: pr.number, - reviewOutputKey, - }, - "Evidence bundle", - ); - logger.info( - { prNumber: pr.number, reviewOutputKey }, - "Submitted silent approval (no issues found)", - ); - } - } catch (err) { - logger.error( - { err, prNumber: pr.number }, - "Failed to submit approval", - ); - } - } - if (shouldProcessReviewOutput) { logger.info( { @@ -3924,98 +3811,52 @@ export function createReviewHandler(deps: { ); try { - const publishReviewDetails = async (detailsBody: string): Promise<"append" | "standalone" | null> => { - if (result.published || cleanApprovalSummaryPublished) { - if (canPublishVisibleOutput("deterministic Review Details append")) { - try { + const fullDetailsBody = buildReviewDetailsBody(); + + if (result.published) { + // Summary comment was posted -- append Review Details to it + if (canPublishVisibleOutput("deterministic Review Details append")) { + try { + setReviewWorkPhase("publish"); + await appendReviewDetailsToSummary({ + octokit: extractionOctokit, + owner: apiOwner, + repo: apiRepo, + prNumber: pr.number, + reviewOutputKey, + reviewDetailsBlock: fullDetailsBody, + botHandles: [githubApp.getAppSlug(), "claude"], + requireDegradationDisclosure: authorClassification.searchEnrichment.degraded, + reviewBoundedness, + recheckCanPublish: () => + canPublishVisibleOutput("deterministic Review Details append"), + }); + } catch (appendErr) { + // Fallback: post standalone if append fails (e.g., summary comment not found yet) + logger.warn( + { ...baseLog, gate: "review-details-output", gateResult: "append-fallback", err: appendErr }, + "Failed to append Review Details to summary comment; posting standalone", + ); + if (canPublishVisibleOutput("deterministic Review Details standalone comment")) { setReviewWorkPhase("publish"); - const appended = await appendReviewDetailsToSummary({ + await upsertReviewDetailsComment({ octokit: extractionOctokit, owner: apiOwner, repo: apiRepo, prNumber: pr.number, reviewOutputKey, - reviewDetailsBlock: detailsBody, + body: fullDetailsBody, botHandles: [githubApp.getAppSlug(), "claude"], - requireDegradationDisclosure: authorClassification.searchEnrichment.degraded, - reviewBoundedness, recheckCanPublish: () => - canPublishVisibleOutput("deterministic Review Details append"), + canPublishVisibleOutput("deterministic Review Details standalone comment"), }); - return appended ? "append" : null; - } catch (appendErr) { - logger.warn( - { ...baseLog, gate: "review-details-output", gateResult: "append-fallback", err: appendErr }, - "Failed to append Review Details to summary comment; posting standalone", - ); - if (canPublishVisibleOutput("deterministic Review Details standalone comment")) { - setReviewWorkPhase("publish"); - const publishedStandalone = await upsertReviewDetailsComment({ - octokit: extractionOctokit, - owner: apiOwner, - repo: apiRepo, - prNumber: pr.number, - reviewOutputKey, - body: detailsBody, - botHandles: [githubApp.getAppSlug(), "claude"], - recheckCanPublish: () => - canPublishVisibleOutput("deterministic Review Details standalone comment"), - }); - return publishedStandalone ? "standalone" : null; - } } } - return null; } - + } else { + // No summary comment (clean review) -- post standalone Review Details + // FORMAT-11 exemption: no summary exists to embed into; standalone preserves metrics visibility if (canPublishVisibleOutput("deterministic Review Details standalone comment")) { - setReviewWorkPhase("publish"); - const publishedStandalone = await upsertReviewDetailsComment({ - octokit: extractionOctokit, - owner: apiOwner, - repo: apiRepo, - prNumber: pr.number, - reviewOutputKey, - body: detailsBody, - botHandles: [githubApp.getAppSlug(), "claude"], - recheckCanPublish: () => - canPublishVisibleOutput("deterministic Review Details standalone comment"), - }); - return publishedStandalone ? "standalone" : null; - } - - return null; - }; - - const publicationMode = await publishReviewDetails(buildReviewDetailsBody()); - - if (publicationMode && publicationPhaseStartedAt !== undefined) { - reviewPhaseTimings.set( - "publication", - createReviewPhaseTiming({ - name: "publication", - status: "completed", - durationMs: Math.max(0, Date.now() - publicationPhaseStartedAt), - }), - ); - - const finalizedDetailsBody = buildReviewDetailsBody(); - if (publicationMode === "append") { - setReviewWorkPhase("publish"); - await appendReviewDetailsToSummary({ - octokit: extractionOctokit, - owner: apiOwner, - repo: apiRepo, - prNumber: pr.number, - reviewOutputKey, - reviewDetailsBlock: finalizedDetailsBody, - botHandles: [githubApp.getAppSlug(), "claude"], - requireDegradationDisclosure: authorClassification.searchEnrichment.degraded, - reviewBoundedness, - recheckCanPublish: () => - canPublishVisibleOutput("finalized Review Details timing update"), - }); - } else { setReviewWorkPhase("publish"); await upsertReviewDetailsComment({ octokit: extractionOctokit, @@ -4023,10 +3864,10 @@ export function createReviewHandler(deps: { repo: apiRepo, prNumber: pr.number, reviewOutputKey, - body: finalizedDetailsBody, + body: fullDetailsBody, botHandles: [githubApp.getAppSlug(), "claude"], recheckCanPublish: () => - canPublishVisibleOutput("finalized Review Details timing update"), + canPublishVisibleOutput("deterministic Review Details standalone comment"), }); } } @@ -4791,7 +4632,6 @@ export function createReviewHandler(deps: { // PR-issue linking (PRLINK-03) — reuse from initial review linkedIssues: linkedIssueResult, structuralImpact: structuralImpactForReview, - gitDiffInspectionAvailable, }); setReviewWorkPhaseForAttempt(retryReviewWorkAttempt.attemptId, "executor-dispatch"); @@ -4814,7 +4654,6 @@ export function createReviewHandler(deps: { totalFiles: timeoutTotalFiles, enableCheckpointTool: retryCheckpointEnabled, enableCommentTools: false, - gitDiffInspectionAvailable, }); const retryCheckpoint = (await knowledgeStore?.getCheckpoint?.(retryReviewOutputKey)) ?? null; @@ -5028,42 +4867,106 @@ export function createReviewHandler(deps: { } } - if (result.conclusion === "failure" && !(result.published ?? false)) { - const exhaustedTurnBudget = - result.stopReason === "max_turns" || - result.failureSubtype === "error_max_turns"; - const failureBody = exhaustedTurnBudget - ? [ - "> **Kodiai ran out of steps while reviewing this PR**", - "", - "_The review run ended before it could publish comments or an approval._", - "", - ...(result.stopReason ? [`Stop reason: ${result.stopReason}`] : []), - ...(result.failureSubtype ? [`Failure subtype: ${result.failureSubtype}`] : []), - "", - "Try requesting another review after narrowing the scope or improving the available review context.", - ].join("\n") - : [ - "> **Kodiai completed the review run but could not publish review output**", - "", - ...(result.stopReason ? [`Stop reason: ${result.stopReason}`] : []), - ...(result.failureSubtype ? [`Failure subtype: ${result.failureSubtype}`] : []), - ...(result.errorMessage ? [result.errorMessage] : []), - "", - "Try requesting another review if you want a fresh attempt.", - ].join("\n"); - - const octokit = await githubApp.getInstallationOctokit(event.installationId); - if (canPublishVisibleOutput("failure fallback comment")) { - setReviewWorkPhase("publish"); - await postOrUpdateErrorComment(octokit, { + // Auto-approval: only when autoApprove is enabled AND execution succeeded AND + // the model produced zero GitHub-visible output (no summary comment, no inline comments). + if (config.review.autoApprove && result.conclusion === "success") { + try { + // If the review execution published any output (summary comment, inline comments, etc.), + // do NOT auto-approve. Auto-approval is only valid when the bot produced zero output. + if (result.published) { + logger.info( + { + prNumber: pr.number, + gate: "auto-approve", + gateResult: "skipped", + skipReason: "output-published", + }, + "Skipping auto-approval because review output was published", + ); + return; + } + + const octokit = await githubApp.getInstallationOctokit(event.installationId); + const appSlug = githubApp.getAppSlug(); + + // Double-check via a scan for the review output marker. This provides + // defense-in-depth if the executor didn't report published=true. + const idempotencyCheck = await ensureReviewOutputNotPublished({ + octokit, owner: apiOwner, repo: apiRepo, - issueNumber: pr.number, - }, sanitizeOutgoingMentions(failureBody, [githubApp.getAppSlug(), "claude"]), logger); + prNumber: pr.number, + reviewOutputKey, + }); + + if (!idempotencyCheck.shouldPublish) { + logger.info( + { + prNumber: pr.number, + gate: "auto-approve", + gateResult: "skipped", + skipReason: "output-marker-present", + existingLocation: idempotencyCheck.existingLocation, + }, + "Skipping auto-approval because review output marker was published", + ); + return; + } + + { + if (!canPublishVisibleOutput("auto-approval")) { + return; + } + + setReviewWorkPhase("publish"); + const approvalEvidence = [ + `Review prompt covered ${promptFiles.length} changed file${promptFiles.length === 1 ? "" : "s"}.`, + ]; + const approvalConfidence = depBumpContext?.mergeConfidence + ? renderApprovalConfidence(depBumpContext.mergeConfidence) + : null; + + await octokit.rest.pulls.createReview({ + owner: apiOwner, + repo: apiRepo, + pull_number: pr.number, + event: "APPROVE", + body: sanitizeOutgoingMentions( + buildApprovedReviewBody({ + reviewOutputKey, + evidence: approvalEvidence, + approvalConfidence, + }), + [appSlug, "claude"], + ), + }); + + logger.info( + { + evidenceType: "review", + outcome: "submitted-approval", + deliveryId: event.id, + installationId: event.installationId, + owner: apiOwner, + repoName: apiRepo, + repo: `${apiOwner}/${apiRepo}`, + prNumber: pr.number, + reviewOutputKey, + }, + "Evidence bundle", + ); + logger.info( + { prNumber: pr.number, reviewOutputKey }, + "Submitted silent approval (no issues found)", + ); + } + } catch (err) { + logger.error( + { err, prNumber: pr.number }, + "Failed to submit approval", + ); } } - } catch (err) { if (!reviewPhaseTimings.has("workspace preparation") && workspacePhaseStartedAt !== undefined) { reviewPhaseTimings.set( diff --git a/src/index.ts b/src/index.ts index 29ff09f7..25a917e1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -42,9 +42,7 @@ import { createDbClient, type Sql } from "./db/client.ts"; import { runMigrations } from "./db/migrate.ts"; import { createContributorProfileStore } from "./contributor/index.ts"; import { createSlackCommandRoutes } from "./routes/slack-commands.ts"; -import { createSlackRelayWebhookRoutes } from "./routes/slack-relay-webhooks.ts"; import { createSlackClient } from "./slack/client.ts"; -import { deliverWebhookRelayEvent } from "./slack/webhook-relay-delivery.ts"; import { createSlackAssistantHandler } from "./slack/assistant-handler.ts"; import { createSlackWriteRunner } from "./slack/write-runner.ts"; import { createRequestTracker } from "./lifecycle/request-tracker.ts"; @@ -544,16 +542,6 @@ const app = new Hono(); // Mount routes app.route("/webhooks", createWebhookRoutes({ config, logger, dedup, githubApp, eventRouter, requestTracker, webhookQueueStore, shutdownManager })); -app.route("/webhooks/slack/relay", createSlackRelayWebhookRoutes({ - config, - logger, - onAcceptedRelay: async (event) => { - await deliverWebhookRelayEvent({ - slackClient, - event, - }); - }, -})); app.route("/webhooks/slack", createSlackEventRoutes({ config, logger, diff --git a/src/jobs/fork-manager.test.ts b/src/jobs/fork-manager.test.ts new file mode 100644 index 00000000..cef58eca --- /dev/null +++ b/src/jobs/fork-manager.test.ts @@ -0,0 +1,314 @@ +import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; +import type { Logger } from "pino"; +import type { BotUserClient } from "../auth/bot-user.ts"; +import { createForkManager } from "./fork-manager.ts"; + +type LogCall = { bindings: Record; message: string }; + +function createMockLogger() { + const debugCalls: LogCall[] = []; + const infoCalls: LogCall[] = []; + const warnCalls: LogCall[] = []; + const errorCalls: LogCall[] = []; + return { + logger: createMockLoggerWithArrays(debugCalls, infoCalls, warnCalls, errorCalls), + debugCalls, + infoCalls, + warnCalls, + errorCalls, + }; +} + +function createMockLoggerWithArrays( + debugCalls: LogCall[], + infoCalls: LogCall[], + warnCalls: LogCall[], + errorCalls: LogCall[], +): Logger { + const trace = mock(() => undefined); + const fatal = mock(() => undefined); + return { + debug: (bindings: Record, message: string) => { + debugCalls.push({ bindings, message }); + }, + info: (bindings: Record, message: string) => { + infoCalls.push({ bindings, message }); + }, + warn: (bindings: Record, message: string) => { + warnCalls.push({ bindings, message }); + }, + error: (bindings: Record, message: string) => { + errorCalls.push({ bindings, message }); + }, + trace, + fatal, + child: () => createMockLoggerWithArrays(debugCalls, infoCalls, warnCalls, errorCalls), + } as unknown as Logger; +} + +function createEnabledBotClient(overrides?: { + login?: string; + reposGet?: ReturnType; + createFork?: ReturnType; + request?: ReturnType; + deleteRef?: ReturnType; +}): BotUserClient { + const reposGet = + overrides?.reposGet ?? + mock(async (params: { owner: string; repo: string }) => ({ + data: { + full_name: `${params.owner}/${params.repo}`, + source: undefined, + }, + })); + const createFork = overrides?.createFork ?? mock(async () => ({ data: {} })); + const request = overrides?.request ?? mock(async () => ({ data: {} })); + const deleteRef = overrides?.deleteRef ?? mock(async () => ({ data: {} })); + + return { + enabled: true, + login: overrides?.login ?? "kodiai-bot", + octokit: { + rest: { + repos: { + get: reposGet, + createFork, + }, + git: { + deleteRef, + }, + }, + request, + } as unknown as BotUserClient["octokit"], + }; +} + +describe("createForkManager", () => { + const originalSetTimeout = globalThis.setTimeout; + + beforeEach(() => { + globalThis.setTimeout = ((handler: Parameters[0]) => { + if (typeof handler === "function") { + handler(); + } + return 0 as unknown as ReturnType; + }) as typeof globalThis.setTimeout; + }); + + afterEach(() => { + globalThis.setTimeout = originalSetTimeout; + }); + + test("disabled mode throws from all operations", async () => { + const { logger } = createMockLogger(); + const manager = createForkManager( + { + enabled: false, + login: "", + octokit: {} as BotUserClient["octokit"], + }, + logger, + ); + + expect(manager.enabled).toBe(false); + expect(() => manager.getBotPat()).toThrow("Fork manager is not available. Bot user client is not configured."); + await expect(manager.ensureFork("xbmc", "xbmc")).rejects.toThrow( + "Fork manager is not available. Bot user client is not configured.", + ); + await expect(manager.syncFork("kodiai-bot", "xbmc", "main")).rejects.toThrow( + "Fork manager is not available. Bot user client is not configured.", + ); + await expect(manager.deleteForkBranch("kodiai-bot", "xbmc", "feature")).rejects.toThrow( + "Fork manager is not available. Bot user client is not configured.", + ); + }); + + test("ensureFork reuses the in-memory cache and logs the cache hit", async () => { + const { logger, debugCalls, infoCalls } = createMockLogger(); + const reposGet = mock(async (params: { owner: string; repo: string }) => ({ + data: { + full_name: `${params.owner}/${params.repo}`, + source: { + full_name: "xbmc/xbmc", + }, + }, + })); + const createFork = mock(async () => { + throw new Error("should not create a fork when one already exists"); + }); + + const manager = createForkManager( + createEnabledBotClient({ reposGet, createFork }), + logger, + "ghp_test-token", + ); + + await expect(manager.ensureFork("xbmc", "xbmc")).resolves.toEqual({ + forkOwner: "kodiai-bot", + forkRepo: "xbmc", + }); + await expect(manager.ensureFork("xbmc", "xbmc")).resolves.toEqual({ + forkOwner: "kodiai-bot", + forkRepo: "xbmc", + }); + + expect(reposGet).toHaveBeenCalledTimes(1); + expect(createFork).not.toHaveBeenCalled(); + expect(infoCalls).toContainEqual({ + bindings: { owner: "xbmc", repo: "xbmc", forkOwner: "kodiai-bot", forkRepo: "xbmc" }, + message: "Found existing fork", + }); + expect(debugCalls).toContainEqual({ + bindings: { + owner: "xbmc", + repo: "xbmc", + cached: { forkOwner: "kodiai-bot", forkRepo: "xbmc" }, + }, + message: "Fork cache hit", + }); + }); + + test("ensureFork reuses an existing matching fork without creating a new fork", async () => { + const { logger, infoCalls } = createMockLogger(); + const reposGet = mock(async () => ({ + data: { + full_name: "kodiai-bot/xbmc", + source: { + full_name: "xbmc/xbmc", + }, + }, + })); + const createFork = mock(async () => ({ data: {} })); + + const manager = createForkManager(createEnabledBotClient({ reposGet, createFork }), logger, "ghp_test-token"); + + await expect(manager.ensureFork("xbmc", "xbmc")).resolves.toEqual({ + forkOwner: "kodiai-bot", + forkRepo: "xbmc", + }); + + expect(createFork).not.toHaveBeenCalled(); + expect(infoCalls).toContainEqual({ + bindings: { owner: "xbmc", repo: "xbmc", forkOwner: "kodiai-bot", forkRepo: "xbmc" }, + message: "Found existing fork", + }); + }); + + test("ensureFork creates a fork and polls through initial not-ready responses", async () => { + const { logger, infoCalls } = createMockLogger(); + let callCount = 0; + const reposGet = mock(async (params: { owner: string; repo: string }) => { + callCount += 1; + if (callCount === 1) { + const error = Object.assign(new Error("Not Found"), { status: 404 }); + throw error; + } + if (callCount === 2) { + const error = Object.assign(new Error("Fork not ready"), { status: 404 }); + throw error; + } + if (callCount === 3) { + const error = Object.assign(new Error("Still provisioning"), { status: 404 }); + throw error; + } + return { + data: { + full_name: `${params.owner}/${params.repo}`, + source: { + full_name: "xbmc/xbmc", + }, + }, + }; + }); + const createFork = mock(async () => ({ data: {} })); + + const manager = createForkManager(createEnabledBotClient({ reposGet, createFork }), logger, "ghp_test-token"); + + await expect(manager.ensureFork("xbmc", "xbmc")).resolves.toEqual({ + forkOwner: "kodiai-bot", + forkRepo: "xbmc", + }); + + expect(createFork).toHaveBeenCalledTimes(1); + expect(createFork).toHaveBeenCalledWith({ + owner: "xbmc", + repo: "xbmc", + default_branch_only: true, + }); + expect(reposGet).toHaveBeenCalledTimes(4); + expect(infoCalls).toContainEqual({ + bindings: { owner: "xbmc", repo: "xbmc" }, + message: "Creating fork", + }); + expect(infoCalls).toContainEqual({ + bindings: { owner: "xbmc", repo: "xbmc", forkOwner: "kodiai-bot", forkRepo: "xbmc" }, + message: "Fork created and ready", + }); + }); + + test("syncFork rewrites 409 conflicts into a descriptive error and logs a warning", async () => { + const { logger, debugCalls, warnCalls } = createMockLogger(); + const request = mock(async () => { + const error = Object.assign(new Error("Conflict"), { status: 409 }); + throw error; + }); + + const manager = createForkManager(createEnabledBotClient({ request }), logger, "ghp_test-token"); + + await expect(manager.syncFork("kodiai-bot", "xbmc", "main")).rejects.toThrow( + "Merge conflict syncing fork kodiai-bot/xbmc branch main with upstream. A git-based fallback may be needed.", + ); + + expect(request).toHaveBeenCalledWith("POST /repos/{owner}/{repo}/merge-upstream", { + owner: "kodiai-bot", + repo: "xbmc", + branch: "main", + }); + expect(debugCalls).toContainEqual({ + bindings: { forkOwner: "kodiai-bot", forkRepo: "xbmc", branch: "main" }, + message: "Syncing fork with upstream", + }); + expect(warnCalls).toContainEqual({ + bindings: { + forkOwner: "kodiai-bot", + forkRepo: "xbmc", + branch: "main", + error: expect.any(Error), + }, + message: "Fork sync hit merge conflict", + }); + }); + + test("deleteForkBranch is best-effort and logs failures", async () => { + const { logger, warnCalls } = createMockLogger(); + const deleteRef = mock(async () => { + throw new Error("network down"); + }); + + const manager = createForkManager(createEnabledBotClient({ deleteRef }), logger, "ghp_test-token"); + + await expect(manager.deleteForkBranch("kodiai-bot", "xbmc", "feature/test")).resolves.toBeUndefined(); + + expect(deleteRef).toHaveBeenCalledWith({ + owner: "kodiai-bot", + repo: "xbmc", + ref: "heads/feature/test", + }); + expect(warnCalls).toContainEqual({ + bindings: { + forkOwner: "kodiai-bot", + forkRepo: "xbmc", + branch: "feature/test", + error: expect.any(Error), + }, + message: "Failed to delete fork branch (best-effort)", + }); + }); + + test("getBotPat throws when the PAT is absent", () => { + const { logger } = createMockLogger(); + const manager = createForkManager(createEnabledBotClient(), logger); + + expect(() => manager.getBotPat()).toThrow("Bot PAT not provided to ForkManager"); + }); +}); diff --git a/src/jobs/fork-manager.ts b/src/jobs/fork-manager.ts index f95377dd..db9d589e 100644 --- a/src/jobs/fork-manager.ts +++ b/src/jobs/fork-manager.ts @@ -120,6 +120,7 @@ export function createForkManager(botClient: BotUserClient, logger: Logger, botP logger.info({ forkOwner, forkRepo, branch }, "Fork synced with upstream"); } catch (error: unknown) { if (typeof error === "object" && error !== null && "status" in error && error.status === 409) { + logger.warn({ forkOwner, forkRepo, branch, error }, "Fork sync hit merge conflict"); throw new Error( `Merge conflict syncing fork ${forkOwner}/${forkRepo} branch ${branch} with upstream. A git-based fallback may be needed.`, ); diff --git a/src/jobs/gist-publisher.test.ts b/src/jobs/gist-publisher.test.ts new file mode 100644 index 00000000..2201b457 --- /dev/null +++ b/src/jobs/gist-publisher.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, mock, test } from "bun:test"; +import type { Logger } from "pino"; +import type { BotUserClient } from "../auth/bot-user.ts"; +import { createGistPublisher } from "./gist-publisher.ts"; + +type LogCall = { bindings: Record; message: string }; + +function createMockLogger() { + const infoCalls: LogCall[] = []; + return { + logger: createMockLoggerWithArrays(infoCalls), + infoCalls, + }; +} + +function createMockLoggerWithArrays(infoCalls: LogCall[]): Logger { + return { + info: (bindings: Record, message: string) => { + infoCalls.push({ bindings, message }); + }, + debug: mock(() => undefined), + warn: mock(() => undefined), + error: mock(() => undefined), + trace: mock(() => undefined), + fatal: mock(() => undefined), + child: () => createMockLoggerWithArrays(infoCalls), + } as unknown as Logger; +} + +function createEnabledBotClient(createGist = mock(async () => ({ + data: { html_url: "https://gist.github.com/kodiai/abc123", id: "abc123" }, +}))): BotUserClient { + return { + enabled: true, + login: "kodiai-bot", + octokit: { + rest: { + gists: { + create: createGist, + }, + }, + } as unknown as BotUserClient["octokit"], + }; +} + +describe("createGistPublisher", () => { + test("disabled mode reports unavailable publisher and throws on create", async () => { + const { logger } = createMockLogger(); + const publisher = createGistPublisher( + { + enabled: false, + login: "", + octokit: {} as BotUserClient["octokit"], + }, + logger, + ); + + expect(publisher.enabled).toBe(false); + await expect( + publisher.createPatchGist({ + owner: "xbmc", + repo: "xbmc", + summary: "Fix playback regression", + patch: "diff --git a/file b/file", + }), + ).rejects.toThrow("Gist publisher is not available. Bot user client is not configured."); + }); + + test("creates a secret gist with repo summary description and timestamped patch filename", async () => { + const createGist = mock(async () => ({ + data: { + html_url: "https://gist.github.com/kodiai/def456", + id: "def456", + }, + })); + const { logger, infoCalls } = createMockLogger(); + const publisher = createGistPublisher(createEnabledBotClient(createGist), logger); + + const result = await publisher.createPatchGist({ + owner: "xbmc", + repo: "xbmc", + summary: "Fix playback regression", + patch: "diff --git a/src/player.ts b/src/player.ts\n+patched\n", + }); + + expect(publisher.enabled).toBe(true); + expect(result).toEqual({ + htmlUrl: "https://gist.github.com/kodiai/def456", + id: "def456", + }); + expect(createGist).toHaveBeenCalledTimes(1); + + const createArgs = (createGist.mock.calls as unknown as Array<[{ + description: string; + public: boolean; + files: Record; + }]>)[0]![0]; + + expect(createArgs.description).toBe("[kodiai] Patch for xbmc/xbmc: Fix playback regression"); + expect(createArgs.public).toBe(false); + + const filenames = Object.keys(createArgs.files); + expect(filenames).toHaveLength(1); + expect(filenames[0]).toMatch(/^xbmc-xbmc-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z\.patch$/); + expect(createArgs.files[filenames[0]!]!.content).toBe("diff --git a/src/player.ts b/src/player.ts\n+patched\n"); + + expect(infoCalls).toContainEqual({ + bindings: { + owner: "xbmc", + repo: "xbmc", + filename: filenames[0], + }, + message: "Creating secret gist with patch", + }); + expect(infoCalls).toContainEqual({ + bindings: { + id: "def456", + htmlUrl: "https://gist.github.com/kodiai/def456", + }, + message: "Secret gist created", + }); + }); +}); diff --git a/src/knowledge/cluster-scheduler.test.ts b/src/knowledge/cluster-scheduler.test.ts new file mode 100644 index 00000000..8c334e1a --- /dev/null +++ b/src/knowledge/cluster-scheduler.test.ts @@ -0,0 +1,117 @@ +import { afterEach, beforeEach, describe, expect, it, mock, vi } from "bun:test"; +import type { Logger } from "pino"; + +function createMockLogger(): Logger { + return { + debug: mock(() => {}), + info: mock(() => {}), + warn: mock(() => {}), + error: mock(() => {}), + child: mock(() => createMockLogger()), + } as unknown as Logger; +} + +describe("createClusterScheduler", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + mock.restore(); + }); + + it("continues to later repos when one pipeline run fails", async () => { + const runClusterPipeline = mock(async ({ repo }: { repo: string }) => { + if (repo === "xbmc/first") { + throw new Error("repo failed"); + } + }); + + mock.module("./cluster-store.ts", () => ({ + createClusterStore: mock(() => ({ mocked: true })), + })); + mock.module("./cluster-pipeline.ts", () => ({ + runClusterPipeline, + })); + + const { createClusterScheduler } = await import("./cluster-scheduler.ts"); + const logger = createMockLogger(); + const scheduler = createClusterScheduler({ + sql: {} as never, + taskRouter: {} as never, + logger, + repos: ["xbmc/first", "xbmc/second"], + }); + + await scheduler.runNow(); + + expect(runClusterPipeline).toHaveBeenCalledTimes(2); + expect(runClusterPipeline.mock.calls.map((call) => call[0].repo)).toEqual([ + "xbmc/first", + "xbmc/second", + ]); + expect(logger.error).toHaveBeenCalledWith( + { err: expect.any(Error), repo: "xbmc/first" }, + "Cluster pipeline failed for repo (fail-open)", + ); + expect(logger.info).toHaveBeenCalledWith( + { repo: "xbmc/second" }, + "Cluster pipeline completed for repo", + ); + }); + + it("start schedules the initial run and recurring interval, and stop clears both", async () => { + const runClusterPipeline = mock(async () => {}); + const createClusterStore = mock(() => ({ mocked: true })); + const logger = createMockLogger(); + + mock.module("./cluster-store.ts", () => ({ createClusterStore })); + mock.module("./cluster-pipeline.ts", () => ({ runClusterPipeline })); + + const { createClusterScheduler } = await import("./cluster-scheduler.ts"); + const scheduler = createClusterScheduler({ + sql: {} as never, + taskRouter: {} as never, + logger, + repos: ["xbmc/one"], + }); + + scheduler.start(); + scheduler.start(); + expect(vi.getTimerCount()).toBe(1); + expect(logger.debug).toHaveBeenCalledWith( + "Cluster scheduler already started, skipping duplicate start", + ); + + vi.advanceTimersByTime(120_000); + await Promise.resolve(); + expect(runClusterPipeline).toHaveBeenCalledTimes(1); + expect(vi.getTimerCount()).toBe(1); + + scheduler.stop(); + scheduler.stop(); + expect(vi.getTimerCount()).toBe(0); + expect(createClusterStore).toHaveBeenCalledTimes(1); + }); + + it("runNow handles an empty repo list without invoking the pipeline", async () => { + const runClusterPipeline = mock(async () => {}); + + mock.module("./cluster-store.ts", () => ({ + createClusterStore: mock(() => ({ mocked: true })), + })); + mock.module("./cluster-pipeline.ts", () => ({ runClusterPipeline })); + + const { createClusterScheduler } = await import("./cluster-scheduler.ts"); + const scheduler = createClusterScheduler({ + sql: {} as never, + taskRouter: {} as never, + logger: createMockLogger(), + repos: [], + }); + + await expect(scheduler.runNow()).resolves.toBeUndefined(); + expect(runClusterPipeline).not.toHaveBeenCalled(); + }); +}); diff --git a/src/knowledge/cluster-scheduler.ts b/src/knowledge/cluster-scheduler.ts index 81cad2f2..ca601d68 100644 --- a/src/knowledge/cluster-scheduler.ts +++ b/src/knowledge/cluster-scheduler.ts @@ -48,6 +48,11 @@ export function createClusterScheduler( return { start() { + if (startupTimer || intervalTimer) { + logger.debug("Cluster scheduler already started, skipping duplicate start"); + return; + } + startupTimer = setTimeout(() => { runAll().catch((err) => { logger.error({ err }, "Cluster scheduler initial run failed"); diff --git a/src/knowledge/isolation.test.ts b/src/knowledge/isolation.test.ts new file mode 100644 index 00000000..65fb28b7 --- /dev/null +++ b/src/knowledge/isolation.test.ts @@ -0,0 +1,199 @@ +import { describe, expect, test } from "bun:test"; +import type { Logger } from "pino"; +import { createIsolationLayer } from "./isolation.ts"; +import type { LearningMemoryRecord, LearningMemoryStore } from "./types.ts"; + +function createMockLogger(): Logger { + const logger = { + info: () => {}, + warn: () => {}, + error: () => {}, + debug: () => {}, + trace: () => {}, + fatal: () => {}, + child: () => logger, + level: "silent", + } as unknown as Logger; + + return logger; +} + +type Candidate = { memoryId: number; distance: number }; + +type StoreState = { + repoResults?: Candidate[]; + ownerResults?: Candidate[]; + records?: Record; +}; + +function makeRecord(memoryId: number, sourceRepo: string): LearningMemoryRecord { + return { + id: memoryId, + repo: sourceRepo, + owner: sourceRepo.split("/")[0] ?? "owner", + findingId: memoryId, + reviewId: memoryId + 100, + sourceRepo, + findingText: `finding-${memoryId}`, + severity: "major", + category: "correctness", + filePath: `src/file-${memoryId}.ts`, + outcome: "accepted", + embeddingModel: "test-model", + embeddingDim: 3, + stale: false, + createdAt: "2025-01-01T00:00:00Z", + }; +} + +function createStore(state: StoreState = {}): LearningMemoryStore { + const records = state.records ?? {}; + + return { + async writeMemory() {}, + async retrieveMemories() { + return state.repoResults ?? []; + }, + async retrieveMemoriesForOwner() { + return state.ownerResults ?? []; + }, + async getMemoryRecord(memoryId: number) { + return records[memoryId] ?? null; + }, + async markStale() { + return 0; + }, + async purgeStaleEmbeddings() { + return 0; + }, + close() {}, + }; +} + +describe("createIsolationLayer", () => { + test("non-adaptive mode filters over-threshold repo and shared results", async () => { + const layer = createIsolationLayer({ + memoryStore: createStore({ + repoResults: [ + { memoryId: 1, distance: 0.2 }, + { memoryId: 2, distance: 0.8 }, + ], + ownerResults: [ + { memoryId: 3, distance: 0.4 }, + { memoryId: 4, distance: 0.9 }, + ], + records: { + 1: makeRecord(1, "owner/repo"), + 3: makeRecord(3, "owner/shared"), + }, + }), + logger: createMockLogger(), + }); + + const result = await layer.retrieveWithIsolation({ + queryEmbedding: new Float32Array([0.1, 0.2]), + repo: "owner/repo", + owner: "owner", + sharingEnabled: true, + topK: 5, + distanceThreshold: 0.5, + adaptive: false, + logger: createMockLogger(), + }); + + expect(result.results.map((entry) => entry.memoryId)).toEqual([1, 3]); + expect(result.results.map((entry) => entry.distance)).toEqual([0.2, 0.4]); + expect(result.provenance.totalCandidates).toBe(2); + expect(result.provenance.sharedPoolUsed).toBe(true); + expect(result.provenance.query.internalTopK).toBe(5); + }); + + test("adaptive mode expands internal topK and preserves over-threshold candidates", async () => { + const layer = createIsolationLayer({ + memoryStore: createStore({ + repoResults: [ + { memoryId: 1, distance: 0.2 }, + { memoryId: 2, distance: 0.95 }, + ], + records: { + 1: makeRecord(1, "owner/repo"), + 2: makeRecord(2, "owner/repo"), + }, + }), + logger: createMockLogger(), + }); + + const result = await layer.retrieveWithIsolation({ + queryEmbedding: new Float32Array([0.1]), + repo: "owner/repo", + owner: "owner", + sharingEnabled: false, + topK: 3, + distanceThreshold: 0.3, + adaptive: true, + logger: createMockLogger(), + }); + + expect(result.results.map((entry) => entry.memoryId)).toEqual([1, 2]); + expect(result.results[1]?.distance).toBe(0.95); + expect(result.provenance.query.internalTopK).toBe(20); + expect(result.provenance.sharedPoolUsed).toBe(false); + }); + + test("dedupes shared-pool collisions by first-seen closest candidate and keeps truthful provenance when sharing is disabled", async () => { + const layer = createIsolationLayer({ + memoryStore: createStore({ + repoResults: [ + { memoryId: 10, distance: 0.15 }, + { memoryId: 11, distance: 0.4 }, + ], + ownerResults: [ + { memoryId: 10, distance: 0.8 }, + { memoryId: 12, distance: 0.3 }, + ], + records: { + 10: makeRecord(10, "owner/repo"), + 11: makeRecord(11, "owner/repo"), + 12: makeRecord(12, "owner/other-repo"), + }, + }), + logger: createMockLogger(), + }); + + const sharedEnabled = await layer.retrieveWithIsolation({ + queryEmbedding: new Float32Array([0.1]), + repo: "owner/repo", + owner: "owner", + sharingEnabled: true, + topK: 3, + distanceThreshold: 1, + adaptive: false, + logger: createMockLogger(), + }); + + expect(sharedEnabled.results.map((entry) => ({ id: entry.memoryId, distance: entry.distance }))).toEqual([ + { id: 10, distance: 0.15 }, + { id: 12, distance: 0.3 }, + { id: 11, distance: 0.4 }, + ]); + expect(sharedEnabled.provenance.repoSources.sort()).toEqual(["owner/other-repo", "owner/repo"]); + expect(sharedEnabled.provenance.sharedPoolUsed).toBe(true); + expect(sharedEnabled.provenance.totalCandidates).toBe(4); + + const sharingDisabled = await layer.retrieveWithIsolation({ + queryEmbedding: new Float32Array([0.1]), + repo: "owner/repo", + owner: "owner", + sharingEnabled: false, + topK: 3, + distanceThreshold: 1, + adaptive: false, + logger: createMockLogger(), + }); + + expect(sharingDisabled.results.map((entry) => entry.memoryId)).toEqual([10, 11]); + expect(sharingDisabled.provenance.repoSources).toEqual(["owner/repo"]); + expect(sharingDisabled.provenance.sharedPoolUsed).toBe(false); + expect(sharingDisabled.provenance.totalCandidates).toBe(2); + }); +}); diff --git a/src/knowledge/issue-retrieval.test.ts b/src/knowledge/issue-retrieval.test.ts new file mode 100644 index 00000000..0e91a690 --- /dev/null +++ b/src/knowledge/issue-retrieval.test.ts @@ -0,0 +1,179 @@ +import { describe, expect, test } from "bun:test"; +import type { Logger } from "pino"; +import { searchIssues } from "./issue-retrieval.ts"; +import type { IssueSearchResult, IssueStore } from "./issue-types.ts"; +import type { EmbeddingProvider, EmbeddingResult } from "./types.ts"; + +function createMockLogger(): Logger { + const logger = { + info: () => {}, + warn: () => {}, + error: () => {}, + debug: () => {}, + trace: () => {}, + fatal: () => {}, + child: () => logger, + level: "silent", + } as unknown as Logger; + + return logger; +} + +function createEmbeddingProvider(opts: { returnNull?: boolean } = {}): EmbeddingProvider { + return { + async generate(): Promise { + if (opts.returnNull) return null; + return { + embedding: new Float32Array([0.1, 0.2, 0.3]), + model: "test-model", + dimensions: 3, + }; + }, + get model() { + return "test-model"; + }, + get dimensions() { + return 3; + }, + }; +} + +function makeIssueResult(overrides: Partial<{ + distance: number; + body: string | null; + issueNumber: number; + title: string; + state: string; + repo: string; + authorLogin: string; + githubCreatedAt: string; +}> = {}): IssueSearchResult { + const body = Object.prototype.hasOwnProperty.call(overrides, "body") + ? overrides.body + : "Issue body"; + + return { + distance: overrides.distance ?? 0.2, + record: { + id: 1, + createdAt: "2025-01-01T00:00:00Z", + repo: overrides.repo ?? "owner/repo", + owner: "owner", + issueNumber: overrides.issueNumber ?? 42, + title: overrides.title ?? "Fix crash", + body, + state: overrides.state ?? "open", + authorLogin: overrides.authorLogin ?? "alice", + authorAssociation: "MEMBER", + labelNames: [], + templateSlug: null, + commentCount: 0, + assignees: [], + milestone: null, + reactionCount: 0, + isPullRequest: false, + locked: false, + embedding: null, + embeddingModel: "test-model", + githubCreatedAt: overrides.githubCreatedAt ?? "2025-01-02T00:00:00Z", + githubUpdatedAt: null, + closedAt: null, + }, + }; +} + +function createStore(results: IssueSearchResult[]): IssueStore { + return { + async upsert() {}, + async delete() {}, + async getByNumber() { + return null; + }, + async searchByEmbedding() { + return results; + }, + async searchByFullText() { + return []; + }, + async findSimilar() { + return []; + }, + async countByRepo() { + return 0; + }, + async upsertComment() {}, + async deleteComment() {}, + async getCommentsByIssue() { + return []; + }, + async searchCommentsByEmbedding() { + return []; + }, + }; +} + +describe("searchIssues", () => { + test("returns empty array when embedding generation returns null", async () => { + const matches = await searchIssues({ + store: createStore([makeIssueResult()]), + embeddingProvider: createEmbeddingProvider({ returnNull: true }), + query: "crash", + repo: "owner/repo", + topK: 5, + logger: createMockLogger(), + }); + + expect(matches).toEqual([]); + }); + + test("filters by threshold and maps metadata into issue matches", async () => { + const matches = await searchIssues({ + store: createStore([ + makeIssueResult({ distance: 0.25, issueNumber: 12, title: "Keep me", body: "Relevant body" }), + makeIssueResult({ distance: 0.8, issueNumber: 13, title: "Drop me" }), + ]), + embeddingProvider: createEmbeddingProvider(), + query: "relevant issue", + repo: "owner/repo", + topK: 5, + distanceThreshold: 0.3, + stateFilter: "open", + logger: createMockLogger(), + }); + + expect(matches).toHaveLength(1); + expect(matches[0]).toMatchObject({ + distance: 0.25, + repo: "owner/repo", + issueNumber: 12, + title: "Keep me", + state: "open", + authorLogin: "alice", + githubCreatedAt: "2025-01-02T00:00:00Z", + source: "issue", + }); + expect(matches[0]?.chunkText).toBe("#12 Keep me\n\nRelevant body"); + }); + + test("truncates bodies to exactly 2000 characters and handles missing or empty bodies", async () => { + const matches = await searchIssues({ + store: createStore([ + makeIssueResult({ issueNumber: 21, title: "Long", body: "x".repeat(2100) }), + makeIssueResult({ issueNumber: 22, title: "Missing", body: null, distance: 0.3 }), + makeIssueResult({ issueNumber: 23, title: "Empty", body: "", distance: 0.4 }), + ]), + embeddingProvider: createEmbeddingProvider(), + query: "body mapping", + repo: "owner/repo", + topK: 5, + distanceThreshold: 0.5, + logger: createMockLogger(), + }); + + expect(matches).toHaveLength(3); + expect(matches[0]?.chunkText).toBe(`#21 Long\n\n${"x".repeat(2000)}`); + expect(matches[0]?.chunkText.length).toBe(2010); + expect(matches[1]?.chunkText).toBe("#22 Missing\n\n"); + expect(matches[2]?.chunkText).toBe("#23 Empty\n\n"); + }); +}); diff --git a/src/knowledge/store.test.ts b/src/knowledge/store.test.ts index 293a0626..14808871 100644 --- a/src/knowledge/store.test.ts +++ b/src/knowledge/store.test.ts @@ -5,7 +5,7 @@ import type { KnowledgeStore } from "./types.ts"; import type { Sql } from "../db/client.ts"; import { runMigrations } from "../db/migrate.ts"; -const DATABASE_URL = process.env.DATABASE_URL ?? "postgresql://kodiai:kodiai@localhost:5432/kodiai"; +const TEST_DB_URL = process.env.TEST_DATABASE_URL; const mockLogger = { info: () => {}, @@ -36,17 +36,16 @@ async function truncateAll(): Promise { CASCADE`; } -beforeAll(async () => { - sql = postgres(DATABASE_URL, { max: 5, idle_timeout: 20, connect_timeout: 10 }); - await runMigrations(sql); - store = createKnowledgeStore({ sql, logger: mockLogger }); -}); - -afterAll(async () => { - await sql.end(); -}); +describe.skipIf(!TEST_DB_URL)("KnowledgeStore", () => { + beforeAll(async () => { + sql = postgres(TEST_DB_URL!, { max: 5, idle_timeout: 20, connect_timeout: 10 }); + await runMigrations(sql); + store = createKnowledgeStore({ sql, logger: mockLogger }); + }); -describe("KnowledgeStore", () => { + afterAll(async () => { + await sql.end(); + }); beforeEach(async () => { await truncateAll(); }); diff --git a/src/knowledge/test-coverage-exemptions.ts b/src/knowledge/test-coverage-exemptions.ts new file mode 100644 index 00000000..7d85e06b --- /dev/null +++ b/src/knowledge/test-coverage-exemptions.ts @@ -0,0 +1,26 @@ +export const M060_S01_RUNTIME_TARGETS = [ + "src/knowledge/isolation.ts", + "src/knowledge/wiki-fetch.ts", + "src/knowledge/issue-retrieval.ts", + "src/knowledge/wiki-popularity-config.ts", + "src/knowledge/wiki-linkshere-fetcher.ts", + "src/knowledge/wiki-popularity-scorer.ts", + "src/knowledge/cluster-scheduler.ts", +] as const; + +export const M060_S01_TYPE_ONLY_EXEMPTIONS = [ + "src/knowledge/canonical-code-types.ts", + "src/knowledge/cluster-types.ts", + "src/knowledge/code-snippet-types.ts", + "src/knowledge/issue-types.ts", + "src/knowledge/review-comment-types.ts", + "src/knowledge/types.ts", + "src/knowledge/wiki-publisher-types.ts", + "src/knowledge/wiki-staleness-types.ts", + "src/knowledge/wiki-types.ts", + "src/knowledge/wiki-update-types.ts", + "src/knowledge/wiki-voice-types.ts", +] as const; + +export type M060S01RuntimeTarget = (typeof M060_S01_RUNTIME_TARGETS)[number]; +export type M060S01TypeOnlyExemption = (typeof M060_S01_TYPE_ONLY_EXEMPTIONS)[number]; diff --git a/src/knowledge/wiki-fetch.test.ts b/src/knowledge/wiki-fetch.test.ts new file mode 100644 index 00000000..428e138b --- /dev/null +++ b/src/knowledge/wiki-fetch.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from "bun:test"; +import { buildWikiApiUrl, withWikiHeaders, type FetchFn } from "./wiki-fetch.ts"; + +describe("buildWikiApiUrl", () => { + test("uses the /api.php path with serialized params", () => { + const params = new URLSearchParams({ action: "query", format: "json" }); + + expect(buildWikiApiUrl("https://kodi.wiki", params)).toBe( + "https://kodi.wiki/api.php?action=query&format=json", + ); + }); + + test("appends /api.php even when the base URL already has query params", () => { + const params = new URLSearchParams({ action: "parse", page: "Foo Bar" }); + + expect(buildWikiApiUrl("https://kodi.wiki?foo=bar", params)).toBe( + "https://kodi.wiki?foo=bar/api.php?action=parse&page=Foo+Bar", + ); + }); +}); + +describe("withWikiHeaders", () => { + test("adds the Kodiai user agent when headers are omitted", async () => { + const calls: Array<{ input: string | URL | Request; init?: RequestInit }> = []; + const fetchFn: FetchFn = async (input, init) => { + calls.push({ input, init }); + return new Response(null, { status: 204 }); + }; + + const wrapped = withWikiHeaders(fetchFn); + await wrapped("https://kodi.wiki/api.php?action=query"); + + expect(calls).toHaveLength(1); + expect(calls[0]?.init?.headers).toEqual({ + "User-Agent": "Kodiai/1.0 (+https://github.com/xbmc/kodiai)", + }); + }); + + test("preserves existing headers and overrides any preexisting user agent", async () => { + const calls: Array<{ input: string | URL | Request; init?: RequestInit }> = []; + const fetchFn: FetchFn = async (input, init) => { + calls.push({ input, init }); + return new Response("ok"); + }; + + const wrapped = withWikiHeaders(fetchFn); + await wrapped("https://kodi.wiki/api.php", { + method: "POST", + headers: { + Accept: "application/json", + "User-Agent": "OldAgent/0.1", + }, + }); + + expect(calls[0]?.init?.method).toBe("POST"); + expect(calls[0]?.init?.headers).toEqual({ + Accept: "application/json", + "User-Agent": "Kodiai/1.0 (+https://github.com/xbmc/kodiai)", + }); + }); +}); diff --git a/src/knowledge/wiki-linkshere-fetcher.test.ts b/src/knowledge/wiki-linkshere-fetcher.test.ts new file mode 100644 index 00000000..b7bdf5b1 --- /dev/null +++ b/src/knowledge/wiki-linkshere-fetcher.test.ts @@ -0,0 +1,255 @@ +import { afterEach, beforeEach, describe, expect, it, mock, vi } from "bun:test"; +import type { Logger } from "pino"; +import { + fetchAllLinkshereCounts, +} from "./wiki-linkshere-fetcher.ts"; +import { + LINKSHERE_BATCH_SIZE, + LINKSHERE_MAX_PER_PAGE, + LINKSHERE_RATE_LIMIT_MS, +} from "./wiki-popularity-config.ts"; + +function createMockLogger(): Logger { + return { + debug: mock(() => {}), + info: mock(() => {}), + warn: mock(() => {}), + error: mock(() => {}), + child: mock(() => createMockLogger()), + } as unknown as Logger; +} + +function makeJsonResponse(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + }); +} + +describe("fetchAllLinkshereCounts", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + mock.restore(); + }); + + it("returns an empty map without fetching when no page IDs are provided", async () => { + const fetchFn = mock(async () => makeJsonResponse({})) as typeof globalThis.fetch; + + const counts = await fetchAllLinkshereCounts({ + baseUrl: "https://kodi.wiki", + pageIds: [], + fetchFn, + logger: createMockLogger(), + }); + + expect(fetchFn).not.toHaveBeenCalled(); + expect([...counts.entries()]).toEqual([]); + }); + + it("batches page IDs, follows pagination, and preserves zero-count pages", async () => { + const batchOneIds = Array.from({ length: LINKSHERE_BATCH_SIZE }, (_, i) => i + 1); + const batchTwoIds = [LINKSHERE_BATCH_SIZE + 1, LINKSHERE_BATCH_SIZE + 2]; + + const fetchFn = mock(async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(String(input)); + const pageIds = url.searchParams.get("pageids")?.split("|").map(Number) ?? []; + const lhcontinue = url.searchParams.get("lhcontinue"); + + expect((init?.headers as Record | undefined)?.["User-Agent"]).toContain("Kodiai/1.0"); + + if (pageIds[0] === 1 && !lhcontinue) { + return makeJsonResponse({ + continue: { lhcontinue: "next-page", continue: "-||" }, + query: { + pages: { + "1": { + pageid: 1, + title: "Page 1", + linkshere: [{ pageid: 10, ns: 0, title: "A" }], + }, + "2": { + pageid: 2, + title: "Page 2", + linkshere: [ + { pageid: 11, ns: 0, title: "B" }, + { pageid: 12, ns: 0, title: "C" }, + ], + }, + }, + }, + }); + } + + if (pageIds[0] === 1 && lhcontinue === "next-page") { + return makeJsonResponse({ + query: { + pages: { + "1": { + pageid: 1, + title: "Page 1", + linkshere: [ + { pageid: 13, ns: 0, title: "D" }, + { pageid: 14, ns: 0, title: "E" }, + ], + }, + "2": { + pageid: 2, + title: "Page 2", + }, + }, + }, + }); + } + + if (pageIds[0] === LINKSHERE_BATCH_SIZE + 1) { + return makeJsonResponse({ + query: { + pages: { + [String(LINKSHERE_BATCH_SIZE + 1)]: { + pageid: LINKSHERE_BATCH_SIZE + 1, + title: "Page 51", + linkshere: [{ pageid: 99, ns: 0, title: "Z" }], + }, + [String(LINKSHERE_BATCH_SIZE + 2)]: { + pageid: LINKSHERE_BATCH_SIZE + 2, + title: "Page 52", + linkshere: [], + }, + }, + }, + }); + } + + throw new Error(`Unexpected fetch URL: ${url.toString()}`); + }) as typeof globalThis.fetch; + + const promise = fetchAllLinkshereCounts({ + baseUrl: "https://kodi.wiki", + pageIds: [...batchOneIds, ...batchTwoIds], + fetchFn, + logger: createMockLogger(), + }); + + await Promise.resolve(); + await Promise.resolve(); + vi.advanceTimersByTime(LINKSHERE_RATE_LIMIT_MS); + await Promise.resolve(); + vi.advanceTimersByTime(LINKSHERE_RATE_LIMIT_MS); + const counts = await promise; + + expect(fetchFn).toHaveBeenCalledTimes(3); + expect(counts.get(1)).toBe(3); + expect(counts.get(2)).toBe(2); + expect(counts.get(LINKSHERE_BATCH_SIZE + 1)).toBe(1); + expect(counts.get(LINKSHERE_BATCH_SIZE + 2)).toBe(0); + }); + + it("caps per-page accumulation at the configured maximum", async () => { + const overCap = LINKSHERE_MAX_PER_PAGE - 1; + let callCount = 0; + const fetchFn = mock(async (_input: string | URL | Request) => { + callCount += 1; + if (callCount === 1) { + return makeJsonResponse({ + continue: { lhcontinue: "next-page", continue: "-||" }, + query: { + pages: { + "42": { + pageid: 42, + title: "Very Popular", + linkshere: Array.from({ length: overCap }, (_, i) => ({ + pageid: i + 1, + ns: 0, + title: `Ref ${i + 1}`, + })), + }, + }, + }, + }); + } + + return makeJsonResponse({ + query: { + pages: { + "42": { + pageid: 42, + title: "Very Popular", + linkshere: [{ pageid: 999999, ns: 0, title: "Overflow" }], + }, + }, + }, + }); + }) as typeof globalThis.fetch; + + const logger = createMockLogger(); + const promise = fetchAllLinkshereCounts({ + baseUrl: "https://kodi.wiki", + pageIds: [42], + fetchFn, + logger, + }); + + await Promise.resolve(); + await Promise.resolve(); + vi.advanceTimersByTime(LINKSHERE_RATE_LIMIT_MS); + const counts = await promise; + + expect(fetchFn).toHaveBeenCalledTimes(2); + expect(counts.get(42)).toBe(LINKSHERE_MAX_PER_PAGE); + expect(logger.debug).toHaveBeenCalledWith( + { pageId: 42, title: "Very Popular" }, + "Linkshere count capped at maximum", + ); + }); + + it("logs a warning for a failed batch and continues with later batches", async () => { + const pageIds = Array.from({ length: LINKSHERE_BATCH_SIZE + 1 }, (_, i) => i + 1); + const logger = createMockLogger(); + const fetchFn = mock(async (input: string | URL | Request) => { + const url = new URL(String(input)); + const firstPageId = Number(url.searchParams.get("pageids")?.split("|")[0]); + if (firstPageId === 1) { + throw new Error("first batch exploded"); + } + return makeJsonResponse({ + query: { + pages: { + [String(LINKSHERE_BATCH_SIZE + 1)]: { + pageid: LINKSHERE_BATCH_SIZE + 1, + title: "Recovered page", + linkshere: [{ pageid: 7, ns: 0, title: "Still counted" }], + }, + }, + }, + }); + }) as typeof globalThis.fetch; + + const promise = fetchAllLinkshereCounts({ + baseUrl: "https://kodi.wiki", + pageIds, + fetchFn, + logger, + }); + + await Promise.resolve(); + await Promise.resolve(); + vi.advanceTimersByTime(LINKSHERE_RATE_LIMIT_MS); + await Promise.resolve(); + const counts = await promise; + + expect(logger.warn).toHaveBeenCalledWith( + { + err: expect.any(Error), + batchIdx: 0, + batchSize: LINKSHERE_BATCH_SIZE, + }, + "Linkshere batch failed, continuing with remaining batches", + ); + expect(counts.get(1)).toBe(0); + expect(counts.get(LINKSHERE_BATCH_SIZE + 1)).toBe(1); + }); +}); diff --git a/src/knowledge/wiki-popularity-config.test.ts b/src/knowledge/wiki-popularity-config.test.ts new file mode 100644 index 00000000..6291448d --- /dev/null +++ b/src/knowledge/wiki-popularity-config.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, test } from "bun:test"; +import { + CITATION_WINDOW_DAYS, + POPULARITY_WEIGHTS, + RECENCY_HALF_LIFE_DAYS, + RECENCY_LAMBDA, + computeCompositeScore, +} from "./wiki-popularity-config.ts"; + +describe("wiki popularity config", () => { + test("exports the configured scoring constants", () => { + expect(CITATION_WINDOW_DAYS).toBe(90); + expect(RECENCY_HALF_LIFE_DAYS).toBe(90); + expect(POPULARITY_WEIGHTS).toEqual({ + inboundLinks: 0.3, + citationFrequency: 0.5, + editRecency: 0.2, + }); + expect( + POPULARITY_WEIGHTS.inboundLinks + + POPULARITY_WEIGHTS.citationFrequency + + POPULARITY_WEIGHTS.editRecency, + ).toBe(1); + expect(RECENCY_LAMBDA).toBeCloseTo(Math.LN2 / 90, 10); + }); + + test("returns zero normalized link and citation scores when min equals max", () => { + const result = computeCompositeScore({ + inboundLinks: 10, + citationCount: 3, + daysSinceEdit: 0, + normalization: { + minInboundLinks: 10, + maxInboundLinks: 10, + minCitationCount: 3, + maxCitationCount: 3, + }, + }); + + expect(result.editRecencyScore).toBe(1); + expect(result.compositeScore).toBeCloseTo(POPULARITY_WEIGHTS.editRecency, 10); + }); + + test("applies exponential recency decay at the configured half life", () => { + const fresh = computeCompositeScore({ + inboundLinks: 5, + citationCount: 10, + daysSinceEdit: 0, + normalization: { + minInboundLinks: 0, + maxInboundLinks: 10, + minCitationCount: 0, + maxCitationCount: 20, + }, + }); + + const halfLifeOld = computeCompositeScore({ + inboundLinks: 5, + citationCount: 10, + daysSinceEdit: RECENCY_HALF_LIFE_DAYS, + normalization: { + minInboundLinks: 0, + maxInboundLinks: 10, + minCitationCount: 0, + maxCitationCount: 20, + }, + }); + + expect(fresh.editRecencyScore).toBe(1); + expect(halfLifeOld.editRecencyScore).toBeCloseTo(0.5, 10); + expect(halfLifeOld.compositeScore).toBeLessThan(fresh.compositeScore); + }); + + test("combines normalized link, citation, and recency signals with configured weights", () => { + const result = computeCompositeScore({ + inboundLinks: 40, + citationCount: 15, + daysSinceEdit: 30, + normalization: { + minInboundLinks: 10, + maxInboundLinks: 70, + minCitationCount: 5, + maxCitationCount: 25, + }, + }); + + const normalizedLinks = (40 - 10) / (70 - 10); + const normalizedCitations = (15 - 5) / (25 - 5); + const expectedRecency = Math.exp(-RECENCY_LAMBDA * 30); + const expectedComposite = + POPULARITY_WEIGHTS.inboundLinks * normalizedLinks + + POPULARITY_WEIGHTS.citationFrequency * normalizedCitations + + POPULARITY_WEIGHTS.editRecency * expectedRecency; + + expect(result.editRecencyScore).toBeCloseTo(expectedRecency, 10); + expect(result.compositeScore).toBeCloseTo(expectedComposite, 10); + }); +}); diff --git a/src/knowledge/wiki-popularity-scorer.test.ts b/src/knowledge/wiki-popularity-scorer.test.ts new file mode 100644 index 00000000..ea5c2657 --- /dev/null +++ b/src/knowledge/wiki-popularity-scorer.test.ts @@ -0,0 +1,299 @@ +import { afterEach, beforeEach, describe, expect, it, mock, setSystemTime, vi } from "bun:test"; +import type { Logger } from "pino"; +import { + CITATION_WINDOW_DAYS, + computeCompositeScore, +} from "./wiki-popularity-config.ts"; +import { + createWikiPopularityScorer, + type WikiPopularityScoringResult, +} from "./wiki-popularity-scorer.ts"; + +function createMockLogger(): Logger { + return { + debug: mock(() => {}), + info: mock(() => {}), + warn: mock(() => {}), + error: mock(() => {}), + child: mock(() => createMockLogger()), + } as unknown as Logger; +} + +type DistinctPageRow = { + page_id: number; + title: string; + last_modified: string | null; +}; + +function createSqlReturning(rows: DistinctPageRow[]) { + return mock(async () => rows) as any; +} + +function createPopularityStore(overrides: Partial<{ + getCitationCounts(windowDays: number): Promise>; + cleanupOldCitations(windowDays: number): Promise; + upsertPopularity(records: Array>): Promise; +}> = {}) { + return { + getCitationCounts: mock(async (_windowDays: number) => new Map()), + cleanupOldCitations: mock(async (_windowDays: number) => 0), + upsertPopularity: mock(async (_records: Array>) => {}), + ...overrides, + }; +} + +describe("createWikiPopularityScorer", () => { + beforeEach(() => { + vi.useFakeTimers(); + setSystemTime(new Date("2026-04-01T00:00:00.000Z")); + }); + + afterEach(() => { + vi.useRealTimers(); + mock.restore(); + }); + + it("returns a skip result when there are no wiki pages to score", async () => { + const logger = createMockLogger(); + const popularityStore = createPopularityStore(); + const fetchFn = mock(async () => { + throw new Error("fetch should not run"); + }) as typeof globalThis.fetch; + + const scorer = createWikiPopularityScorer({ + sql: createSqlReturning([]), + logger, + wikiPageStore: {} as never, + popularityStore: popularityStore as never, + wikiBaseUrl: "https://kodi.wiki", + fetchFn, + }); + + const result = await scorer.runNow(); + + expect(result).toMatchObject({ + skipped: true, + skipReason: "no_wiki_pages", + pagesScored: 0, + citationsAggregated: 0, + citationsCleaned: 0, + }); + expect(fetchFn).not.toHaveBeenCalled(); + expect(popularityStore.upsertPopularity).not.toHaveBeenCalled(); + }); + + it("scores pages and upserts computed popularity records", async () => { + const pages: DistinctPageRow[] = [ + { page_id: 11, title: "Alpha", last_modified: "2026-03-31T00:00:00.000Z" }, + { page_id: 22, title: "Beta", last_modified: null }, + ]; + const citationCounts = new Map([[11, 4], [22, 1]]); + const popularityStore = createPopularityStore({ + getCitationCounts: mock(async (windowDays: number) => { + expect(windowDays).toBe(CITATION_WINDOW_DAYS); + return citationCounts; + }), + cleanupOldCitations: mock(async (windowDays: number) => { + expect(windowDays).toBe(CITATION_WINDOW_DAYS); + return 3; + }), + }); + + const fetchFn = mock(async (input: string | URL | Request) => { + const url = new URL(String(input)); + const requested = url.searchParams.get("pageids")?.split("|").map(Number); + expect(requested).toEqual([11, 22]); + return new Response(JSON.stringify({ + query: { + pages: { + "11": { + pageid: 11, + title: "Alpha", + linkshere: [ + { pageid: 1, ns: 0, title: "A" }, + { pageid: 2, ns: 0, title: "B" }, + { pageid: 3, ns: 0, title: "C" }, + ], + }, + "22": { + pageid: 22, + title: "Beta", + linkshere: [{ pageid: 4, ns: 0, title: "D" }], + }, + }, + }, + }), { status: 200, headers: { "content-type": "application/json" } }); + }) as typeof globalThis.fetch; + + const logger = createMockLogger(); + const scorer = createWikiPopularityScorer({ + sql: createSqlReturning(pages), + logger, + wikiPageStore: {} as never, + popularityStore: popularityStore as never, + wikiBaseUrl: "https://kodi.wiki", + fetchFn, + }); + + const result = await scorer.runNow(); + + expect(result).toMatchObject({ + skipped: false, + pagesScored: 2, + citationsAggregated: 2, + citationsCleaned: 3, + }); + + expect(popularityStore.upsertPopularity).toHaveBeenCalledTimes(1); + const upserted = (popularityStore.upsertPopularity as ReturnType).mock.calls[0]?.[0] as Array<{ + pageId: number; + pageTitle: string; + inboundLinks: number; + citationCount: number; + editRecencyScore: number; + compositeScore: number; + }>; + + const alphaExpected = computeCompositeScore({ + inboundLinks: 3, + citationCount: 4, + daysSinceEdit: 1, + normalization: { + maxInboundLinks: 3, + minInboundLinks: 1, + maxCitationCount: 4, + minCitationCount: 1, + }, + }); + const betaExpected = computeCompositeScore({ + inboundLinks: 1, + citationCount: 1, + daysSinceEdit: 365, + normalization: { + maxInboundLinks: 3, + minInboundLinks: 1, + maxCitationCount: 4, + minCitationCount: 1, + }, + }); + + expect(upserted).toEqual([ + { + pageId: 11, + pageTitle: "Alpha", + inboundLinks: 3, + citationCount: 4, + editRecencyScore: alphaExpected.editRecencyScore, + compositeScore: alphaExpected.compositeScore, + }, + { + pageId: 22, + pageTitle: "Beta", + inboundLinks: 1, + citationCount: 1, + editRecencyScore: betaExpected.editRecencyScore, + compositeScore: betaExpected.compositeScore, + }, + ]); + }); + + it("returns already_running when an overlapping runNow call arrives", async () => { + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + + const popularityStore = createPopularityStore({ + getCitationCounts: mock(async () => { + await gate; + return new Map(); + }), + }); + + const scorer = createWikiPopularityScorer({ + sql: createSqlReturning([{ page_id: 11, title: "Alpha", last_modified: null }]), + logger: createMockLogger(), + wikiPageStore: {} as never, + popularityStore: popularityStore as never, + wikiBaseUrl: "https://kodi.wiki", + fetchFn: mock(async () => new Response(JSON.stringify({ + query: { pages: { "11": { pageid: 11, title: "Alpha", linkshere: [] } } }, + }), { status: 200, headers: { "content-type": "application/json" } })) as typeof globalThis.fetch, + }); + + const firstRun = scorer.runNow(); + await Promise.resolve(); + + const secondRun = await scorer.runNow(); + release(); + await firstRun; + + expect(secondRun).toEqual({ + pagesScored: 0, + citationsAggregated: 0, + citationsCleaned: 0, + durationMs: 0, + skipped: true, + skipReason: "already_running", + } satisfies WikiPopularityScoringResult); + }); + + it("fails open with a skip reason when scoring throws", async () => { + const scorer = createWikiPopularityScorer({ + sql: createSqlReturning([{ page_id: 11, title: "Alpha", last_modified: null }]), + logger: createMockLogger(), + wikiPageStore: {} as never, + popularityStore: createPopularityStore({ + getCitationCounts: mock(async () => { + throw new Error("citation store unavailable"); + }), + }) as never, + wikiBaseUrl: "https://kodi.wiki", + fetchFn: mock(async () => new Response(JSON.stringify({ + query: { pages: { "11": { pageid: 11, title: "Alpha", linkshere: [] } } }, + }), { status: 200, headers: { "content-type": "application/json" } })) as typeof globalThis.fetch, + }); + + const result = await scorer.runNow(); + + expect(result).toEqual({ + pagesScored: 0, + citationsAggregated: 0, + citationsCleaned: 0, + durationMs: 0, + skipped: true, + skipReason: "error: citation store unavailable", + }); + }); + + it("starts once, schedules recurring runs, and stops cleanly", async () => { + const sql = createSqlReturning([]); + const logger = createMockLogger(); + const scorer = createWikiPopularityScorer({ + sql, + logger, + wikiPageStore: {} as never, + popularityStore: createPopularityStore() as never, + wikiBaseUrl: "https://kodi.wiki", + intervalMs: 100, + startupDelayMs: 10, + }); + + scorer.start(); + scorer.start(); + expect(vi.getTimerCount()).toBe(1); + + vi.advanceTimersByTime(10); + await Promise.resolve(); + expect(sql).toHaveBeenCalledTimes(1); + expect(vi.getTimerCount()).toBe(1); + + vi.advanceTimersByTime(100); + await Promise.resolve(); + expect(sql).toHaveBeenCalledTimes(2); + + scorer.stop(); + scorer.stop(); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/src/knowledge/wiki-popularity-scorer.ts b/src/knowledge/wiki-popularity-scorer.ts index 8546b03d..8303be78 100644 --- a/src/knowledge/wiki-popularity-scorer.ts +++ b/src/knowledge/wiki-popularity-scorer.ts @@ -237,6 +237,11 @@ export function createWikiPopularityScorer( return { start() { + if (startupHandle || intervalHandle) { + logger.debug("Wiki popularity scorer already started, skipping duplicate start"); + return; + } + const intervalMs = opts.intervalMs ?? DEFAULT_INTERVAL_MS; const delayMs = opts.startupDelayMs ?? DEFAULT_STARTUP_DELAY_MS; logger.info({ intervalMs, startupDelayMs: delayMs }, "Wiki popularity scorer starting"); diff --git a/src/knowledge/wiki-store.test.ts b/src/knowledge/wiki-store.test.ts index c94dacab..9f5ab9d9 100644 --- a/src/knowledge/wiki-store.test.ts +++ b/src/knowledge/wiki-store.test.ts @@ -4,6 +4,8 @@ import { createDbClient, type Sql } from "../db/client.ts"; import { runMigrations } from "../db/migrate.ts"; import type { WikiPageStore, WikiPageChunk } from "./wiki-types.ts"; +const TEST_DB_URL = process.env.TEST_DATABASE_URL; + const mockLogger = { info: () => {}, warn: () => {}, @@ -48,17 +50,13 @@ function makeEmbedding(seed: number = 42): Float32Array { return arr; } -describe("WikiPageStore (pgvector)", () => { +describe.skipIf(!TEST_DB_URL)("WikiPageStore (pgvector)", () => { let sql: Sql; let store: WikiPageStore; let close: () => Promise; beforeAll(async () => { - if (!process.env.DATABASE_URL) { - console.warn("Skipping WikiPageStore tests: DATABASE_URL not set"); - return; - } - const db = createDbClient({ logger: mockLogger }); + const db = createDbClient({ connectionString: TEST_DB_URL!, logger: mockLogger }); sql = db.sql; close = db.close; await runMigrations(sql); diff --git a/src/lib/review-utils.test.ts b/src/lib/review-utils.test.ts index 893b40f3..3c2aa3fd 100644 --- a/src/lib/review-utils.test.ts +++ b/src/lib/review-utils.test.ts @@ -82,15 +82,6 @@ describe("formatReviewDetailsSummary", () => { } }); - it("uses an explicit completedAt timestamp when provided", () => { - const result = formatReviewDetailsSummary({ - ...BASE_PARAMS, - completedAt: "2026-04-20T03:21:00.000Z", - }); - - expect(result).toContain("- Review completed: 2026-04-20T03:21:00.000Z"); - }); - it("renders usage line when usageLimit is present", () => { const result = formatReviewDetailsSummary({ ...BASE_PARAMS, diff --git a/src/lib/review-utils.ts b/src/lib/review-utils.ts index c63bf2c0..0607b17e 100644 --- a/src/lib/review-utils.ts +++ b/src/lib/review-utils.ts @@ -386,7 +386,6 @@ export function formatReviewDetailsSummary(params: { structuralImpact?: StructuralImpactPayload | null; phaseTimingSummary?: ReviewDetailsPhaseTimingSummary | null; timeoutProgress?: TimeoutReviewDetailsProgress | null; - completedAt?: string; }): string { const { reviewOutputKey, @@ -406,7 +405,6 @@ export function formatReviewDetailsSummary(params: { structuralImpact, phaseTimingSummary, timeoutProgress, - completedAt, } = params; const formatProfileLine = (label: string, profile: ResolvedReviewProfile): string => { @@ -462,7 +460,7 @@ export function formatReviewDetailsSummary(params: { ] : [profileLine]), `- Contributor experience: ${contributorExperience.text}`, - `- Review completed: ${completedAt ?? new Date().toISOString()}`, + `- Review completed: ${new Date().toISOString()}`, ]; if (phaseTimingSummary) { diff --git a/src/routes/slack-commands.test.ts b/src/routes/slack-commands.test.ts index 103f3a79..1f2b37f4 100644 --- a/src/routes/slack-commands.test.ts +++ b/src/routes/slack-commands.test.ts @@ -38,7 +38,6 @@ function createTestConfig(): AppConfig { slackKodiaiChannelId: "C123KODIAI", slackDefaultRepo: "xbmc/xbmc", slackAssistantModel: "claude-3-5-haiku-latest", - slackWebhookRelaySources: [], port: 3000, logLevel: "info", botAllowList: [], diff --git a/src/routes/slack-events.test.ts b/src/routes/slack-events.test.ts index e9fb74b4..6075e2b5 100644 --- a/src/routes/slack-events.test.ts +++ b/src/routes/slack-events.test.ts @@ -31,7 +31,6 @@ function createTestConfig(): AppConfig { slackKodiaiChannelId: "C123KODIAI", slackDefaultRepo: "xbmc/xbmc", slackAssistantModel: "claude-3-5-haiku-latest", - slackWebhookRelaySources: [], port: 3000, logLevel: "info", botAllowList: [], diff --git a/src/routes/webhooks.test.ts b/src/routes/webhooks.test.ts new file mode 100644 index 00000000..ec89da6a --- /dev/null +++ b/src/routes/webhooks.test.ts @@ -0,0 +1,314 @@ +import { describe, expect, test } from "bun:test"; +import { createHmac } from "node:crypto"; +import { Hono } from "hono"; +import type { Logger } from "pino"; +import type { AppConfig } from "../config.ts"; +import type { GitHubApp } from "../auth/github-app.ts"; +import type { Deduplicator } from "../webhook/dedup.ts"; +import type { EventRouter, WebhookEvent } from "../webhook/types.ts"; +import type { RequestTracker, ShutdownManager, WebhookQueueStore, WebhookQueueEntry } from "../lifecycle/types.ts"; +import { createWebhookRoutes } from "./webhooks.ts"; + +const WEBHOOK_SECRET = "github-webhook-secret"; + +function createTestLogger(): Logger { + return { + info: () => undefined, + warn: () => undefined, + error: () => undefined, + debug: () => undefined, + trace: () => undefined, + fatal: () => undefined, + child: () => createTestLogger(), + } as unknown as Logger; +} + +function createTestConfig(): AppConfig { + return { + githubAppId: "12345", + githubPrivateKey: "-----BEGIN PRIVATE KEY-----\nTEST\n-----END PRIVATE KEY-----", + webhookSecret: WEBHOOK_SECRET, + slackSigningSecret: "slack-signing-secret", + slackBotToken: "xoxb-test-token", + slackBotUserId: "U123BOT", + slackKodiaiChannelId: "C123KODIAI", + slackDefaultRepo: "xbmc/xbmc", + slackAssistantModel: "claude-3-5-haiku-latest", + port: 3000, + logLevel: "info", + botAllowList: [], + slackWikiChannelId: "", + wikiStalenessThresholdDays: 30, + wikiGithubOwner: "xbmc", + wikiGithubRepo: "xbmc", + botUserLogin: "", + botUserPat: "", + addonRepos: [], + mcpInternalBaseUrl: "", + acaJobImage: "", + acaResourceGroup: "rg-kodiai", + acaJobName: "caj-kodiai-agent", + }; +} + +function signGithubRequest(body: string): string { + return `sha256=${createHmac("sha256", WEBHOOK_SECRET).update(body).digest("hex")}`; +} + +function createHeaders(body: string, overrides?: Record): Headers { + const headers = new Headers({ + "content-type": "application/json", + "x-hub-signature-256": signGithubRequest(body), + "x-github-delivery": "delivery-123", + "x-github-event": "pull_request", + "user-agent": "GitHub-Hookshot/test", + }); + + for (const [key, value] of Object.entries(overrides ?? {})) { + if (value === undefined) { + headers.delete(key); + } else { + headers.set(key, value); + } + } + + return headers; +} + +function flushMicrotasks(): Promise { + return Promise.resolve().then(() => Promise.resolve()); +} + +function createApp(options?: { + dedupIsDuplicate?: boolean; + dispatchImpl?: (event: WebhookEvent) => Promise; + isShuttingDown?: boolean; +}) { + const dispatchedEvents: WebhookEvent[] = []; + const queuedEntries: Array> = []; + const trackedJobs: string[] = []; + const cleanupCalls: string[] = []; + const cleanupSignal = Promise.withResolvers(); + + const dedup: Deduplicator = { + isDuplicate: () => options?.dedupIsDuplicate ?? false, + }; + + const eventRouter: EventRouter = { + register: () => undefined, + dispatch: async (event) => { + dispatchedEvents.push(event); + await options?.dispatchImpl?.(event); + }, + }; + + const requestTracker: RequestTracker = { + trackRequest: () => () => undefined, + trackJob: () => { + const jobId = `job-${trackedJobs.length + 1}`; + trackedJobs.push(jobId); + return () => { + cleanupCalls.push(jobId); + cleanupSignal.resolve(jobId); + }; + }, + activeCount: () => ({ requests: 0, jobs: trackedJobs.length - cleanupCalls.length, total: trackedJobs.length - cleanupCalls.length }), + waitForDrain: async () => undefined, + }; + + const webhookQueueStore: WebhookQueueStore = { + enqueue: async (entry) => { + queuedEntries.push(entry); + }, + dequeuePending: async () => [], + markCompleted: async () => undefined, + markFailed: async () => undefined, + }; + + const shutdownManager: ShutdownManager = { + start: () => undefined, + isShuttingDown: () => options?.isShuttingDown ?? false, + }; + + const app = new Hono(); + app.route( + "/webhooks", + createWebhookRoutes({ + config: createTestConfig(), + logger: createTestLogger(), + dedup, + githubApp: {} as GitHubApp, + eventRouter, + requestTracker, + webhookQueueStore, + shutdownManager, + }), + ); + + return { app, dispatchedEvents, queuedEntries, trackedJobs, cleanupCalls, cleanupSignal }; +} + +describe("createWebhookRoutes", () => { + test("returns 401 before dispatch when the signature is missing", async () => { + const invalidJson = "{ not valid json"; + const { app, dispatchedEvents, trackedJobs } = createApp(); + + const response = await app.request("http://localhost/webhooks/github", { + method: "POST", + headers: createHeaders(invalidJson, { "x-hub-signature-256": undefined }), + body: invalidJson, + }); + + expect(response.status).toBe(401); + expect(dispatchedEvents).toHaveLength(0); + expect(trackedJobs).toHaveLength(0); + }); + + test("returns 401 before dispatch when the signature is invalid even if the payload is malformed", async () => { + const invalidJson = "{ not valid json"; + const { app, dispatchedEvents, trackedJobs } = createApp(); + + const response = await app.request("http://localhost/webhooks/github", { + method: "POST", + headers: createHeaders(invalidJson, { "x-hub-signature-256": "sha256=not-valid" }), + body: invalidJson, + }); + + expect(response.status).toBe(401); + expect(dispatchedEvents).toHaveLength(0); + expect(trackedJobs).toHaveLength(0); + }); + + test("short-circuits duplicate deliveries without dispatching", async () => { + const body = JSON.stringify({ action: "opened", installation: { id: 42 } }); + const { app, dispatchedEvents, trackedJobs, cleanupCalls } = createApp({ dedupIsDuplicate: true }); + + const response = await app.request("http://localhost/webhooks/github", { + method: "POST", + headers: createHeaders(body), + body, + }); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ received: true }); + expect(dispatchedEvents).toHaveLength(0); + expect(trackedJobs).toHaveLength(0); + expect(cleanupCalls).toHaveLength(0); + }); + + test("accepts a signed event, dispatches the constructed webhook event asynchronously, and cleans up tracking after dispatch settles", async () => { + const body = JSON.stringify({ + action: "opened", + installation: { id: 42 }, + repository: { + full_name: "acme/widgets", + owner: { login: "acme" }, + name: "widgets", + }, + sender: { login: "octocat" }, + pull_request: { number: 101 }, + }); + const dispatchGate = Promise.withResolvers(); + const { app, dispatchedEvents, trackedJobs, cleanupCalls, cleanupSignal } = createApp({ + dispatchImpl: async () => { + await dispatchGate.promise; + }, + }); + + const response = await app.request("http://localhost/webhooks/github", { + method: "POST", + headers: createHeaders(body), + body, + }); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ received: true }); + expect(trackedJobs).toEqual(["job-1"]); + + await flushMicrotasks(); + + expect(dispatchedEvents).toEqual([ + { + id: "delivery-123", + name: "pull_request", + installationId: 42, + payload: { + action: "opened", + installation: { id: 42 }, + repository: { + full_name: "acme/widgets", + owner: { login: "acme" }, + name: "widgets", + }, + sender: { login: "octocat" }, + pull_request: { number: 101 }, + }, + }, + ]); + expect(cleanupCalls).toHaveLength(0); + + dispatchGate.resolve(); + expect(await cleanupSignal.promise).toBe("job-1"); + + expect(cleanupCalls).toEqual(["job-1"]); + }); + + test("keeps the immediate accepted response and cleanup when async dispatch rejects", async () => { + const body = JSON.stringify({ action: "opened", installation: { id: 7 } }); + const { app, dispatchedEvents, trackedJobs, cleanupCalls } = createApp({ + dispatchImpl: async () => { + throw new Error("dispatch failed"); + }, + }); + + const response = await app.request("http://localhost/webhooks/github", { + method: "POST", + headers: createHeaders(body), + body, + }); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ received: true }); + expect(trackedJobs).toEqual(["job-1"]); + + await flushMicrotasks(); + + expect(dispatchedEvents).toHaveLength(1); + expect(cleanupCalls).toEqual(["job-1"]); + }); + + test("queues signed events during shutdown and preserves delivery metadata with the original raw body", async () => { + const body = '{\n "action": "opened",\n "installation": { "id": 77 },\n "sender": { "login": "octocat" }\n}'; + const { app, dispatchedEvents, queuedEntries, trackedJobs, cleanupCalls } = createApp({ isShuttingDown: true }); + + const response = await app.request("http://localhost/webhooks/github", { + method: "POST", + headers: createHeaders(body, { + "x-github-delivery": "delivery-shutdown", + "x-github-event": "issue_comment", + "user-agent": undefined, + }), + body, + }); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ received: true, queued: true }); + expect(dispatchedEvents).toHaveLength(0); + expect(trackedJobs).toHaveLength(0); + expect(cleanupCalls).toHaveLength(0); + expect(queuedEntries).toEqual([ + { + source: "github", + deliveryId: "delivery-shutdown", + eventName: "issue_comment", + headers: { + "x-hub-signature-256": signGithubRequest(body), + "x-github-delivery": "delivery-shutdown", + "x-github-event": "issue_comment", + "user-agent": "unknown", + }, + body, + }, + ]); + }); +}); diff --git a/src/slack/assistant-handler.test.ts b/src/slack/assistant-handler.test.ts index 30d1f335..015f796e 100644 --- a/src/slack/assistant-handler.test.ts +++ b/src/slack/assistant-handler.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, mock, test } from "bun:test"; +import type { Logger } from "pino"; import { createSlackAssistantHandler, type SlackAssistantAddressedPayload, @@ -6,6 +7,30 @@ import { } from "./assistant-handler.ts"; import { SLACK_WRITE_CONFIRMATION_TIMEOUT_MS } from "./write-intent.ts"; +type LogCall = { bindings: Record; message: string }; + +function createMockLogger() { + const warnCalls: LogCall[] = []; + return { + logger: createMockLoggerWithArrays(warnCalls), + warnCalls, + }; +} + +function createMockLoggerWithArrays(warnCalls: LogCall[]): Logger { + return { + warn: (bindings: Record, message: string) => { + warnCalls.push({ bindings, message }); + }, + debug: mock(() => undefined), + info: mock(() => undefined), + error: mock(() => undefined), + trace: mock(() => undefined), + fatal: mock(() => undefined), + child: () => createMockLoggerWithArrays(warnCalls), + } as unknown as Logger; +} + function createAddressedPayload(text: string): SlackAssistantAddressedPayload { return { channel: "C123KODIAI", @@ -380,6 +405,63 @@ describe("createSlackAssistantHandler", () => { ]); }); + test("blocks secret-matching write reply text and replaces it with the security policy message", async () => { + const published: string[] = []; + const { logger, warnCalls } = createMockLogger(); + const blockedToken = `ghp_${"A".repeat(36)}`; + + const handler = createSlackAssistantHandler({ + createWorkspace: async () => ({ + dir: "/tmp/workspace", + cleanup: async () => undefined, + }), + execute: async () => ({ answerText: "should not run" }), + runWrite: async () => ({ + outcome: "success", + prUrl: "https://github.com/xbmc/xbmc/pull/125", + responseText: "ignored by formatter", + retryCommand: "apply: print deployment summary", + mirrors: [ + { + url: "https://github.com/xbmc/xbmc/issues/1#issuecomment-11", + excerpt: `Leaked token ${blockedToken}`, + }, + ], + }), + publishInThread: async ({ text }) => { + published.push(text); + }, + logger, + defaultRepo: "xbmc/xbmc", + }); + + const result = await handler.handle(createAddressedPayload("apply: print deployment summary")); + + expect(result).toEqual({ + outcome: "answered", + route: "write", + repo: "xbmc/xbmc", + publishedText: + "Write run complete.\n" + + "- Changed: print deployment summary\n" + + "- Where: xbmc/xbmc\n" + + "PR: https://github.com/xbmc/xbmc/pull/125\n\n" + + "Mirrored GitHub comments:\n" + + "- https://github.com/xbmc/xbmc/issues/1#issuecomment-11\n" + + ` Leaked token ${blockedToken}`, + }); + expect(published).toEqual([ + "Write run started for xbmc/xbmc.", + "Milestone: running write execution and preparing PR output.", + "[Response blocked by security policy]", + ]); + expect(published.join("\n")).not.toContain(blockedToken); + expect(warnCalls).toContainEqual({ + bindings: { matchedPattern: "github-pat" }, + message: "Outgoing secret scan blocked Slack publish", + }); + }); + test("ambiguous conversational write intent stays read-only and publishes exact rerun command", async () => { let workspaceCalls = 0; let executionCalls = 0; diff --git a/src/slack/write-runner.test.ts b/src/slack/write-runner.test.ts index 4d60b698..363fd5ad 100644 --- a/src/slack/write-runner.test.ts +++ b/src/slack/write-runner.test.ts @@ -1,8 +1,507 @@ -import { describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { join, dirname } from "node:path"; +import { tmpdir } from "node:os"; import { createSlackWriteRunner } from "./write-runner.ts"; import { WritePolicyError } from "../jobs/workspace.ts"; +const TEST_INPUT = { + owner: "xbmc", + repo: "xbmc", + channel: "C123", + threadTs: "1700000000.000111", + messageTs: "1700000000.000222", + prompt: "apply update", + request: "update src/file.ts", + keyword: "apply" as const, +}; + +type TestWorkspace = { + dir: string; + cleanup: () => Promise; +}; + +type TestLogger = { + warn: Array<{ payload: Record; message: string }>; + info: Array<{ payload: Record; message: string }>; + error: Array<{ payload: Record; message: string }>; + logger: { + warn: (payload: Record, message: string) => void; + info: (payload: Record, message: string) => void; + error: (payload: Record, message: string) => void; + }; +}; + +const tempDirs: string[] = []; + +async function createGitWorkspace(options?: { + originOwner?: string; + originRepo?: string; + defaultBranch?: string; + files?: Record; +}): Promise { + const dir = await mkdtemp(join(tmpdir(), "slack-write-runner-")); + tempDirs.push(dir); + + const originOwner = options?.originOwner ?? "xbmc-bot"; + const originRepo = options?.originRepo ?? "xbmc"; + const defaultBranch = options?.defaultBranch ?? "main"; + const files = options?.files ?? { "src/file.ts": "export const value = 1;\n" }; + + await Bun.$`git -C ${dir} init -b ${defaultBranch}`.quiet(); + await Bun.$`git -C ${dir} config user.name test`.quiet(); + await Bun.$`git -C ${dir} config user.email test@example.com`.quiet(); + + for (const [relativePath, content] of Object.entries(files)) { + const fullPath = join(dir, relativePath); + await Bun.file(fullPath).write(content); + } + + await Bun.$`git -C ${dir} add -A`.quiet(); + await Bun.$`git -C ${dir} commit -m initial`.quiet(); + await Bun.$`git -C ${dir} remote add origin https://github.com/${originOwner}/${originRepo}.git`.quiet(); + + return { + dir, + cleanup: async () => { + await rm(dir, { recursive: true, force: true }); + }, + }; +} + +function createLogger(): TestLogger { + const warn: TestLogger["warn"] = []; + const info: TestLogger["info"] = []; + const error: TestLogger["error"] = []; + + return { + warn, + info, + error, + logger: { + warn: (payload, message) => warn.push({ payload, message }), + info: (payload, message) => info.push({ payload, message }), + error: (payload, message) => error.push({ payload, message }), + }, + }; +} + +async function overwriteFiles(dir: string, files: Record): Promise { + for (const [relativePath, content] of Object.entries(files)) { + const fullPath = join(dir, relativePath); + await mkdir(dirname(fullPath), { recursive: true }); + await writeFile(fullPath, content, "utf8"); + } +} + +beforeEach(() => { + tempDirs.length = 0; +}); + +afterEach(async () => { + await Promise.all(tempDirs.map((dir) => rm(dir, { recursive: true, force: true }))); +}); + describe("createSlackWriteRunner", () => { + test("routes fork single-file changes to a patch gist and preserves mirrored comments", async () => { + const workspace = await createGitWorkspace(); + const logger = createLogger(); + const gistCalls: Array> = []; + const prCalls: Array> = []; + const commitCalls: Array> = []; + const forkCalls: string[] = []; + + const runner = createSlackWriteRunner({ + resolveRepoInstallationContext: async () => ({ installationId: 42, defaultBranch: "main" }), + createWorkspace: async () => workspace as never, + loadRepoConfig: async () => ({ + config: { + write: { + enabled: true, + allowPaths: ["src/**"], + denyPaths: [], + secretScan: { enabled: true }, + }, + } as never, + warnings: [], + }), + forkManager: { + enabled: true, + ensureFork: async () => { + forkCalls.push("ensureFork"); + return { forkOwner: "xbmc-bot", forkRepo: "xbmc" }; + }, + syncFork: async () => { + forkCalls.push("syncFork"); + }, + deleteForkBranch: async () => undefined, + getBotPat: () => "bot-pat", + }, + gistPublisher: { + enabled: true, + createPatchGist: async (input) => { + gistCalls.push(input as unknown as Record); + return { htmlUrl: "https://gist.github.com/kodiai/g1", id: "g1" }; + }, + }, + execute: async () => { + await overwriteFiles(workspace.dir, { "src/file.ts": "export const value = 2;\n" }); + return { + conclusion: "success", + costUsd: 0, + numTurns: 1, + durationMs: 100, + sessionId: "session-1", + published: true, + errorMessage: undefined, + model: "test-model", + inputTokens: 1, + outputTokens: 1, + cacheReadTokens: 0, + cacheCreationTokens: 0, + stopReason: "end_turn", + resultText: "Applied changes", + publishEvents: [ + { + type: "comment", + url: "https://github.com/xbmc/xbmc/issues/12#issuecomment-99", + excerpt: "Posted follow-up comment", + }, + ], + }; + }, + commitBranchAndPush: async (input) => { + commitCalls.push(input as unknown as Record); + return { branchName: "kodiai/slack/apply-abc123", headSha: "deadbeef" }; + }, + createPullRequest: async (input) => { + prCalls.push(input as unknown as Record); + return { htmlUrl: "https://github.com/xbmc/xbmc/pull/321" }; + }, + logger: logger.logger as never, + }); + + const result = await runner.run(TEST_INPUT); + + expect(result.outcome).toBe("success"); + if (result.outcome !== "success") throw new Error("expected success"); + + expect(result.gistUrl).toBe("https://gist.github.com/kodiai/g1"); + expect(result.prUrl).toBeUndefined(); + expect(result.responseText).toContain("Created patch gist: https://gist.github.com/kodiai/g1"); + expect(result.responseText).toContain("curl -sL https://gist.github.com/kodiai/g1.patch | git apply"); + expect(result.mirrors).toEqual([ + { + url: "https://github.com/xbmc/xbmc/issues/12#issuecomment-99", + excerpt: "Posted follow-up comment", + }, + ]); + + expect(forkCalls).toEqual(["ensureFork", "syncFork"]); + expect(gistCalls).toHaveLength(1); + expect(gistCalls[0]!).toMatchObject({ + owner: "xbmc", + repo: "xbmc", + summary: "update src/file.ts", + }); + expect(String(gistCalls[0]!.patch)).toContain("+export const value = 2;"); + expect(commitCalls).toHaveLength(0); + expect(prCalls).toHaveLength(0); + expect(logger.info).toContainEqual({ + payload: { owner: "xbmc", repo: "xbmc", forkOwner: "xbmc-bot" }, + message: "Fork ensured and synced for Slack write-mode", + }); + }); + + test("falls back from fork PR creation to a patch gist and logs the warning", async () => { + const workspace = await createGitWorkspace({ + files: { + "src/file.ts": "export const value = 1;\n", + "src/other.ts": "export const other = 1;\n", + "README.md": "# repo\n", + "docs/notes.md": "hello\n", + }, + }); + const logger = createLogger(); + const gistCalls: Array> = []; + const commitCalls: Array> = []; + const prCalls: Array> = []; + + const runner = createSlackWriteRunner({ + resolveRepoInstallationContext: async () => ({ installationId: 42, defaultBranch: "main" }), + createWorkspace: async () => workspace as never, + loadRepoConfig: async () => ({ + config: { + write: { + enabled: true, + allowPaths: ["src/**"], + denyPaths: [], + secretScan: { enabled: true }, + }, + } as never, + warnings: [], + }), + forkManager: { + enabled: true, + ensureFork: async () => ({ forkOwner: "xbmc-bot", forkRepo: "xbmc" }), + syncFork: async () => undefined, + deleteForkBranch: async () => undefined, + getBotPat: () => "bot-pat", + }, + gistPublisher: { + enabled: true, + createPatchGist: async (input) => { + gistCalls.push(input as unknown as Record); + return { htmlUrl: "https://gist.github.com/kodiai/fallback", id: "fallback" }; + }, + }, + execute: async () => { + await overwriteFiles(workspace.dir, { + "src/file.ts": "export const value = 2;\n", + "src/other.ts": "export const other = 2;\n", + "src/third.ts": "export const third = 2;\n", + "README.md": "# changed\n", + }); + return { + conclusion: "success", + costUsd: 0, + numTurns: 1, + durationMs: 100, + sessionId: "session-1", + published: true, + errorMessage: undefined, + model: "test-model", + inputTokens: 1, + outputTokens: 1, + cacheReadTokens: 0, + cacheCreationTokens: 0, + stopReason: "end_turn", + resultText: "Applied changes", + publishEvents: [ + { + type: "comment", + url: "https://github.com/xbmc/xbmc/pull/55#issuecomment-88", + excerpt: "Mirror survives gist fallback", + }, + ], + }; + }, + commitBranchAndPush: async (input) => { + commitCalls.push(input as unknown as Record); + throw new Error("push failed"); + }, + createPullRequest: async (input) => { + prCalls.push(input as unknown as Record); + return { htmlUrl: "https://github.com/xbmc/xbmc/pull/321" }; + }, + logger: logger.logger as never, + }); + + const result = await runner.run({ + ...TEST_INPUT, + request: "fix files across the repo", + }); + + expect(result.outcome).toBe("success"); + if (result.outcome !== "success") throw new Error("expected success"); + + expect(result.gistUrl).toBe("https://gist.github.com/kodiai/fallback"); + expect(result.prUrl).toBeUndefined(); + expect(result.responseText).toContain("Could not create PR from fork, but here is the patch as a gist:"); + expect(result.responseText).toContain("https://gist.github.com/kodiai/fallback"); + expect(result.mirrors).toEqual([ + { + url: "https://github.com/xbmc/xbmc/pull/55#issuecomment-88", + excerpt: "Mirror survives gist fallback", + }, + ]); + + expect(commitCalls).toHaveLength(1); + expect(prCalls).toHaveLength(0); + expect(gistCalls).toHaveLength(1); + expect(String(gistCalls[0]!.patch)).toContain("+export const other = 2;"); + expect(logger.warn).toContainEqual({ + payload: { err: expect.any(Error), owner: "xbmc", repo: "xbmc" }, + message: "Fork-based PR creation failed; falling back to gist", + }); + }); + + test("returns write-policy refusal before gist fallback when fork commit is blocked", async () => { + const workspace = await createGitWorkspace({ + files: { + "src/file.ts": "export const value = 1;\n", + "README.md": "# repo\n", + "docs/notes.md": "hello\n", + "package.json": '{"name":"test","version":"1.0.0"}\n', + }, + }); + const gistCalls: Array> = []; + const logger = createLogger(); + + const runner = createSlackWriteRunner({ + resolveRepoInstallationContext: async () => ({ installationId: 42, defaultBranch: "main" }), + createWorkspace: async () => workspace as never, + loadRepoConfig: async () => ({ + config: { + write: { + enabled: true, + allowPaths: ["src/**"], + denyPaths: [], + secretScan: { enabled: true }, + }, + } as never, + warnings: [], + }), + forkManager: { + enabled: true, + ensureFork: async () => ({ forkOwner: "xbmc-bot", forkRepo: "xbmc" }), + syncFork: async () => undefined, + deleteForkBranch: async () => undefined, + getBotPat: () => "bot-pat", + }, + gistPublisher: { + enabled: true, + createPatchGist: async (input) => { + gistCalls.push(input as unknown as Record); + return { htmlUrl: "https://gist.github.com/kodiai/blocked", id: "blocked" }; + }, + }, + execute: async () => { + await overwriteFiles(workspace.dir, { + "src/file.ts": "export const value = 2;\n", + "README.md": "# change\n", + "docs/notes.md": "hello\n", + "package.json": '{"name":"test"}\n', + }); + return { + conclusion: "success", + costUsd: 0, + numTurns: 1, + durationMs: 100, + sessionId: "session-1", + published: true, + errorMessage: undefined, + model: "test-model", + inputTokens: 1, + outputTokens: 1, + cacheReadTokens: 0, + cacheCreationTokens: 0, + stopReason: "end_turn", + }; + }, + commitBranchAndPush: async () => { + throw new WritePolicyError("write-policy-not-allowed", "blocked", { + path: "README.md", + rule: "allowPaths", + }); + }, + createPullRequest: async () => ({ htmlUrl: "https://github.com/xbmc/xbmc/pull/321" }), + logger: logger.logger as never, + }); + + const result = await runner.run({ + ...TEST_INPUT, + request: "touch several top-level files", + }); + + expect(result.outcome).toBe("refusal"); + if (result.outcome !== "refusal") throw new Error("expected refusal"); + + expect(result.reason).toBe("policy"); + expect(result.responseText).toContain("Reason: write-policy-not-allowed"); + expect(result.responseText).toContain("File: README.md"); + expect(result.responseText).toContain("Retry command: apply: touch several top-level files"); + expect(gistCalls).toHaveLength(0); + expect(logger.warn).toContainEqual({ + payload: { err: expect.any(WritePolicyError), owner: "xbmc", repo: "xbmc" }, + message: "Fork-based PR creation failed; falling back to gist", + }); + }); + + test("uses legacy direct-push PR routing when fork and gist helpers are unavailable", async () => { + const workspace = await createGitWorkspace(); + const logger = createLogger(); + const commitCalls: Array> = []; + const prCalls: Array> = []; + + const runner = createSlackWriteRunner({ + resolveRepoInstallationContext: async () => ({ installationId: 42, defaultBranch: "main" }), + createWorkspace: async () => workspace as never, + loadRepoConfig: async () => ({ + config: { + write: { + enabled: true, + allowPaths: ["src/**"], + denyPaths: [], + secretScan: { enabled: true }, + }, + } as never, + warnings: [], + }), + execute: async () => { + await overwriteFiles(workspace.dir, { "src/file.ts": "export const value = 2;\n" }); + return { + conclusion: "success", + costUsd: 0, + numTurns: 1, + durationMs: 100, + sessionId: "session-1", + published: true, + errorMessage: undefined, + model: "test-model", + inputTokens: 1, + outputTokens: 1, + cacheReadTokens: 0, + cacheCreationTokens: 0, + stopReason: "end_turn", + publishEvents: [ + { + type: "comment", + url: "https://github.com/xbmc/xbmc/issues/12#issuecomment-99", + excerpt: "Posted follow-up comment", + }, + ], + }; + }, + commitBranchAndPush: async (input) => { + commitCalls.push(input as unknown as Record); + return { branchName: "kodiai/slack/apply-abc123", headSha: "deadbeef" }; + }, + createPullRequest: async (input) => { + prCalls.push(input as unknown as Record); + return { htmlUrl: "https://github.com/xbmc/xbmc/pull/321" }; + }, + logger: logger.logger as never, + }); + + const result = await runner.run(TEST_INPUT); + + expect(result.outcome).toBe("success"); + if (result.outcome !== "success") throw new Error("expected success"); + + expect(result.prUrl).toBe("https://github.com/xbmc/xbmc/pull/321"); + expect(result.gistUrl).toBeUndefined(); + expect(result.responseText).toContain("Opened PR: https://github.com/xbmc/xbmc/pull/321"); + expect(commitCalls).toHaveLength(1); + expect(commitCalls[0]).toMatchObject({ + token: undefined, + policy: { + allowPaths: ["src/**"], + denyPaths: [], + secretScanEnabled: true, + }, + }); + expect(prCalls).toHaveLength(1); + expect(prCalls[0]).toMatchObject({ + head: "kodiai/slack/apply-abc123", + base: "main", + }); + expect(logger.warn).toContainEqual({ + payload: { owner: "xbmc", repo: "xbmc" }, + message: "Slack write-mode active without BOT_USER_PAT; using legacy direct-push behavior", + }); + }); + test("returns success with PR URL and mirrored comment metadata", async () => { const executeCalls: Array> = []; const prCalls: Array> = []; @@ -61,16 +560,7 @@ describe("createSlackWriteRunner", () => { }, }); - const result = await runner.run({ - owner: "xbmc", - repo: "xbmc", - channel: "C123", - threadTs: "1700000000.000111", - messageTs: "1700000000.000222", - prompt: "apply update", - request: "update src/file.ts", - keyword: "apply", - }); + const result = await runner.run(TEST_INPUT); expect(result.outcome).toBe("success"); if (result.outcome !== "success") throw new Error("expected success"); diff --git a/src/telemetry/store.test.ts b/src/telemetry/store.test.ts index 1ca6dd8b..181a0b87 100644 --- a/src/telemetry/store.test.ts +++ b/src/telemetry/store.test.ts @@ -6,7 +6,7 @@ import type { ResilienceEventRecord } from "./types.ts"; import type { RateLimitEventRecord } from "./types.ts"; import type { Sql } from "../db/client.ts"; -const DATABASE_URL = process.env.DATABASE_URL ?? "postgresql://kodiai:kodiai@localhost:5432/kodiai"; +const TEST_DB_URL = process.env.TEST_DATABASE_URL; const mockLogger = { info: () => {}, @@ -93,16 +93,15 @@ async function truncateAll(): Promise { CASCADE`; } -beforeAll(async () => { - sql = postgres(DATABASE_URL, { max: 5, idle_timeout: 20, connect_timeout: 10 }); - store = createTelemetryStore({ sql, logger: mockLogger }); -}); - -afterAll(async () => { - await sql.end(); -}); +describe.skipIf(!TEST_DB_URL)("TelemetryStore", () => { + beforeAll(async () => { + sql = postgres(TEST_DB_URL!, { max: 5, idle_timeout: 20, connect_timeout: 10 }); + store = createTelemetryStore({ sql, logger: mockLogger }); + }); -describe("TelemetryStore", () => { + afterAll(async () => { + await sql.end(); + }); beforeEach(async () => { await truncateAll(); }); diff --git a/src/webhook/dedup.test.ts b/src/webhook/dedup.test.ts new file mode 100644 index 00000000..e3ac0c48 --- /dev/null +++ b/src/webhook/dedup.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, test } from "bun:test"; +import { createDeduplicator } from "./dedup.ts"; + +function createClock(start = 0) { + let now = start; + return { + now: () => now, + advance: (ms: number) => { + now += ms; + }, + }; +} + +describe("createDeduplicator", () => { + test("treats the first delivery ID as new and the second as duplicate", () => { + const deduplicator = createDeduplicator(); + + expect(deduplicator.isDuplicate("delivery-1")).toBe(false); + expect(deduplicator.isDuplicate("delivery-1")).toBe(true); + }); + + test("expires cached delivery IDs exactly at the TTL boundary using the injected clock", () => { + const clock = createClock(1_000); + const deduplicator = createDeduplicator({ ttlMs: 5_000, now: clock.now }); + + expect(deduplicator.isDuplicate("delivery-ttl")).toBe(false); + + clock.advance(4_999); + expect(deduplicator.isDuplicate("delivery-ttl")).toBe(true); + + clock.advance(1); + expect(deduplicator.isDuplicate("delivery-ttl")).toBe(false); + expect(deduplicator.isDuplicate("delivery-ttl")).toBe(true); + }); + + test("caches unexpected delivery ID strings as first-seen values without throwing", () => { + const deduplicator = createDeduplicator(); + const malformedDeliveryId = " delivery:\u0000odd\nvalue "; + + expect(deduplicator.isDuplicate(malformedDeliveryId)).toBe(false); + expect(deduplicator.isDuplicate(malformedDeliveryId)).toBe(true); + }); +}); diff --git a/src/webhook/filters.test.ts b/src/webhook/filters.test.ts new file mode 100644 index 00000000..cf204387 --- /dev/null +++ b/src/webhook/filters.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, mock, test } from "bun:test"; +import type { Logger } from "pino"; +import { createBotFilter } from "./filters.ts"; + +function createCaptureLogger() { + const debug = mock(() => undefined); + const logger = { + debug, + info: mock(() => undefined), + warn: mock(() => undefined), + error: mock(() => undefined), + trace: mock(() => undefined), + fatal: mock(() => undefined), + child: () => logger, + } as unknown as Logger; + + return { logger, debug }; +} + +describe("createBotFilter", () => { + test("rejects self-events even when the app slug is allow-listed and sender casing varies", () => { + const { logger, debug } = createCaptureLogger(); + const filter = createBotFilter("kodiai", ["kodiai"], logger); + + expect(filter.shouldProcess({ type: "Bot", login: "KoDiAi[BoT]" })).toBe(false); + expect(debug).toHaveBeenCalledWith( + { sender: "KoDiAi[BoT]" }, + "Filtered: event from app itself", + ); + }); + + test("passes human senders through without consulting the allow-list", () => { + const { logger, debug } = createCaptureLogger(); + const filter = createBotFilter("kodiai", ["dependabot"], logger); + + expect(filter.shouldProcess({ type: "User", login: "Alice" })).toBe(true); + expect(debug).not.toHaveBeenCalled(); + }); + + test("allows allow-listed bots after normalizing [bot] suffix and login casing", () => { + const { logger, debug } = createCaptureLogger(); + const filter = createBotFilter("kodiai", ["dependabot"], logger); + + expect(filter.shouldProcess({ type: "Bot", login: "Dependabot[BoT]" })).toBe(true); + expect(debug).toHaveBeenCalledWith( + { sender: "Dependabot[BoT]" }, + "Bot on allow-list, passing through", + ); + }); + + test("rejects bot senders that are not on the allow-list", () => { + const { logger, debug } = createCaptureLogger(); + const filter = createBotFilter("kodiai", ["renovate"], logger); + + expect(filter.shouldProcess({ type: "Bot", login: "Dependabot" })).toBe(false); + expect(debug).toHaveBeenCalledWith( + { sender: "Dependabot", type: "Bot" }, + "Filtered: bot account not on allow-list", + ); + }); +}); diff --git a/src/webhook/router.test.ts b/src/webhook/router.test.ts new file mode 100644 index 00000000..a0ca5778 --- /dev/null +++ b/src/webhook/router.test.ts @@ -0,0 +1,217 @@ +import { describe, expect, mock, test } from "bun:test"; +import type { Logger } from "pino"; +import { createEventRouter } from "./router.ts"; +import type { BotFilter, EventHandler, WebhookEvent } from "./types.ts"; + +function createCaptureLogger() { + const entries: Array<{ level: string; data?: Record; message: string }> = []; + + const capture = (level: string) => (data: unknown, message?: string) => { + if (typeof data === "string") { + entries.push({ level, message: data }); + return; + } + + entries.push({ + level, + data: (data ?? {}) as Record, + message: message ?? "", + }); + }; + + const logger = { + debug: capture("debug"), + info: capture("info"), + warn: capture("warn"), + error: capture("error"), + trace: capture("trace"), + fatal: capture("fatal"), + child: () => logger, + } as unknown as Logger; + + return { logger, entries }; +} + +function createEvent(overrides: Partial = {}): WebhookEvent { + return { + id: "delivery-123", + name: "pull_request", + installationId: 42, + payload: { + action: "opened", + sender: { type: "User", login: "alice" }, + }, + ...overrides, + }; +} + +describe("createEventRouter", () => { + test("dispatches both specific and general handlers for the same matching delivery", async () => { + const { logger, entries } = createCaptureLogger(); + const shouldProcess = mock(() => true); + const router = createEventRouter({ shouldProcess } satisfies BotFilter, logger); + const calls: string[] = []; + + const specificHandler: EventHandler = mock(async (event) => { + calls.push(`specific:${event.id}`); + }); + const generalHandler: EventHandler = mock(async (event) => { + calls.push(`general:${event.id}`); + }); + + router.register("pull_request.opened", specificHandler); + router.register("pull_request", generalHandler); + + const event = createEvent(); + await router.dispatch(event); + + expect(shouldProcess).toHaveBeenCalledWith({ type: "User", login: "alice" }); + expect(specificHandler).toHaveBeenCalledTimes(1); + expect(generalHandler).toHaveBeenCalledTimes(1); + expect(calls).toEqual(["specific:delivery-123", "general:delivery-123"]); + + const evaluationLog = entries.find( + (entry) => entry.message === "Router evaluated dispatch keys", + ); + expect(evaluationLog?.data?.specificKey).toBe("pull_request.opened"); + expect(evaluationLog?.data?.generalKey).toBe("pull_request"); + expect(evaluationLog?.data?.specificHandlerCount).toBe(1); + expect(evaluationLog?.data?.generalHandlerCount).toBe(1); + expect(evaluationLog?.data?.matchedHandlerCount).toBe(2); + + const completionLog = entries.find( + (entry) => entry.message === "Dispatched to 2 handler(s)", + ); + expect(completionLog?.data?.succeeded).toBe(2); + expect(completionLog?.data?.failed).toBe(0); + }); + + test("resolves unmatched events without throwing or invoking handlers when no keys match", async () => { + const { logger, entries } = createCaptureLogger(); + const shouldProcess = mock(() => true); + const router = createEventRouter({ shouldProcess } satisfies BotFilter, logger); + const unrelatedHandler: EventHandler = mock(async () => undefined); + + router.register("issue_comment.created", unrelatedHandler); + + await expect( + router.dispatch( + createEvent({ + name: "pull_request", + payload: { + sender: { type: "User", login: "alice" }, + }, + }), + ), + ).resolves.toBeUndefined(); + + expect(shouldProcess).toHaveBeenCalledWith({ type: "User", login: "alice" }); + expect(unrelatedHandler).not.toHaveBeenCalled(); + + const evaluationLog = entries.find( + (entry) => entry.message === "Router evaluated dispatch keys", + ); + expect(evaluationLog?.data?.specificKey).toBeUndefined(); + expect(evaluationLog?.data?.generalKey).toBe("pull_request"); + expect(evaluationLog?.data?.matchedHandlerCount).toBe(0); + + expect(entries).toContainEqual({ + level: "info", + data: { + deliveryId: "delivery-123", + event: "pull_request", + action: undefined, + specificKey: undefined, + generalKey: "pull_request", + matchedHandlerCount: 0, + filtered: false, + }, + message: "Event skipped because no handlers matched", + }); + }); + + test("short-circuits handler execution when the bot filter rejects the sender", async () => { + const { logger, entries } = createCaptureLogger(); + const shouldProcess = mock(() => false); + const router = createEventRouter({ shouldProcess } satisfies BotFilter, logger); + const specificHandler: EventHandler = mock(async () => undefined); + const generalHandler: EventHandler = mock(async () => undefined); + + router.register("pull_request.opened", specificHandler); + router.register("pull_request", generalHandler); + + await expect( + router.dispatch( + createEvent({ + payload: { + action: "opened", + sender: { type: "Bot", login: "dependabot[bot]" }, + }, + }), + ), + ).resolves.toBeUndefined(); + + expect(shouldProcess).toHaveBeenCalledWith({ + type: "Bot", + login: "dependabot[bot]", + }); + expect(specificHandler).not.toHaveBeenCalled(); + expect(generalHandler).not.toHaveBeenCalled(); + + expect(entries).toContainEqual({ + level: "info", + data: { + deliveryId: "delivery-123", + event: "pull_request", + action: "opened", + sender: "dependabot[bot]", + filtered: true, + filterReason: "bot-filter", + }, + message: "Event filtered before dispatch", + }); + expect(entries.some((entry) => entry.message === "Router evaluated dispatch keys")).toBe(false); + }); + + test("isolates one rejected handler so a sibling matched handler still runs and dispatch resolves", async () => { + const { logger, entries } = createCaptureLogger(); + const shouldProcess = mock(() => true); + const router = createEventRouter({ shouldProcess } satisfies BotFilter, logger); + const calls: string[] = []; + + const rejectingHandler: EventHandler = mock(async (event) => { + calls.push(`specific:${event.id}`); + throw new Error("specific handler exploded"); + }); + const fulfillingHandler: EventHandler = mock(async (event) => { + calls.push(`general:${event.id}`); + }); + + router.register("pull_request.opened", rejectingHandler); + router.register("pull_request", fulfillingHandler); + + await expect(router.dispatch(createEvent())).resolves.toBeUndefined(); + + expect(rejectingHandler).toHaveBeenCalledTimes(1); + expect(fulfillingHandler).toHaveBeenCalledTimes(1); + expect(calls).toEqual(["specific:delivery-123", "general:delivery-123"]); + + const failureLog = entries.find( + (entry) => entry.message === "Handler failed during dispatch", + ); + expect(failureLog?.data?.deliveryId).toBe("delivery-123"); + expect(failureLog?.data?.event).toBe("pull_request"); + expect(failureLog?.data?.action).toBe("opened"); + expect(failureLog?.data?.reason).toBeInstanceOf(Error); + expect((failureLog?.data?.reason as Error | undefined)?.message).toBe( + "specific handler exploded", + ); + + const completionLog = entries.find( + (entry) => entry.message === "Dispatched to 2 handler(s)", + ); + expect(completionLog?.data?.matchedHandlerCount).toBe(2); + expect(completionLog?.data?.succeeded).toBe(1); + expect(completionLog?.data?.failed).toBe(1); + }); +}); diff --git a/src/webhook/verify.test.ts b/src/webhook/verify.test.ts new file mode 100644 index 00000000..269828de --- /dev/null +++ b/src/webhook/verify.test.ts @@ -0,0 +1,47 @@ +import { afterEach, describe, expect, mock, test } from "bun:test"; +import { createHmac } from "node:crypto"; + +const SECRET = "test-webhook-secret"; +const BODY = JSON.stringify({ action: "review_requested", pull_request: { number: 42 } }); + +function sign(body: string, secret = SECRET): string { + return `sha256=${createHmac("sha256", secret).update(body).digest("hex")}`; +} + +afterEach(() => { + mock.restore(); +}); + +describe("verifyWebhookSignature", () => { + test("accepts a signature generated from the exact raw body string", async () => { + const { verifyWebhookSignature } = await import("./verify.ts"); + + await expect(verifyWebhookSignature(SECRET, BODY, sign(BODY))).resolves.toBe(true); + }); + + test("rejects signatures generated from a different body string", async () => { + const { verifyWebhookSignature } = await import("./verify.ts"); + const signatureForDifferentBody = sign(`${BODY}\n`); + + await expect( + verifyWebhookSignature(SECRET, BODY, signatureForDifferentBody), + ).resolves.toBe(false); + }); + + test("fails closed for malformed signature formats", async () => { + const { verifyWebhookSignature } = await import("./verify.ts"); + + await expect(verifyWebhookSignature(SECRET, BODY, "sha1=deadbeef")).resolves.toBe(false); + await expect(verifyWebhookSignature(SECRET, BODY, "not-even-a-signature")).resolves.toBe(false); + }); + + test("returns false when the underlying verifier throws", async () => { + const { verifyWebhookSignature } = await import("./verify.ts"); + const verifyThrow = mock(async () => { + throw new Error("boom"); + }) as unknown as typeof import("@octokit/webhooks-methods").verify; + + await expect(verifyWebhookSignature(SECRET, BODY, sign(BODY), verifyThrow)).resolves.toBe(false); + expect(verifyThrow).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/webhook/verify.ts b/src/webhook/verify.ts index ad449f45..6fc45662 100644 --- a/src/webhook/verify.ts +++ b/src/webhook/verify.ts @@ -5,13 +5,16 @@ import { verify } from "@octokit/webhooks-methods"; * Wraps @octokit/webhooks-methods which handles timing-safe comparison * and the sha256= prefix format. */ +export type WebhookVerifyFn = typeof verify; + export async function verifyWebhookSignature( secret: string, payload: string, signature: string, + verifyFn: WebhookVerifyFn = verify, ): Promise { try { - return await verify(secret, payload, signature); + return await verifyFn(secret, payload, signature); } catch { return false; } From 9b940dfea815eb67833b276a4c03c3b941841fd3 Mon Sep 17 00:00:00 2001 From: Keith Herrington Date: Tue, 21 Apr 2026 22:44:50 -0700 Subject: [PATCH 2/4] fix: address PR review issues --- .github/workflows/ci.yml | 11 +++++- scripts/verify-m057-s03.test.ts | 66 ++++++++++++++++++++++++++++++++ scripts/verify-m057-s03.ts | 51 +++++++++++++++++------- src/handlers/mention.test.ts | 2 +- src/handlers/mention.ts | 2 +- src/knowledge/wiki-fetch.test.ts | 4 +- src/knowledge/wiki-fetch.ts | 12 +++++- 7 files changed, 127 insertions(+), 21 deletions(-) create mode 100644 scripts/verify-m057-s03.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25ed6cee..cbd719e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: - uses: oven-sh/setup-bun@v2 with: bun-version: 1.3.8 - - run: bun install + - run: bun install --frozen-lockfile - run: bun run lint - run: bun run verify:m056:s03 - run: bun run verify:m059:s01 @@ -40,6 +40,13 @@ jobs: # Keep DB-backed tests on a low concurrency cap and split the suite into # two shorter invocations to avoid cross-file schema interference and runner crashes. # The first run covers scripts plus non-knowledge src tests; src/knowledge stays isolated. - - run: bun test --max-concurrency=2 scripts src + - run: | + mapfile -t non_knowledge_tests < <( + { + find src -maxdepth 1 -type f -name '*.test.ts' + find src -maxdepth 1 -mindepth 1 -type d ! -name knowledge + } | sort + ) + bun test --max-concurrency=2 scripts "${non_knowledge_tests[@]}" - run: bun test --max-concurrency=2 src/knowledge - run: bunx tsc --noEmit diff --git a/scripts/verify-m057-s03.test.ts b/scripts/verify-m057-s03.test.ts new file mode 100644 index 00000000..fd94d1c0 --- /dev/null +++ b/scripts/verify-m057-s03.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, test } from "bun:test"; +import { COMMAND_NAME, runVerifyM057S03 } from "./verify-m057-s03.ts"; + +describe("runVerifyM057S03", () => { + test("runs every command and prints pass output", () => { + const calls: readonly string[][] = []; + const stdout: string[] = []; + const stderr: string[] = []; + + runVerifyM057S03( + [ + ["bun", "test", "./a.test.ts"], + ["bun", "test", "./b.test.ts"], + ], + (command) => { + (calls as string[][]).push([...command]); + return { exitCode: 0 }; + }, + (message) => stdout.push(message), + (message) => stderr.push(message), + (code) => { + throw new Error(`unexpected exit(${code})`); + }, + ); + + expect(calls).toEqual([ + ["bun", "test", "./a.test.ts"], + ["bun", "test", "./b.test.ts"], + ]); + expect(stderr).toEqual([]); + expect(stdout).toEqual([ + "→ bun test ./a.test.ts\n", + "→ bun test ./b.test.ts\n", + `${COMMAND_NAME} passed\n`, + ]); + }); + + test("writes failure output and exits on first failing command", () => { + const stdout: string[] = []; + const stderr: string[] = []; + let exitCode: number | undefined; + + expect(() => + runVerifyM057S03( + [ + ["bun", "test", "./a.test.ts"], + ["bun", "test", "./b.test.ts"], + ], + (command) => ({ exitCode: command[2] === "./a.test.ts" ? 0 : 1 }), + (message) => stdout.push(message), + (message) => stderr.push(message), + (code) => { + exitCode = code; + throw new Error(`exit(${code})`); + }, + ), + ).toThrow("exit(1)"); + + expect(exitCode).toBe(1); + expect(stdout).toEqual([ + "→ bun test ./a.test.ts\n", + "→ bun test ./b.test.ts\n", + ]); + expect(stderr).toEqual([`${COMMAND_NAME} failed: bun test ./b.test.ts\n`]); + }); +}); diff --git a/scripts/verify-m057-s03.ts b/scripts/verify-m057-s03.ts index cae7a3d1..6a433285 100644 --- a/scripts/verify-m057-s03.ts +++ b/scripts/verify-m057-s03.ts @@ -1,5 +1,5 @@ -const COMMAND_NAME = "verify:m057:s03" as const; -const TEST_COMMANDS = [ +export const COMMAND_NAME = "verify:m057:s03" as const; +export const TEST_COMMANDS = [ ["bun", "test", "./src/jobs/fork-manager.test.ts"], ["bun", "test", "./src/jobs/gist-publisher.test.ts"], ["bun", "test", "./src/slack/write-runner.test.ts"], @@ -8,25 +8,48 @@ const TEST_COMMANDS = [ type TestCommand = (typeof TEST_COMMANDS)[number]; -function formatCommand(command: TestCommand): string { +type SpawnSyncResult = { + exitCode: number | null; +}; + +type SpawnSyncFn = (command: readonly string[]) => SpawnSyncResult; +type WriteFn = (message: string) => void; +type ExitFn = (code: number) => never; + +function formatCommand(command: readonly string[]): string { return command.join(" "); } -for (const command of TEST_COMMANDS) { - const [cmd, ...args] = command; - const formatted = formatCommand(command); - process.stdout.write(`→ ${formatted}\n`); - - const result = Bun.spawnSync({ - cmd: [cmd, ...args], +function defaultSpawnSync(command: readonly string[]): SpawnSyncResult { + return Bun.spawnSync({ + cmd: [...command], stdout: "inherit", stderr: "inherit", }); +} - if (result.exitCode !== 0) { - process.stderr.write(`${COMMAND_NAME} failed: ${formatted}\n`); - process.exit(result.exitCode); +export function runVerifyM057S03( + commands: readonly TestCommand[] = TEST_COMMANDS, + spawnSyncFn: SpawnSyncFn = defaultSpawnSync, + stdoutWrite: WriteFn = (message) => process.stdout.write(message), + stderrWrite: WriteFn = (message) => process.stderr.write(message), + exitFn: ExitFn = (code) => process.exit(code), +): void { + for (const command of commands) { + const formatted = formatCommand(command); + stdoutWrite(`→ ${formatted}\n`); + + const result = spawnSyncFn(command); + + if (result.exitCode !== 0) { + stderrWrite(`${COMMAND_NAME} failed: ${formatted}\n`); + exitFn(result.exitCode ?? 1); + } } + + stdoutWrite(`${COMMAND_NAME} passed\n`); } -process.stdout.write(`${COMMAND_NAME} passed\n`); +if (import.meta.main) { + runVerifyM057S03(); +} diff --git a/src/handlers/mention.test.ts b/src/handlers/mention.test.ts index c2088da4..07d71893 100644 --- a/src/handlers/mention.test.ts +++ b/src/handlers/mention.test.ts @@ -4778,7 +4778,7 @@ describe("createMentionHandler write intent gating", () => { expect(issueReplies.join("\n")).not.toContain(blockedToken); expect(warnCalls).toContainEqual({ bindings: { matchedPattern: "github-pat" }, - message: "Outgoing secret scan blocked mention reply publish", + message: "Outgoing secret scan blocked original mention reply content; publishing placeholder", }); await workspaceFixture.cleanup(); diff --git a/src/handlers/mention.ts b/src/handlers/mention.ts index 17f0c13e..c06e07cb 100644 --- a/src/handlers/mention.ts +++ b/src/handlers/mention.ts @@ -1260,7 +1260,7 @@ export function createMentionHandler(deps: { if (scanResult.blocked) { logger.warn( { matchedPattern: scanResult.matchedPattern }, - "Outgoing secret scan blocked mention reply publish", + "Outgoing secret scan blocked original mention reply content; publishing placeholder", ); sanitizedBody = "[Response blocked by security policy]"; } diff --git a/src/knowledge/wiki-fetch.test.ts b/src/knowledge/wiki-fetch.test.ts index 428e138b..a91f221d 100644 --- a/src/knowledge/wiki-fetch.test.ts +++ b/src/knowledge/wiki-fetch.test.ts @@ -10,11 +10,11 @@ describe("buildWikiApiUrl", () => { ); }); - test("appends /api.php even when the base URL already has query params", () => { + test("appends /api.php to the pathname and preserves existing query params", () => { const params = new URLSearchParams({ action: "parse", page: "Foo Bar" }); expect(buildWikiApiUrl("https://kodi.wiki?foo=bar", params)).toBe( - "https://kodi.wiki?foo=bar/api.php?action=parse&page=Foo+Bar", + "https://kodi.wiki/api.php?foo=bar&action=parse&page=Foo+Bar", ); }); }); diff --git a/src/knowledge/wiki-fetch.ts b/src/knowledge/wiki-fetch.ts index b4f06c23..057a056c 100644 --- a/src/knowledge/wiki-fetch.ts +++ b/src/knowledge/wiki-fetch.ts @@ -17,7 +17,17 @@ export function buildWikiApiUrl( baseUrl: string, params: URLSearchParams, ): string { - return `${baseUrl}/api.php?${params.toString()}`; + const url = new URL(baseUrl); + const pathname = url.pathname.endsWith("/") ? url.pathname.slice(0, -1) : url.pathname; + url.pathname = `${pathname}/api.php`; + + const mergedParams = new URLSearchParams(url.search); + for (const [key, value] of params.entries()) { + mergedParams.set(key, value); + } + url.search = mergedParams.toString(); + + return url.toString(); } /** From c9df016349b7eafbe0cc63267b1a984920cf19c0 Mon Sep 17 00:00:00 2001 From: Keith Herrington Date: Tue, 21 Apr 2026 23:10:42 -0700 Subject: [PATCH 3/4] fix: restore review failure fallback comments --- src/handlers/review.test.ts | 100 ++++++++++++++- src/handlers/review.ts | 234 ++++++++++++++++++++++++++---------- 2 files changed, 268 insertions(+), 66 deletions(-) diff --git a/src/handlers/review.test.ts b/src/handlers/review.test.ts index e931e278..dbb3ce00 100644 --- a/src/handlers/review.test.ts +++ b/src/handlers/review.test.ts @@ -5219,6 +5219,7 @@ describe("createReviewHandler finding extraction", () => { return { data: { id: 88 } }; }, updateComment: async (params: { body: string }) => { + updateCommentCalls += 1; updatedSummaryBody = params.body; return { data: {} }; }, @@ -11630,6 +11631,7 @@ describe("createReviewHandler Review Details phase timing publication", () => { let updatedSummaryBody: string | undefined; let standaloneDetailsBody: string | undefined; let createCommentCalls = 0; + let updateCommentCalls = 0; let issueCommentListCalls = 0; const reviewOutputKey = buildReviewOutputKey({ @@ -11747,6 +11749,7 @@ describe("createReviewHandler Review Details phase timing publication", () => { ); expect(createCommentCalls).toBe(0); + expect(updatedSummaryBody).toBeDefined(); expect(standaloneDetailsBody).toBeUndefined(); expect(updatedSummaryBody).toBeDefined(); expect(updatedSummaryBody).toContain("Review Details"); @@ -11808,7 +11811,7 @@ describe("createReviewHandler Review Details phase timing publication", () => { standaloneDetailsBody = params.body; return { data: { id: 188 } }; }, - updateComment: async () => { + updateComment: async (params: { body: string }) => { updateCommentCalls += 1; return { data: {} }; }, @@ -12143,3 +12146,98 @@ describe("createReviewHandler bounded review disclosure", () => { }); }); + + +describe("createReviewHandler failure fallback publication", () => { + test("posts a helpful PR error comment when review execution fails without publishing output", async () => { + const handlers = new Map Promise>(); + const workspaceFixture = await createWorkspaceFixture(); + + const createdCommentBodies: string[] = []; + + createReviewHandler({ + eventRouter: { + register: (eventKey, handler) => { + handlers.set(eventKey, handler); + }, + dispatch: async () => undefined, + }, + jobQueue: { + enqueue: async ( + _installationId: number, + fn: (metadata: JobQueueRunMetadata) => Promise, + ) => fn(createQueueRunMetadata()), + getQueueSize: () => 0, + getPendingCount: () => 0, + getActiveJobs: getEmptyActiveJobs, + } as unknown as JobQueue, + workspaceManager: { + create: async () => ({ + dir: workspaceFixture.dir, + cleanup: async () => undefined, + }), + cleanupStale: async () => 0, + } as WorkspaceManager, + githubApp: { + getAppSlug: () => "kodiai", + getInstallationOctokit: async () => ({ + rest: { + pulls: { + listReviewComments: async () => ({ data: [] }), + listReviews: async () => ({ data: [] }), + listCommits: async () => ({ data: [] }), + createReview: async () => ({ data: {} }), + }, + issues: { + listComments: async () => ({ data: [] }), + createComment: async (params: { body: string }) => { + createdCommentBodies.push(params.body); + return { data: { id: 501 } }; + }, + updateComment: async (params: { body: string }) => { + createdCommentBodies.push(params.body); + return { data: {} }; + }, + }, + reactions: { + createForIssue: async () => ({ data: {} }), + }, + search: { + issuesAndPullRequests: async () => ({ data: { total_count: 4 } }), + }, + }, + }) as never, + } as unknown as GitHubApp, + executor: { + execute: async () => ({ + conclusion: "failure", + published: false, + stopReason: "max_turns", + failureSubtype: "error_max_turns", + durationMs: 1, + numTurns: 25, + sessionId: "session-failure-fallback", + costUsd: 0, + }), + } as never, + telemetryStore: noopTelemetryStore, + logger: createNoopLogger(), + }); + + const handler = handlers.get("pull_request.review_requested"); + expect(handler).toBeDefined(); + + await handler!( + buildReviewRequestedEvent({ + requested_reviewer: { login: "kodiai[bot]" }, + }), + ); + + const failureComment = createdCommentBodies.find((body) => body.includes("Kodiai ran out of steps while reviewing this PR")); + expect(failureComment).toBeDefined(); + expect(failureComment).toContain("Stop reason: max_turns"); + expect(failureComment).toContain("Failure subtype: error_max_turns"); + + await workspaceFixture.cleanup(); + }); +}); diff --git a/src/handlers/review.ts b/src/handlers/review.ts index f24d28fd..17b7320c 100644 --- a/src/handlers/review.ts +++ b/src/handlers/review.ts @@ -579,9 +579,9 @@ async function appendReviewDetailsToSummary(params: { } // Insert review details block INSIDE the summary's
block (before - // the last
tag) so it renders as a nested collapsible. The review - // output marker () that follows the - // summary's stays outside both blocks. + // the last tag) so it renders as a nested collapsible. If a + // Review Details block already exists, replace it so the helper remains safe + // to call again after final publication timing is known. const closingTag = ''; const lastCloseIdx = summaryBody.lastIndexOf(closingTag); let updatedBody: string; @@ -591,11 +591,13 @@ async function appendReviewDetailsToSummary(params: { } else { const before = summaryBody.slice(0, lastCloseIdx); const after = summaryBody.slice(lastCloseIdx); - updatedBody = `${before}\n\n${updatedReviewDetails}\n${after}`; + const existingReviewDetailsPattern = /\n?
\s*\n?Review Details<\/summary>[\s\S]*?<\/details>\n?/; + const beforeWithoutExistingDetails = before.replace(existingReviewDetailsPattern, "\n").trimEnd(); + updatedBody = `${beforeWithoutExistingDetails}\n\n${updatedReviewDetails}\n${after}`; } if (params.recheckCanPublish && !params.recheckCanPublish()) { - return; + return undefined; } await octokit.rest.issues.updateComment({ @@ -604,6 +606,7 @@ async function appendReviewDetailsToSummary(params: { comment_id: summaryComment.id, body: sanitizeOutgoingMentions(updatedBody, botHandles), }); + return summaryComment.id; } async function resolveAuthorTier(params: { @@ -3795,6 +3798,21 @@ export function createReviewHandler(deps: { : reviewDetailsBody; }; + const finalizePublicationPhaseTiming = (): void => { + if (publicationPhaseStartedAt === undefined) { + return; + } + + reviewPhaseTimings.set( + "publication", + createReviewPhaseTiming({ + name: "publication", + status: "completed", + durationMs: Math.max(0, Date.now() - publicationPhaseStartedAt), + }), + ); + }; + if (shouldProcessReviewOutput) { logger.info( { @@ -3831,6 +3849,21 @@ export function createReviewHandler(deps: { recheckCanPublish: () => canPublishVisibleOutput("deterministic Review Details append"), }); + + finalizePublicationPhaseTiming(); + await appendReviewDetailsToSummary({ + octokit: extractionOctokit, + owner: apiOwner, + repo: apiRepo, + prNumber: pr.number, + reviewOutputKey, + reviewDetailsBlock: buildReviewDetailsBody(), + botHandles: [githubApp.getAppSlug(), "claude"], + requireDegradationDisclosure: authorClassification.searchEnrichment.degraded, + reviewBoundedness, + recheckCanPublish: () => + canPublishVisibleOutput("finalized Review Details append"), + }); } catch (appendErr) { // Fallback: post standalone if append fails (e.g., summary comment not found yet) logger.warn( @@ -3839,7 +3872,7 @@ export function createReviewHandler(deps: { ); if (canPublishVisibleOutput("deterministic Review Details standalone comment")) { setReviewWorkPhase("publish"); - await upsertReviewDetailsComment({ + const reviewDetailsCommentId = await upsertReviewDetailsComment({ octokit: extractionOctokit, owner: apiOwner, repo: apiRepo, @@ -3850,6 +3883,22 @@ export function createReviewHandler(deps: { recheckCanPublish: () => canPublishVisibleOutput("deterministic Review Details standalone comment"), }); + + finalizePublicationPhaseTiming(); + if ( + reviewDetailsCommentId !== undefined && + canPublishVisibleOutput("finalized Review Details standalone comment") + ) { + await extractionOctokit.rest.issues.updateComment({ + owner: apiOwner, + repo: apiRepo, + comment_id: reviewDetailsCommentId, + body: sanitizeOutgoingMentions( + buildReviewDetailsBody(), + [githubApp.getAppSlug(), "claude"], + ), + }); + } } } } @@ -3858,7 +3907,7 @@ export function createReviewHandler(deps: { // FORMAT-11 exemption: no summary exists to embed into; standalone preserves metrics visibility if (canPublishVisibleOutput("deterministic Review Details standalone comment")) { setReviewWorkPhase("publish"); - await upsertReviewDetailsComment({ + const reviewDetailsCommentId = await upsertReviewDetailsComment({ octokit: extractionOctokit, owner: apiOwner, repo: apiRepo, @@ -3869,6 +3918,22 @@ export function createReviewHandler(deps: { recheckCanPublish: () => canPublishVisibleOutput("deterministic Review Details standalone comment"), }); + + finalizePublicationPhaseTiming(); + if ( + reviewDetailsCommentId !== undefined && + canPublishVisibleOutput("finalized Review Details timing update") + ) { + await extractionOctokit.rest.issues.updateComment({ + owner: apiOwner, + repo: apiRepo, + comment_id: reviewDetailsCommentId, + body: sanitizeOutgoingMentions( + buildReviewDetailsBody(), + [githubApp.getAppSlug(), "claude"], + ), + }); + } } } } catch (err) { @@ -4867,6 +4932,47 @@ export function createReviewHandler(deps: { } } + if (result.conclusion === "failure" && !(result.published ?? false)) { + const exhaustedTurnBudget = + result.stopReason === "max_turns" || + result.failureSubtype === "error_max_turns"; + const failureBody = exhaustedTurnBudget + ? [ + "> **Kodiai ran out of steps while reviewing this PR**", + "", + "_The review run ended before it could publish comments or an approval._", + "", + ...(result.stopReason ? [`Stop reason: ${result.stopReason}`] : []), + ...(result.failureSubtype ? [`Failure subtype: ${result.failureSubtype}`] : []), + "", + "Try requesting another review after narrowing the scope or improving the available review context.", + ].join("\n") + : [ + "> **Kodiai completed the review run but could not publish review output**", + "", + ...(result.stopReason ? [`Stop reason: ${result.stopReason}`] : []), + ...(result.failureSubtype ? [`Failure subtype: ${result.failureSubtype}`] : []), + ...(result.errorMessage ? [result.errorMessage] : []), + "", + "Try requesting another review if you want a fresh attempt.", + ].join("\n"); + + const octokit = await githubApp.getInstallationOctokit(event.installationId); + if (canPublishVisibleOutput("failure fallback comment")) { + setReviewWorkPhase("publish"); + await postOrUpdateErrorComment( + octokit, + { + owner: apiOwner, + repo: apiRepo, + issueNumber: pr.number, + }, + sanitizeOutgoingMentions(failureBody, [githubApp.getAppSlug(), "claude"]), + logger, + ); + } + } + // Auto-approval: only when autoApprove is enabled AND execution succeeded AND // the model produced zero GitHub-visible output (no summary comment, no inline comments). if (config.review.autoApprove && result.conclusion === "success") { @@ -4913,53 +5019,51 @@ export function createReviewHandler(deps: { return; } - { - if (!canPublishVisibleOutput("auto-approval")) { - return; - } - - setReviewWorkPhase("publish"); - const approvalEvidence = [ - `Review prompt covered ${promptFiles.length} changed file${promptFiles.length === 1 ? "" : "s"}.`, - ]; - const approvalConfidence = depBumpContext?.mergeConfidence - ? renderApprovalConfidence(depBumpContext.mergeConfidence) - : null; - - await octokit.rest.pulls.createReview({ - owner: apiOwner, - repo: apiRepo, - pull_number: pr.number, - event: "APPROVE", - body: sanitizeOutgoingMentions( - buildApprovedReviewBody({ - reviewOutputKey, - evidence: approvalEvidence, - approvalConfidence, - }), - [appSlug, "claude"], - ), - }); + if (!canPublishVisibleOutput("auto-approval")) { + return; + } - logger.info( - { - evidenceType: "review", - outcome: "submitted-approval", - deliveryId: event.id, - installationId: event.installationId, - owner: apiOwner, - repoName: apiRepo, - repo: `${apiOwner}/${apiRepo}`, - prNumber: pr.number, + setReviewWorkPhase("publish"); + const approvalEvidence = [ + `Review prompt covered ${promptFiles.length} changed file${promptFiles.length === 1 ? "" : "s"}.`, + ]; + const approvalConfidence = depBumpContext?.mergeConfidence + ? renderApprovalConfidence(depBumpContext.mergeConfidence) + : null; + + await octokit.rest.pulls.createReview({ + owner: apiOwner, + repo: apiRepo, + pull_number: pr.number, + event: "APPROVE", + body: sanitizeOutgoingMentions( + buildApprovedReviewBody({ reviewOutputKey, - }, - "Evidence bundle", - ); - logger.info( - { prNumber: pr.number, reviewOutputKey }, - "Submitted silent approval (no issues found)", - ); - } + evidence: approvalEvidence, + approvalConfidence, + }), + [appSlug, "claude"], + ), + }); + + logger.info( + { + evidenceType: "review", + outcome: "submitted-approval", + deliveryId: event.id, + installationId: event.installationId, + owner: apiOwner, + repoName: apiRepo, + repo: `${apiOwner}/${apiRepo}`, + prNumber: pr.number, + reviewOutputKey, + }, + "Evidence bundle", + ); + logger.info( + { prNumber: pr.number, reviewOutputKey }, + "Submitted silent approval (no issues found)", + ); } catch (err) { logger.error( { err, prNumber: pr.number }, @@ -5111,19 +5215,19 @@ export function createReviewHandler(deps: { jobType: "pull-request-review", prNumber: pr.number, }); - } finally { - finalizeReviewWorkAttempt(); - } - - logger.info( - { ...baseLog, gate: "enqueue", gateResult: "completed" }, - "Review enqueue completed", - ); + } finally { + finalizeReviewWorkAttempt(); } - // Register for review trigger events - eventRouter.register("pull_request.opened", handleReview); - eventRouter.register("pull_request.ready_for_review", handleReview); - eventRouter.register("pull_request.review_requested", handleReview); - eventRouter.register("pull_request.synchronize", handleReview); + logger.info( + { ...baseLog, gate: "enqueue", gateResult: "completed" }, + "Review enqueue completed", + ); +} + +// Register for review trigger events +eventRouter.register("pull_request.opened", handleReview); +eventRouter.register("pull_request.ready_for_review", handleReview); +eventRouter.register("pull_request.review_requested", handleReview); +eventRouter.register("pull_request.synchronize", handleReview); } From 5f10776e5cb4c92005d6ed2e7e6093415c11c6c6 Mon Sep 17 00:00:00 2001 From: Keith Herrington Date: Tue, 21 Apr 2026 23:48:46 -0700 Subject: [PATCH 4/4] fix: address follow-up PR review comments --- scripts/verify-m057-s02.test.ts | 66 ++++++++++++++++++++++++++++++++ scripts/verify-m057-s02.ts | 47 ++++++++++++++++------- src/jobs/fork-manager.test.ts | 5 ++- src/jobs/fork-manager.ts | 20 +++++++++- src/knowledge/wiki-fetch.test.ts | 8 ++++ src/knowledge/wiki-fetch.ts | 2 +- 6 files changed, 129 insertions(+), 19 deletions(-) create mode 100644 scripts/verify-m057-s02.test.ts diff --git a/scripts/verify-m057-s02.test.ts b/scripts/verify-m057-s02.test.ts new file mode 100644 index 00000000..350d166c --- /dev/null +++ b/scripts/verify-m057-s02.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, test } from "bun:test"; +import { COMMAND_NAME, runVerifyM057S02 } from "./verify-m057-s02.ts"; + +describe("runVerifyM057S02", () => { + test("runs every command and prints pass output", () => { + const calls: readonly string[][] = []; + const stdout: string[] = []; + const stderr: string[] = []; + + runVerifyM057S02( + [ + ["bun", "test", "./a.test.ts"], + ["bun", "test", "./b.test.ts"], + ], + (command) => { + (calls as string[][]).push([...command]); + return { exitCode: 0 }; + }, + (message) => stdout.push(message), + (message) => stderr.push(message), + (code) => { + throw new Error(`unexpected exit(${code})`); + }, + ); + + expect(calls).toEqual([ + ["bun", "test", "./a.test.ts"], + ["bun", "test", "./b.test.ts"], + ]); + expect(stderr).toEqual([]); + expect(stdout).toEqual([ + "→ bun test ./a.test.ts\n", + "→ bun test ./b.test.ts\n", + `${COMMAND_NAME} passed\n`, + ]); + }); + + test("writes failure output and exits on first failing command", () => { + const stdout: string[] = []; + const stderr: string[] = []; + let exitCode: number | undefined; + + expect(() => + runVerifyM057S02( + [ + ["bun", "test", "./a.test.ts"], + ["bun", "test", "./b.test.ts"], + ], + (command) => ({ exitCode: command[2] === "./a.test.ts" ? 0 : 1 }), + (message) => stdout.push(message), + (message) => stderr.push(message), + (code) => { + exitCode = code; + throw new Error(`exit(${code})`); + }, + ), + ).toThrow("exit(1)"); + + expect(exitCode).toBe(1); + expect(stdout).toEqual([ + "→ bun test ./a.test.ts\n", + "→ bun test ./b.test.ts\n", + ]); + expect(stderr).toEqual([`${COMMAND_NAME} failed: bun test ./b.test.ts\n`]); + }); +}); diff --git a/scripts/verify-m057-s02.ts b/scripts/verify-m057-s02.ts index 45e4dbeb..ba330cc4 100644 --- a/scripts/verify-m057-s02.ts +++ b/scripts/verify-m057-s02.ts @@ -5,26 +5,45 @@ const TEST_COMMANDS = [ ] as const; type TestCommand = (typeof TEST_COMMANDS)[number]; +type SpawnSyncFn = (command: TestCommand) => { exitCode: number }; +type WriteFn = (message: string) => void; +type ExitFn = (code: number) => never; function formatCommand(command: TestCommand): string { return command.join(" "); } -for (const command of TEST_COMMANDS) { - const [cmd, ...args] = command; - const formatted = formatCommand(command); - process.stdout.write(`→ ${formatted}\n`); +export function runVerifyM057S02( + commands: readonly TestCommand[] = TEST_COMMANDS, + spawnSyncFn: SpawnSyncFn = (command) => { + const [cmd, ...args] = command; + return Bun.spawnSync({ + cmd: [cmd, ...args], + stdout: "inherit", + stderr: "inherit", + }); + }, + writeStdout: WriteFn = (message) => process.stdout.write(message), + writeStderr: WriteFn = (message) => process.stderr.write(message), + exitFn: ExitFn = (code) => process.exit(code), +): void { + for (const command of commands) { + const formatted = formatCommand(command); + writeStdout(`→ ${formatted}\n`); - const result = Bun.spawnSync({ - cmd: [cmd, ...args], - stdout: "inherit", - stderr: "inherit", - }); - - if (result.exitCode !== 0) { - process.stderr.write(`${COMMAND_NAME} failed: ${formatted}\n`); - process.exit(result.exitCode); + const result = spawnSyncFn(command); + if (result.exitCode !== 0) { + writeStderr(`${COMMAND_NAME} failed: ${formatted}\n`); + exitFn(result.exitCode); + return; + } } + + writeStdout(`${COMMAND_NAME} passed\n`); } -process.stdout.write(`${COMMAND_NAME} passed\n`); +export { COMMAND_NAME, TEST_COMMANDS }; + +if (import.meta.main) { + runVerifyM057S02(); +} diff --git a/src/jobs/fork-manager.test.ts b/src/jobs/fork-manager.test.ts index cef58eca..75252b77 100644 --- a/src/jobs/fork-manager.test.ts +++ b/src/jobs/fork-manager.test.ts @@ -273,7 +273,8 @@ describe("createForkManager", () => { forkOwner: "kodiai-bot", forkRepo: "xbmc", branch: "main", - error: expect.any(Error), + errorStatus: 409, + errorMessage: "Conflict", }, message: "Fork sync hit merge conflict", }); @@ -299,7 +300,7 @@ describe("createForkManager", () => { forkOwner: "kodiai-bot", forkRepo: "xbmc", branch: "feature/test", - error: expect.any(Error), + errorMessage: "network down", }, message: "Failed to delete fork branch (best-effort)", }); diff --git a/src/jobs/fork-manager.ts b/src/jobs/fork-manager.ts index db9d589e..6c4cb697 100644 --- a/src/jobs/fork-manager.ts +++ b/src/jobs/fork-manager.ts @@ -17,6 +17,19 @@ export interface ForkManager { const FORK_POLL_INTERVAL_MS = 2_000; const FORK_POLL_TIMEOUT_MS = 30_000; +function summarizeError(error: unknown): Record { + if (typeof error !== "object" || error === null) { + return {}; + } + + const record = error as Record; + return { + ...(typeof record.status === "number" ? { errorStatus: record.status } : {}), + ...(typeof record.code === "string" || typeof record.code === "number" ? { errorCode: record.code } : {}), + ...(typeof record.message === "string" ? { errorMessage: record.message } : {}), + }; +} + export function createForkManager(botClient: BotUserClient, logger: Logger, botPat?: string): ForkManager { if (!botClient.enabled) { return { @@ -120,7 +133,7 @@ export function createForkManager(botClient: BotUserClient, logger: Logger, botP logger.info({ forkOwner, forkRepo, branch }, "Fork synced with upstream"); } catch (error: unknown) { if (typeof error === "object" && error !== null && "status" in error && error.status === 409) { - logger.warn({ forkOwner, forkRepo, branch, error }, "Fork sync hit merge conflict"); + logger.warn({ forkOwner, forkRepo, branch, ...summarizeError(error) }, "Fork sync hit merge conflict"); throw new Error( `Merge conflict syncing fork ${forkOwner}/${forkRepo} branch ${branch} with upstream. A git-based fallback may be needed.`, ); @@ -138,7 +151,10 @@ export function createForkManager(botClient: BotUserClient, logger: Logger, botP }); logger.info({ forkOwner, forkRepo, branch }, "Deleted fork branch"); } catch (error) { - logger.warn({ forkOwner, forkRepo, branch, error }, "Failed to delete fork branch (best-effort)"); + logger.warn( + { forkOwner, forkRepo, branch, ...summarizeError(error) }, + "Failed to delete fork branch (best-effort)", + ); } }, diff --git a/src/knowledge/wiki-fetch.test.ts b/src/knowledge/wiki-fetch.test.ts index a91f221d..8357a8fe 100644 --- a/src/knowledge/wiki-fetch.test.ts +++ b/src/knowledge/wiki-fetch.test.ts @@ -17,6 +17,14 @@ describe("buildWikiApiUrl", () => { "https://kodi.wiki/api.php?foo=bar&action=parse&page=Foo+Bar", ); }); + + test("does not double-append /api.php when the base URL already targets the API path", () => { + const params = new URLSearchParams({ action: "query", format: "json" }); + + expect(buildWikiApiUrl("https://kodi.wiki/api.php?foo=bar", params)).toBe( + "https://kodi.wiki/api.php?foo=bar&action=query&format=json", + ); + }); }); describe("withWikiHeaders", () => { diff --git a/src/knowledge/wiki-fetch.ts b/src/knowledge/wiki-fetch.ts index 057a056c..7ad1bba2 100644 --- a/src/knowledge/wiki-fetch.ts +++ b/src/knowledge/wiki-fetch.ts @@ -19,7 +19,7 @@ export function buildWikiApiUrl( ): string { const url = new URL(baseUrl); const pathname = url.pathname.endsWith("/") ? url.pathname.slice(0, -1) : url.pathname; - url.pathname = `${pathname}/api.php`; + url.pathname = pathname.endsWith("/api.php") ? pathname : `${pathname}/api.php`; const mergedParams = new URLSearchParams(url.search); for (const [key, value] of params.entries()) {