Merge pull request #49 from agent-ecosystem/feat/skip-check-ids-follow-ups

New '--skip-checks' follow ups

Author: dacharyc

docs/reference/programmatic-api.md

@@ -31,9 +31,9 @@ Pass a second argument to configure sampling, concurrency, and thresholds:
 ```ts
 import { runChecks } from 'afdocs';
 
+// Run specific checks (include-list)
 const report = await runChecks('https://docs.example.com', {
-  checkIds: ['llms-txt-exists', 'llms-txt-valid', 'llms-txt-size'], // include-list
-  skipCheckIds: ['markdown-content-parity'], // exclude-list
+  checkIds: ['llms-txt-exists', 'llms-txt-valid', 'llms-txt-size'],
   samplingStrategy: 'deterministic',
   maxLinksToTest: 20,
   maxConcurrency: 5,
@@ -44,7 +44,12 @@ const report = await runChecks('https://docs.example.com', {
   },
 });
 
-// Or test specific pages with curated sampling:
+// Or run all checks except a few (exclude-list)
+const skipReport = await runChecks('https://docs.example.com', {
+  skipCheckIds: ['markdown-content-parity'],
+});
+
+// Or test specific pages with curated sampling
 const curatedReport = await runChecks('https://docs.example.com', {
   samplingStrategy: 'curated',
   curatedPages: [

src/runner.ts

@@ -72,6 +72,17 @@ export async function runChecks(
   const checkIds = options?.checkIds;
   const skipCheckIds = options?.skipCheckIds ?? [];
 
+  // Warn about overlapping checkIds and skipCheckIds
+  if (checkIds && checkIds.length > 0 && skipCheckIds.length > 0) {
+    const overlap = skipCheckIds.filter((id) => checkIds.includes(id));
+    if (overlap.length > 0) {
+      console.warn(
+        `Warning: ${overlap.join(', ')} listed in both --checks and --skip-checks. ` +
+          `These checks will be skipped.`,
+      );
+    }
+  }
+
   const results: CheckResult[] = [];
 
   for (const check of allChecks) {

test/unit/runner.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect, beforeAll } from 'vitest';
+import { describe, it, expect, beforeAll, vi } from 'vitest';
 import { http, HttpResponse } from 'msw';
 import { setupServer } from 'msw/node';
 import { createContext, normalizeUrl, runChecks } from '../../src/runner.js';
@@ -296,6 +296,7 @@ describe('runner', () => {
   });
 
   it('skips checks listed in skipCheckIds without running them', async () => {
+    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
     server.use(
       http.get('http://skip-ids.local/llms.txt', () => new HttpResponse(null, { status: 404 })),
       http.get(
@@ -326,9 +327,11 @@ describe('runner', () => {
     expect(size).toBeDefined();
     expect(size?.status).toBe('skip');
     expect(size?.message).toContain('dependency');
+    warnSpy.mockRestore();
   });
 
   it('skipCheckIds does not cascade-skip dependent checks', async () => {
+    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
     const content = `# Test\n\n> Summary.\n\n## Links\n\n- [A](http://skip-dep.local/a): A\n`;
     server.use(
       http.get('http://skip-dep.local/llms.txt', () => HttpResponse.text(content)),
@@ -355,6 +358,54 @@ describe('runner', () => {
     const valid = report.results.find((r) => r.id === 'llms-txt-valid');
     expect(valid).toBeDefined();
     expect(valid?.message).not.toContain('dependency');
+    warnSpy.mockRestore();
+  });
+
+  it('warns when checkIds and skipCheckIds overlap', async () => {
+    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+
+    server.use(
+      http.get('http://overlap.local/llms.txt', () => new HttpResponse(null, { status: 404 })),
+      http.get('http://overlap.local/docs/llms.txt', () => new HttpResponse(null, { status: 404 })),
+    );
+
+    const report = await runChecks('http://overlap.local', {
+      checkIds: ['llms-txt-exists', 'llms-txt-valid'],
+      skipCheckIds: ['llms-txt-valid'],
+      requestDelay: 0,
+    });
+
+    expect(warnSpy).toHaveBeenCalledOnce();
+    expect(warnSpy.mock.calls[0][0]).toContain('llms-txt-valid');
+    expect(warnSpy.mock.calls[0][0]).toContain('--checks');
+    expect(warnSpy.mock.calls[0][0]).toContain('--skip-checks');
+
+    // The overlapping check should still be skipped (skip wins)
+    const skipped = report.results.find((r) => r.id === 'llms-txt-valid');
+    expect(skipped?.status).toBe('skip');
+
+    warnSpy.mockRestore();
+  });
+
+  it('does not warn when checkIds and skipCheckIds do not overlap', async () => {
+    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+
+    server.use(
+      http.get('http://no-overlap.local/llms.txt', () => new HttpResponse(null, { status: 404 })),
+      http.get(
+        'http://no-overlap.local/docs/llms.txt',
+        () => new HttpResponse(null, { status: 404 }),
+      ),
+    );
+
+    await runChecks('http://no-overlap.local', {
+      checkIds: ['llms-txt-exists', 'llms-txt-valid'],
+      skipCheckIds: ['llms-txt-size'],
+      requestDelay: 0,
+    });
+
+    expect(warnSpy).not.toHaveBeenCalled();
+    warnSpy.mockRestore();
   });
 
   it('includes timestamp and url in report', async () => {