docs: improvement for agent friendliness + add pipeline

.github/workflows/agent-docs.yml

@@ -0,0 +1,62 @@
+name: Agent-Friendly Docs
+
+on:
+  pull_request:
+    branches: [develop]
+    paths:
+      - www/apps/book/**
+
+jobs:
+  agent-docs-check:
+    if: ${{ startsWith(github.head_ref, 'docs/') }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: "yarn"
+
+      - name: Install dependencies
+        uses: ./.github/actions/cache-deps
+        with:
+          extension: docs
+
+      - name: Install www dependencies
+        working-directory: www
+        run: yarn install
+
+      - name: Build book app
+        working-directory: www
+        run: yarn build:docs
+        env:
+          NEXT_PUBLIC_BASE_URL: "http://localhost:3000"
+          NEXT_PUBLIC_DOCS_URL: "http://localhost:3000"
+          NEXT_PUBLIC_RESOURCES_URL: "http://localhost:3000"
+          NEXT_PUBLIC_USER_GUIDE_URL: "http://localhost:3000"
+          NEXT_PUBLIC_ENV: CI
+          NODE_ENV: production
+
+      - name: Start book app
+        working-directory: www/apps/book
+        run: yarn start &
+        env:
+          NEXT_PUBLIC_BASE_URL: "http://localhost:3000"
+
+      - name: Wait for server
+        run: npx wait-on http://localhost:3000 --timeout 60000
+
+      - name: Run agent-friendly docs checks
+        working-directory: www/apps/book
+        run: yarn test:agent-docs
+        timeout-minutes: 5

www/apps/bloom/app/layout.tsx

@@ -39,6 +39,11 @@ export const metadata: Metadata = {
       },
     ],
   },
+  alternates: {
+    types: {
+      "text/plain": "/llms.txt",
+    },
+  },
   other: {
     "algolia-site-verification":
       process.env.NEXT_PUBLIC_ALGOLIA_SITE_VERIFICATION || "",

www/apps/bloom/app/md-content/[[...slug]]/route.ts

@@ -90,6 +90,7 @@ export async function GET(req: NextRequest, { params }: Params) {
   return new NextResponse(cleanMdContent + PLAINTEXT_DOC_MESSAGE, {
     headers: {
       "Content-Type": "text/markdown",
+      "Cache-Control": "public, max-age=3600, must-revalidate",
     },
     status: 200,
   })

www/apps/bloom/middleware.ts

@@ -0,0 +1,16 @@
+import { NextRequest, NextResponse } from "next/server"
+
+export function middleware(req: NextRequest) {
+  const accept = req.headers.get("accept") ?? ""
+
+  if (accept.includes("text/markdown") || accept.includes("text/plain")) {
+    const { pathname } = req.nextUrl
+    const url = req.nextUrl.clone()
+    url.pathname = `/md-content${pathname === "/" ? "" : pathname}`
+    return NextResponse.rewrite(url)
+  }
+}
+
+export const config = {
+  matcher: "/((?!_next/static|_next/image|favicon.ico|md-content).*)",
+}

www/apps/bloom/next.config.mjs

@@ -145,14 +145,11 @@ const nextConfig = {
           destination: "/md-content/:path*",
         },
         {
-          source: "/:path*",
-          has: [
-            {
-              type: "header",
-              key: "Accept",
-              value: ".*(text/markdown|text/plain).*",
-            },
-          ],
+          source: "/:path*/index.md",
+          destination: "/md-content/:path*",
+        },
+        {
+          source: "/:path*.md",
           destination: "/md-content/:path*",
         },
       ],

www/apps/book/agent-docs.config.yml

@@ -0,0 +1,27 @@
+url: http://localhost:3000
+
+checks:
+  - llms-txt-exists
+  - llms-txt-valid
+  - llms-txt-size
+  - llms-txt-links-resolve
+  - llms-txt-links-markdown
+  - llms-txt-directive
+  - markdown-url-support
+  - content-negotiation
+  - rendering-strategy
+  - page-size-markdown
+  - page-size-html
+  - content-start-position
+  - tabbed-content-serialization
+  - section-header-quality
+  - markdown-code-fence-validity
+  - http-status-codes
+  - redirect-behavior
+  - llms-txt-freshness
+  - auth-gate-detection
+  - auth-alternative-access
+
+options:
+  maxLinksToTest: 1
+  samplingStrategy: deterministic

www/apps/book/agent-docs.test.ts

@@ -0,0 +1,4 @@
+// @ts-ignore
+import { describeAgentDocsPerCheck } from "afdocs/helpers"
+
+describeAgentDocsPerCheck()

www/apps/book/app/layout.tsx

@@ -37,6 +37,11 @@ export const metadata: Metadata = {
       },
     ],
   },
+  alternates: {
+    types: {
+      "text/plain": "/llms.txt",
+    },
+  },
 }
 
 export default function RootLayout({

www/apps/book/app/md-content/[[...slug]]/route.ts

@@ -20,14 +20,15 @@ export async function GET(req: NextRequest, { params }: Params) {
   const { slug = ["/"] } = await params
 
   if (slug[0] === "/") {
-    const llmsFile = readFileSync(
-      path.join(process.cwd(), "public", "llms.txt"),
+    const homepageFile = readFileSync(
+      path.join(process.cwd(), "public", "homepage.md"),
       "utf-8"
     )
 
-    return new NextResponse(llmsFile + PLAINTEXT_DOC_MESSAGE, {
+    return new NextResponse(homepageFile + PLAINTEXT_DOC_MESSAGE, {
       headers: {
         "Content-Type": "text/markdown",
+        "Cache-Control": "public, max-age=3600, must-revalidate",
       },
       status: 200,
     })
@@ -97,6 +98,7 @@ export async function GET(req: NextRequest, { params }: Params) {
   return new NextResponse(cleanMdContent + PLAINTEXT_DOC_MESSAGE, {
     headers: {
       "Content-Type": "text/markdown",
+      "Cache-Control": "public, max-age=3600, must-revalidate",
     },
     status: 200,
   })

www/apps/book/app/page.tsx

@@ -1,3 +1,5 @@
+export const revalidate = 3600
+
 import clsx from "clsx"
 import { AiAssistantChatWindow, MainNav, RootProviders } from "docs-ui"
 import Providers from "../providers"

www/apps/book/middleware.ts

@@ -0,0 +1,36 @@
+import { NextRequest, NextResponse } from "next/server"
+
+const EXCLUDED_PREFIXES = [
+  "/resources",
+  "/api",
+  "/ui",
+  "/user-guide",
+  "/cloud",
+  "/md-content",
+  "/_next",
+  "/favicon",
+]
+
+export function middleware(req: NextRequest) {
+  const accept = req.headers.get("accept") ?? ""
+
+  if (
+    accept.includes("text/markdown") ||
+    accept.includes("text/plain")
+  ) {
+    const { pathname } = req.nextUrl
+    const isExcluded = EXCLUDED_PREFIXES.some((prefix) =>
+      pathname.startsWith(prefix)
+    )
+
+    if (!isExcluded) {
+      const url = req.nextUrl.clone()
+      url.pathname = `/md-content${pathname === "/" ? "" : pathname}`
+      return NextResponse.rewrite(url)
+    }
+  }
+}
+
+export const config = {
+  matcher: "/((?!_next/static|_next/image|favicon.ico).*)",
+}

www/apps/book/next.config.mjs

@@ -130,16 +130,14 @@ const nextConfig = {
           destination: "/md-content/:path*",
         },
         {
-          source: "/:path((?!resources|api|ui|user-guide|cloud).*)*",
-          has: [
-            {
-              type: "header",
-              key: "Accept",
-              value: ".*(text/markdown|text/plain).*",
-            },
-          ],
+          source: "/:path((?!resources|api|ui|user-guide|cloud).*)index.md",
+          destination: "/md-content/:path*",
+        },
+        {
+          source: "/:path((?!resources|api|ui|user-guide|cloud).*).md",
           destination: "/md-content/:path*",
         },
+
       ],
       fallback: [
         {
@@ -227,6 +225,19 @@ const nextConfig = {
       ],
     }
   },
+  async headers() {
+    return [
+      {
+        source: "/",
+        headers: [
+          {
+            key: "Cache-Control",
+            value: "public, max-age=3600, must-revalidate",
+          },
+        ],
+      },
+    ]
+  },
   redirects: async () => {
     const result = await redirects()
 

www/apps/book/package.json

@@ -11,7 +11,8 @@
     "start:monorepo": "yarn start -p 3001",
     "lint": "next lint --fix",
     "lint:content": "eslint --no-config-lookup -c .content.eslint.mjs app/**/*.mdx --fix",
-    "prep": "node ./scripts/prepare.mjs"
+    "prep": "node ./scripts/prepare.mjs",
+    "test:agent-docs": "npx vitest run --config vitest.config.ts"
   },
   "dependencies": {
     "@mdx-js/loader": "^3.1.0",
@@ -38,6 +39,7 @@
     "@types/node": "^20",
     "@types/react": "19.2.9",
     "@types/react-dom": "19.2.3",
+    "afdocs": "latest",
     "autoprefixer": "^10.0.1",
     "build-scripts": "*",
     "eslint": "^9.13.0",