From d82f63d538fb3bd8c856ed57d275dcf9e36ddab5 Mon Sep 17 00:00:00 2001 From: Rich Haines Date: Thu, 2 Apr 2026 14:07:27 +0300 Subject: [PATCH 1/4] Add @vercel/agent-readability dependency --- apps/docs/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/docs/package.json b/apps/docs/package.json index 010088af..866ade8d 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -19,6 +19,7 @@ "@streamdown/code": "^1.0.1", "@streamdown/math": "^1.0.1", "@streamdown/mermaid": "^1.0.1", + "@vercel/agent-readability": "^0.2.1", "@vercel/analytics": "^1.6.1", "@vercel/speed-insights": "^1.3.1", "ai": "^6.0.105", @@ -63,4 +64,4 @@ "tw-animate-css": "^1.4.0", "typescript": "^5.9.3" } -} +} \ No newline at end of file From aae0b55a9ca6a61dc551287d974e4cd10e0f7d7d Mon Sep 17 00:00:00 2001 From: Rich Haines Date: Thu, 2 Apr 2026 14:07:28 +0300 Subject: [PATCH 2/4] Add AI agent detection via @vercel/agent-readability --- apps/docs/proxy.ts | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/apps/docs/proxy.ts b/apps/docs/proxy.ts index 9b4a6358..1b826c9e 100644 --- a/apps/docs/proxy.ts +++ b/apps/docs/proxy.ts @@ -7,6 +7,7 @@ import { } from "next/server"; import { i18n } from "@/lib/geistdocs/i18n"; import { trackMdRequest } from "@/lib/geistdocs/md-tracking"; +import { generateNotFoundMarkdown, isAIAgent } from "@vercel/agent-readability"; const { rewrite: rewriteLLM } = rewritePath( "/*path", @@ -57,6 +58,36 @@ const proxy = (request: NextRequest, context: NextFetchEvent) => { } } + // AI agent detection — rewrite docs pages to markdown for agents + if ( + (pathname === "/docs" || pathname.startsWith("/docs/")) && + !pathname.includes("/llms.mdx/") + ) { + const agentResult = isAIAgent(request); + if (agentResult.detected && !isMarkdownPreferred(request)) { + const result = rewriteLLM(pathname); + + if (result) { + context.waitUntil( + trackMdRequest({ + path: pathname, + userAgent: request.headers.get("user-agent"), + referer: request.headers.get("referer"), + acceptHeader: request.headers.get("accept"), + requestType: "agent-rewrite", + detectionMethod: agentResult.method, + }) + ); + return NextResponse.rewrite(new URL(result, request.nextUrl)); + } + // Agent requested a non-existent docs URL + return new NextResponse(generateNotFoundMarkdown(pathname), { + headers: { "Content-Type": "text/markdown; charset=utf-8" }, + }); + } + } + + // Handle Accept header content negotiation and track the request if (isMarkdownPreferred(request)) { const result = rewriteLLM(pathname); From 60ea54bcfc5d98027f97c6af58f21ee222a6b225 Mon Sep 17 00:00:00 2001 From: molebox Date: Tue, 7 Apr 2026 14:04:33 +0200 Subject: [PATCH 3/4] Fix md-tracking types, add Vary header and discovery options - Update TrackMdRequestParams to include 'agent-rewrite' requestType and detectionMethod - Import DetectionMethod type from @vercel/agent-readability - Include detectionMethod in tracking fetch body - Add Vary: Accept header on agent-rewrite responses - Add discovery options (sitemapUrl, indexUrl) to generateNotFoundMarkdown Co-Authored-By: Claude Opus 4.6 (1M context) --- apps/docs/lib/geistdocs/md-tracking.ts | 9 +++++++-- apps/docs/proxy.ts | 21 ++++++++++++++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/apps/docs/lib/geistdocs/md-tracking.ts b/apps/docs/lib/geistdocs/md-tracking.ts index 4035976f..ad883889 100644 --- a/apps/docs/lib/geistdocs/md-tracking.ts +++ b/apps/docs/lib/geistdocs/md-tracking.ts @@ -1,13 +1,16 @@ +import type { DetectionMethod } from "@vercel/agent-readability"; import { siteId } from "@/geistdocs"; const PLATFORM_URL = "https://geistdocs.com/md-tracking"; interface TrackMdRequestParams { acceptHeader: string | null; + /** Detection method used to identify the agent (only for agent-rewrite requests) */ + detectionMethod?: DetectionMethod | null; path: string; referer: string | null; - /** How the markdown was requested: 'md-url' for direct .md URLs, 'header-negotiated' for Accept header */ - requestType?: "md-url" | "header-negotiated"; + /** How the markdown was requested: 'md-url' for direct .md URLs, 'header-negotiated' for Accept header, 'agent-rewrite' for detected AI agents */ + requestType?: "md-url" | "header-negotiated" | "agent-rewrite"; userAgent: string | null; } @@ -21,6 +24,7 @@ export async function trackMdRequest({ referer, acceptHeader, requestType, + detectionMethod, }: TrackMdRequestParams): Promise { try { const response = await fetch(PLATFORM_URL, { @@ -35,6 +39,7 @@ export async function trackMdRequest({ referer, acceptHeader, requestType, + detectionMethod, }), }); diff --git a/apps/docs/proxy.ts b/apps/docs/proxy.ts index 1b826c9e..4bf2256b 100644 --- a/apps/docs/proxy.ts +++ b/apps/docs/proxy.ts @@ -78,12 +78,23 @@ const proxy = (request: NextRequest, context: NextFetchEvent) => { detectionMethod: agentResult.method, }) ); - return NextResponse.rewrite(new URL(result, request.nextUrl)); + const response = NextResponse.rewrite(new URL(result, request.nextUrl)); + response.headers.set("Vary", "Accept"); + return response; } - // Agent requested a non-existent docs URL - return new NextResponse(generateNotFoundMarkdown(pathname), { - headers: { "Content-Type": "text/markdown; charset=utf-8" }, - }); + // Agent requested a non-existent docs URL — return helpful markdown + return new NextResponse( + generateNotFoundMarkdown(pathname, { + sitemapUrl: "/sitemap.md", + indexUrl: "/llms.txt", + }), + { + headers: { + "Content-Type": "text/markdown; charset=utf-8", + Vary: "Accept", + }, + }, + ); } } From 4ef8154e0f49b4d37668b5eb282a7d99694373e7 Mon Sep 17 00:00:00 2001 From: molebox Date: Tue, 7 Apr 2026 14:10:46 +0200 Subject: [PATCH 4/4] Update pnpm-lock.yaml Co-Authored-By: Claude Opus 4.6 (1M context) --- pnpm-lock.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d82cfd3a..72f6300c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,6 +71,9 @@ importers: '@streamdown/mermaid': specifier: ^1.0.1 version: 1.0.1(react@19.2.3) + '@vercel/agent-readability': + specifier: ^0.2.1 + version: 0.2.1(next@16.1.6(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)) '@vercel/analytics': specifier: ^1.6.1 version: 1.6.1(next@16.1.6(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3) @@ -2959,6 +2962,16 @@ packages: '@ungap/structured-clone@1.3.0': resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} + '@vercel/agent-readability@0.2.1': + resolution: {integrity: sha512-ShT7BzIS/dwKompii8tm5do+NR1g4xL5M3wM7S01xsH6yuYQ7wiTPZEcmHMFLHCsAQg45/mD0hgrufpS3NVunw==} + engines: {node: '>=20.0.0'} + hasBin: true + peerDependencies: + next: '>=14' + peerDependenciesMeta: + next: + optional: true + '@vercel/analytics@1.6.1': resolution: {integrity: sha512-oH9He/bEM+6oKlv3chWuOOcp8Y6fo6/PSro8hEkgCW3pu9/OiCXiUpRUogDh3Fs3LH2sosDrx8CxeOLBEE+afg==} peerDependencies: @@ -9198,6 +9211,10 @@ snapshots: '@ungap/structured-clone@1.3.0': {} + '@vercel/agent-readability@0.2.1(next@16.1.6(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))': + optionalDependencies: + next: 16.1.6(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@vercel/analytics@1.6.1(next@16.1.6(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)': optionalDependencies: next: 16.1.6(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)