diff --git a/apps/docs/lib/geistdocs/md-tracking.ts b/apps/docs/lib/geistdocs/md-tracking.ts index 4035976f..ad883889 100644 --- a/apps/docs/lib/geistdocs/md-tracking.ts +++ b/apps/docs/lib/geistdocs/md-tracking.ts @@ -1,13 +1,16 @@ +import type { DetectionMethod } from "@vercel/agent-readability"; import { siteId } from "@/geistdocs"; const PLATFORM_URL = "https://geistdocs.com/md-tracking"; interface TrackMdRequestParams { acceptHeader: string | null; + /** Detection method used to identify the agent (only for agent-rewrite requests) */ + detectionMethod?: DetectionMethod | null; path: string; referer: string | null; - /** How the markdown was requested: 'md-url' for direct .md URLs, 'header-negotiated' for Accept header */ - requestType?: "md-url" | "header-negotiated"; + /** How the markdown was requested: 'md-url' for direct .md URLs, 'header-negotiated' for Accept header, 'agent-rewrite' for detected AI agents */ + requestType?: "md-url" | "header-negotiated" | "agent-rewrite"; userAgent: string | null; } @@ -21,6 +24,7 @@ export async function trackMdRequest({ referer, acceptHeader, requestType, + detectionMethod, }: TrackMdRequestParams): Promise { try { const response = await fetch(PLATFORM_URL, { @@ -35,6 +39,7 @@ export async function trackMdRequest({ referer, acceptHeader, requestType, + detectionMethod, }), }); diff --git a/apps/docs/package.json b/apps/docs/package.json index 010088af..866ade8d 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -19,6 +19,7 @@ "@streamdown/code": "^1.0.1", "@streamdown/math": "^1.0.1", "@streamdown/mermaid": "^1.0.1", + "@vercel/agent-readability": "^0.2.1", "@vercel/analytics": "^1.6.1", "@vercel/speed-insights": "^1.3.1", "ai": "^6.0.105", @@ -63,4 +64,4 @@ "tw-animate-css": "^1.4.0", "typescript": "^5.9.3" } -} +} \ No newline at end of file diff --git a/apps/docs/proxy.ts b/apps/docs/proxy.ts index 9b4a6358..4bf2256b 100644 --- a/apps/docs/proxy.ts +++ b/apps/docs/proxy.ts @@ -7,6 +7,7 @@ import { } from "next/server"; import { i18n } from "@/lib/geistdocs/i18n"; import { trackMdRequest } from "@/lib/geistdocs/md-tracking"; +import { generateNotFoundMarkdown, isAIAgent } from "@vercel/agent-readability"; const { rewrite: rewriteLLM } = rewritePath( "/*path", @@ -57,6 +58,47 @@ const proxy = (request: NextRequest, context: NextFetchEvent) => { } } + // AI agent detection — rewrite docs pages to markdown for agents + if ( + (pathname === "/docs" || pathname.startsWith("/docs/")) && + !pathname.includes("/llms.mdx/") + ) { + const agentResult = isAIAgent(request); + if (agentResult.detected && !isMarkdownPreferred(request)) { + const result = rewriteLLM(pathname); + + if (result) { + context.waitUntil( + trackMdRequest({ + path: pathname, + userAgent: request.headers.get("user-agent"), + referer: request.headers.get("referer"), + acceptHeader: request.headers.get("accept"), + requestType: "agent-rewrite", + detectionMethod: agentResult.method, + }) + ); + const response = NextResponse.rewrite(new URL(result, request.nextUrl)); + response.headers.set("Vary", "Accept"); + return response; + } + // Agent requested a non-existent docs URL — return helpful markdown + return new NextResponse( + generateNotFoundMarkdown(pathname, { + sitemapUrl: "/sitemap.md", + indexUrl: "/llms.txt", + }), + { + headers: { + "Content-Type": "text/markdown; charset=utf-8", + Vary: "Accept", + }, + }, + ); + } + } + + // Handle Accept header content negotiation and track the request if (isMarkdownPreferred(request)) { const result = rewriteLLM(pathname); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d82cfd3a..72f6300c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,6 +71,9 @@ importers: '@streamdown/mermaid': specifier: ^1.0.1 version: 1.0.1(react@19.2.3) + '@vercel/agent-readability': + specifier: ^0.2.1 + version: 0.2.1(next@16.1.6(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)) '@vercel/analytics': specifier: ^1.6.1 version: 1.6.1(next@16.1.6(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3) @@ -2959,6 +2962,16 @@ packages: '@ungap/structured-clone@1.3.0': resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} + '@vercel/agent-readability@0.2.1': + resolution: {integrity: sha512-ShT7BzIS/dwKompii8tm5do+NR1g4xL5M3wM7S01xsH6yuYQ7wiTPZEcmHMFLHCsAQg45/mD0hgrufpS3NVunw==} + engines: {node: '>=20.0.0'} + hasBin: true + peerDependencies: + next: '>=14' + peerDependenciesMeta: + next: + optional: true + '@vercel/analytics@1.6.1': resolution: {integrity: sha512-oH9He/bEM+6oKlv3chWuOOcp8Y6fo6/PSro8hEkgCW3pu9/OiCXiUpRUogDh3Fs3LH2sosDrx8CxeOLBEE+afg==} peerDependencies: @@ -9198,6 +9211,10 @@ snapshots: '@ungap/structured-clone@1.3.0': {} + '@vercel/agent-readability@0.2.1(next@16.1.6(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))': + optionalDependencies: + next: 16.1.6(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@vercel/analytics@1.6.1(next@16.1.6(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)': optionalDependencies: next: 16.1.6(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)