fix(review): act on docs-vfs review findings

Multi-agent review of the docs/ VFS change. Applied the behavior-preserving
fixes plus two agent-facing bugs that made real pages unreadable.

- docs read no longer hard-fails on oversized pages. Six+ live integration
  references exceed the inline cap (github.mdx is 354KB, sportmonks 513KB),
  so a plain read of them ALWAYS failed and cost a second fetch to recover.
  Truncate to the largest whole-line prefix that fits, keep the true
  totalLines, and tell the model how to page. An explicit offset/limit that
  still overflows is still an error — that one is a caller mistake.
- classify docs fetch failures. Everything collapsed to null, so a permanent
  404 was reported to the agent as "temporarily unavailable, retry shortly",
  inviting a retry loop on a page that will never exist. 4xx (except 429) is
  now permanent and says so; 5xx/429/network/timeout keep the retry wording.
- register search_documentation as a transitional alias for search_docs.
  sim and mothership deploy independently and the rename deleted the old id
  on both sides, so BOTH deploy orders broke docs lookup for the window
  between them. Old params are a subset of the new. Remove once both ship.
- extract the index-page fold (X/index.mdx <-> X.mdx) into docs-path.ts. It
  was re-derived in three places — the manifest generator, the
  source_document reverse mapping, and the search scope filter — which is
  the hand-synced-duplicate shape that has drifted in this repo before.
- grepDocsPage now goes through grepReadResult, the primitive files/ and
  uploads/ grep already use, instead of calling grep directly.
- couldMatchDocsScope delegates to isDocsPath; the bodies were identical.
- drop the dead 'docs' member from AgentContextType.

Tests: 404-vs-5xx-vs-429 classification, network failure, and a docs-path
round-trip asserting one source candidate reproduces every manifest entry.

Verified: tsc clean, 932 copilot tests, biome clean, docs-manifest:check,
check:utils and check:api-validation:strict both pass.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: j15z

apps/sim/lib/copilot/chat/process-contents.ts

@@ -42,7 +42,6 @@ type AgentContextType =
   | 'table'
   | 'file'
   | 'workflow_block'
-  | 'docs'
   | 'folder'
   | 'filefolder'
   | 'active_resource'

apps/sim/lib/copilot/docs/docs-corpus.test.ts

@@ -93,10 +93,29 @@ describe('readDocsPage', () => {
     expect(fetchMock).not.toHaveBeenCalled()
   })
 
-  it('surfaces a docs-site failure as a retryable error', async () => {
+  it('surfaces a docs-site outage as a retryable error', async () => {
     fetchMock.mockResolvedValue({ ok: false, status: 502, text: async () => '' })
     await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/temporarily unavailable/)
   })
+
+  it('treats a network failure as retryable', async () => {
+    fetchMock.mockRejectedValue(new Error('socket hang up'))
+    await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/temporarily unavailable/)
+  })
+
+  it('reports a page the site no longer serves as permanent, not retryable', async () => {
+    fetchMock.mockResolvedValue({ ok: false, status: 404, text: async () => '' })
+    const error = await readDocsPage(`docs/${SAMPLE_PAGE}`).catch((e) => e)
+    expect(error).toBeInstanceOf(DocsCorpusError)
+    expect(error.message).toMatch(/does not serve it/)
+    expect(error.message).toMatch(/retrying will not help/)
+    expect(error.message).not.toMatch(/temporarily unavailable/)
+  })
+
+  it('still treats 429 as retryable rather than permanent', async () => {
+    fetchMock.mockResolvedValue({ ok: false, status: 429, text: async () => '' })
+    await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/temporarily unavailable/)
+  })
 })
 
 describe('grepDocsPage', () => {

apps/sim/lib/copilot/docs/docs-corpus.ts

@@ -1,8 +1,9 @@
 import { createLogger } from '@sim/logger'
 import { toError } from '@sim/utils/errors'
+import { foldDocsIndexPath } from '@/lib/copilot/docs/docs-path'
 import { DOCS_MANIFEST } from '@/lib/copilot/generated/docs-manifest'
 import type { GrepCountEntry, GrepMatch, GrepOptions } from '@/lib/copilot/vfs/operations'
-import { glob as globPaths, grep as grepFiles } from '@/lib/copilot/vfs/operations'
+import { glob as globPaths, grepReadResult } from '@/lib/copilot/vfs/operations'
 
 const logger = createLogger('DocsCorpus')
 
@@ -56,12 +57,11 @@ export function isDocsPath(path: string | undefined): boolean {
  * True when a glob `pattern` could match the docs corpus. Like `uploads/` and
  * `recently-deleted/`, the corpus is opt-in: only a pattern that explicitly
  * starts with `docs/` (or is exactly `docs`) sees it, so a broad `**` glob never
- * drags 300+ doc pages into the result.
+ * drags 300+ doc pages into the result. Same rule as {@link isDocsPath}; the
+ * separate name reads correctly at the glob call site.
  */
 export function couldMatchDocsScope(pattern: string | undefined): boolean {
-  if (!pattern) return false
-  const normalized = normalize(pattern)
-  return normalized === 'docs' || normalized.startsWith(DOCS_PREFIX)
+  return isDocsPath(pattern)
 }
 
 /** Manifest paths (and their virtual directories) matching an explicit `docs/` pattern. */
@@ -82,7 +82,7 @@ export function isDocsPage(path: string): boolean {
  */
 export function docsPathForSourceDocument(sourceDocument: string | null): string | null {
   if (!sourceDocument) return null
-  const path = `${DOCS_PREFIX}${sourceDocument.replace(/^\/+/, '').replace(/\/index\.mdx$/, '.mdx')}`
+  const path = `${DOCS_PREFIX}${foldDocsIndexPath(sourceDocument.replace(/^\/+/, ''))}`
   return docsKeyView.has(path) ? path : null
 }
 
@@ -108,9 +108,16 @@ export interface DocsPage {
  * to its raw-markdown route), so no mapping table is needed. Returns null when
  * the page is not in the manifest or the site does not serve it.
  */
-async function fetchDocsPage(path: string): Promise<string | null> {
+type DocsFetchResult =
+  | { outcome: 'ok'; content: string }
+  /** The site will not serve this path however many times we ask. */
+  | { outcome: 'missing' }
+  /** Transient: 5xx, 429, network error, or timeout. */
+  | { outcome: 'unavailable' }
+
+async function fetchDocsPage(path: string): Promise<DocsFetchResult> {
   const key = normalize(path)
-  if (!docsKeyView.has(key)) return null
+  if (!docsKeyView.has(key)) return { outcome: 'missing' }
   const url = `${DOCS_BASE_URL}/${key.slice(DOCS_PREFIX.length)}`
   try {
     const response = await fetch(url, {
@@ -119,12 +126,13 @@ async function fetchDocsPage(path: string): Promise<string | null> {
     })
     if (!response.ok) {
       logger.warn('Docs page fetch returned a non-OK status', { url, status: response.status })
-      return null
+      const permanent = response.status >= 400 && response.status < 500 && response.status !== 429
+      return { outcome: permanent ? 'missing' : 'unavailable' }
     }
-    return await response.text()
+    return { outcome: 'ok', content: await response.text() }
   } catch (err) {
     logger.warn('Docs page fetch failed', { url, error: toError(err).message })
-    return null
+    return { outcome: 'unavailable' }
   }
 }
 
@@ -144,13 +152,18 @@ export async function readDocsPage(path: string): Promise<DocsPage> {
       `Docs page not found: ${path}. Use glob("docs/**") to list the docs corpus.`
     )
   }
-  const content = await fetchDocsPage(key)
-  if (content === null) {
+  const result = await fetchDocsPage(key)
+  if (result.outcome === 'missing') {
+    throw new DocsCorpusError(
+      `${key} is in the docs index but ${DOCS_BASE_URL} does not serve it — the page was likely moved or removed. Use glob("docs/**") to find the current path; retrying will not help.`
+    )
+  }
+  if (result.outcome === 'unavailable') {
     throw new DocsCorpusError(
       `Could not load ${key} from ${DOCS_BASE_URL} — the docs site is temporarily unavailable. Retry shortly.`
     )
   }
-  return { content, totalLines: content.split('\n').length }
+  return { content: result.content, totalLines: result.content.split('\n').length }
 }
 
 /**
@@ -170,5 +183,5 @@ export async function grepDocsPage(
     )
   }
   const page = await readDocsPage(key)
-  return grepFiles(new Map([[key, page.content]]), pattern, undefined, options)
+  return grepReadResult(key, page, pattern, key, options)
 }

apps/sim/lib/copilot/docs/docs-path.test.ts

@@ -0,0 +1,35 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { docsSourceCandidates, foldDocsIndexPath } from '@/lib/copilot/docs/docs-path'
+import { DOCS_MANIFEST } from '@/lib/copilot/generated/docs-manifest'
+
+describe('foldDocsIndexPath', () => {
+  it('folds a section overview onto the section path', () => {
+    expect(foldDocsIndexPath('workflows/index.mdx')).toBe('workflows.mdx')
+    expect(foldDocsIndexPath('platform/enterprise/index.mdx')).toBe('platform/enterprise.mdx')
+  })
+
+  it('leaves a plain page untouched', () => {
+    expect(foldDocsIndexPath('workflows/blocks/agent.mdx')).toBe('workflows/blocks/agent.mdx')
+    expect(foldDocsIndexPath('agents.mdx')).toBe('agents.mdx')
+  })
+
+  it('does not fold a page merely named index', () => {
+    expect(foldDocsIndexPath('index.mdx')).toBe('index.mdx')
+  })
+})
+
+describe('docsSourceCandidates', () => {
+  it('is the inverse of the fold — one candidate always reproduces the input', () => {
+    for (const publicPath of DOCS_MANIFEST) {
+      const candidates = docsSourceCandidates(publicPath)
+      expect(candidates.map(foldDocsIndexPath)).toContain(publicPath)
+    }
+  })
+
+  it('offers both on-disk layouts for a section path', () => {
+    expect(docsSourceCandidates('workflows.mdx')).toEqual(['workflows.mdx', 'workflows/index.mdx'])
+  })
+})

apps/sim/lib/copilot/docs/docs-path.ts

@@ -0,0 +1,36 @@
+/**
+ * The single definition of how a docs source file maps onto its public path.
+ *
+ * Fumadocs folds a section's `index.mdx` into the section URL itself, so
+ * `workflows/index.mdx` on disk is `/workflows` on the site (and
+ * `/workflows/index.mdx` is a 404). Three places need that rule — the manifest
+ * generator, the `source_document` -> VFS reverse mapping, and the vector
+ * search's scope filter — and hand-syncing it has bitten this repo before, so
+ * it lives here.
+ *
+ * Deliberately dependency-free: `scripts/sync-docs-manifest.ts` imports this by
+ * relative path, and it must not pull in the manifest it generates.
+ */
+
+/** Suffix that marks a section overview page on disk. */
+export const DOCS_INDEX_SUFFIX = '/index.mdx'
+
+/**
+ * Fold an `en`-relative mdx file path onto its public path — the value used as
+ * both the `docs/`-relative VFS path and the docs.sim.ai URL path.
+ */
+export function foldDocsIndexPath(mdxPath: string): string {
+  return mdxPath.endsWith(DOCS_INDEX_SUFFIX)
+    ? `${mdxPath.slice(0, -DOCS_INDEX_SUFFIX.length)}.mdx`
+    : mdxPath
+}
+
+/**
+ * The inverse of {@link foldDocsIndexPath}: the on-disk file names a public
+ * path could have come from. A page is stored either as `<stem>.mdx` or, when
+ * it is a section overview, as `<stem>/index.mdx`.
+ */
+export function docsSourceCandidates(publicPath: string): [string, string] {
+  const stem = publicPath.replace(/\.mdx$/, '')
+  return [`${stem}.mdx`, `${stem}${DOCS_INDEX_SUFFIX}`]
+}

apps/sim/lib/copilot/docs/docs-search.ts

@@ -3,6 +3,7 @@ import { docsEmbeddings } from '@sim/db/schema'
 import { createLogger } from '@sim/logger'
 import { and, eq, like, notLike, or, sql } from 'drizzle-orm'
 import { docsPathForSourceDocument, isDocsDir, isDocsPage } from '@/lib/copilot/docs/docs-corpus'
+import { docsSourceCandidates } from '@/lib/copilot/docs/docs-path'
 import { generateSearchEmbedding } from '@/lib/knowledge/embeddings'
 
 const logger = createLogger('DocsSearch')
@@ -65,10 +66,10 @@ function scopeCondition(path?: string) {
 
   if (isDocsPage(normalized)) {
     // One page: on disk it is either `<tail>.mdx` or `<tail>/index.mdx`.
-    const stem = tail.replace(/\.mdx$/, '')
+    const [pageFile, indexFile] = docsSourceCandidates(tail)
     return or(
-      eq(docsEmbeddings.sourceDocument, `${stem}.mdx`),
-      eq(docsEmbeddings.sourceDocument, `${stem}/index.mdx`)
+      eq(docsEmbeddings.sourceDocument, pageFile),
+      eq(docsEmbeddings.sourceDocument, indexFile)
     )
   }
 

apps/sim/lib/copilot/tools/handlers/vfs.ts

@@ -87,6 +87,33 @@ function hasModelAttachment(result: unknown): boolean {
   )
 }
 
+/**
+ * Trim an oversized docs page to the largest whole-line prefix that fits the
+ * inline budget, preserving the true `totalLines` so the model can page through
+ * the rest with offset/limit.
+ */
+function truncateDocsPageToInlineCap(page: { content: string; totalLines: number }): {
+  output: { content: string; totalLines: number }
+  returnedLines: number
+} {
+  const lines = page.content.split('\n')
+  const notice = (shown: number) =>
+    `\n\n[Page truncated: showing lines 1-${shown} of ${page.totalLines}. Grep this path for the section you need, then read with offset/limit.]`
+
+  let kept = lines.length
+  let content = page.content
+  while (kept > 0) {
+    content = `${lines.slice(0, kept).join('\n')}${notice(kept)}`
+    if (
+      serializedResultSize({ content, totalLines: page.totalLines }) <= TOOL_RESULT_MAX_INLINE_CHARS
+    ) {
+      break
+    }
+    kept = Math.floor(kept / 2)
+  }
+  return { output: { content, totalLines: page.totalLines }, returnedLines: kept }
+}
+
 export async function executeVfsGrep(
   params: Record<string, unknown>,
   context: ExecutionContext
@@ -274,10 +301,24 @@ export async function executeVfsRead(
       const page = await readDocsPage(path)
       const windowed = applyWindow(page)
       if (serializedResultSize(windowed) > TOOL_RESULT_MAX_INLINE_CHARS) {
-        return {
-          success: false,
-          error: `${path} is too large to return inline. Grep that one page for the relevant section, then retry read with offset/limit.`,
+        // Several real docs pages (the largest integration references) exceed the
+        // inline cap, so failing here would make a plain read of them always fail
+        // and cost a second fetch to recover. Truncate to what fits instead and
+        // tell the model how to page — but only when it did not ask for a window,
+        // since an explicit offset/limit that still overflows is a caller error.
+        if (offset !== undefined || limit !== undefined) {
+          return {
+            success: false,
+            error: `${path} is still too large over the requested window. Narrow offset/limit, or grep this page for the section you need.`,
+          }
         }
+        const truncated = truncateDocsPageToInlineCap(page)
+        logger.debug('vfs_read truncated oversized docs page', {
+          path,
+          totalLines: page.totalLines,
+          returnedLines: truncated.returnedLines,
+        })
+        return { success: true, output: truncated.output }
       }
       logger.debug('vfs_read resolved docs page', { path, totalLines: page.totalLines })
       return { success: true, output: windowed }

apps/sim/lib/copilot/tools/server/router.ts

@@ -165,6 +165,11 @@ const baseServerToolRegistry: Record<string, BaseServerTool> = {
   [queryLogsServerTool.name]: queryLogsServerTool,
   [getJobLogsServerTool.name]: getJobLogsServerTool,
   [searchDocsServerTool.name]: searchDocsServerTool,
+  // Transitional alias: sim and mothership deploy independently, so during the
+  // rollout of the search_documentation -> search_docs rename one side is still
+  // emitting the old id. The old params are a subset of the new, so routing them
+  // here is safe. Remove once both repos have shipped the rename.
+  search_documentation: searchDocsServerTool,
   [searchOnlineServerTool.name]: searchOnlineServerTool,
   [setEnvironmentVariablesServerTool.name]: setEnvironmentVariablesServerTool,
   [getCredentialsServerTool.name]: getCredentialsServerTool,

scripts/sync-docs-manifest.ts

@@ -26,6 +26,7 @@
 import { readdir, readFile, writeFile } from 'node:fs/promises'
 import { dirname, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
+import { foldDocsIndexPath } from '../apps/sim/lib/copilot/docs/docs-path'
 import { formatGeneratedSource } from './format-generated-source'
 
 const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url))
@@ -55,7 +56,7 @@ async function collectMdxPaths(dir: string, prefix = ''): Promise<string[]> {
 /** Map an `en`-relative mdx file path to its docs.sim.ai URL path, or null to drop it. */
 function toDocsPath(mdxPath: string): string | null {
   if (mdxPath === 'index.mdx') return null
-  return mdxPath.replace(/\/index\.mdx$/, '.mdx')
+  return foldDocsIndexPath(mdxPath)
 }
 
 function render(paths: string[]): string {