fix(copilot): explain a short or empty search_docs result set

The SQL LIMIT is applied before the similarity-threshold and liveness
filters, so search_docs can return fewer hits than topK — or none, when
every candidate was filtered. An empty array is indistinguishable from "the
documentation does not cover this", which sends the agent off to guess
instead of rephrasing or falling back to glob.

searchDocs now returns the drop counts alongside the results, and the tool
attaches a note when anything was dropped: how many candidates the index
returned, why they went, and what to try next. Silent on the common path.

This does not change which rows are returned or how many — the ordering
issue behind the shortfall is a pre-existing bug the deleted
search-documentation.ts had too, and pushing the threshold into SQL is its
own change.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: j15z

apps/sim/lib/copilot/docs/docs-search.test.ts

@@ -124,7 +124,7 @@ describe('searchDocs results', () => {
         similarity: 0.8,
       },
     ]
-    const results = await searchDocs('cron')
+    const { results } = await searchDocs('cron')
     expect(results).toEqual([
       {
         path: 'docs/workflows.mdx',
@@ -153,7 +153,7 @@ describe('searchDocs results', () => {
         similarity: 0.9,
       },
     ]
-    expect(await searchDocs('cron')).toEqual([])
+    expect((await searchDocs('cron')).results).toEqual([])
   })
 
   it('drops chunks below the similarity threshold', async () => {
@@ -166,6 +166,55 @@ describe('searchDocs results', () => {
         similarity: 0.1,
       },
     ]
-    expect(await searchDocs('cron')).toEqual([])
+    expect((await searchDocs('cron')).results).toEqual([])
+  })
+})
+
+describe('searchDocs shortfall reporting', () => {
+  beforeEach(() => {
+    capturedWhere.value = undefined
+    mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [0.1, 0.2] })
+  })
+
+  it('counts why candidates were dropped so an empty set is explainable', async () => {
+    mockRows.value = [
+      {
+        chunkText: 'a',
+        sourceDocument: 'agents.mdx',
+        sourceLink: 'x',
+        headerText: 'h',
+        similarity: 0.1,
+      },
+      {
+        chunkText: 'b',
+        sourceDocument: 'deleted-page.mdx',
+        sourceLink: 'y',
+        headerText: 'h',
+        similarity: 0.9,
+      },
+    ]
+    const outcome = await searchDocs('cron')
+    expect(outcome).toEqual({
+      results: [],
+      candidatesConsidered: 2,
+      droppedBelowThreshold: 1,
+      droppedStale: 1,
+    })
+  })
+
+  it('reports no drops when every candidate survives', async () => {
+    mockRows.value = [
+      {
+        chunkText: 'a',
+        sourceDocument: 'agents.mdx',
+        sourceLink: 'x',
+        headerText: 'h',
+        similarity: 0.9,
+      },
+    ]
+    const outcome = await searchDocs('cron')
+    expect(outcome.droppedBelowThreshold).toBe(0)
+    expect(outcome.droppedStale).toBe(0)
+    expect(outcome.results).toHaveLength(1)
   })
 })

apps/sim/lib/copilot/docs/docs-search.ts

@@ -27,6 +27,21 @@ export interface DocsSearchResult {
  * section in the docs corpus. Surfaced verbatim so the model can correct itself
  * rather than reading an empty result as "the docs say nothing about this".
  */
+/**
+ * A search result set plus why it may be shorter than `topK`. The SQL LIMIT is
+ * applied before the threshold and liveness filters, so these counts are what
+ * distinguishes "nothing matched" from "matches were filtered out".
+ */
+export interface DocsSearchOutcome {
+  results: DocsSearchResult[]
+  /** Rows the vector search returned before filtering. */
+  candidatesConsidered: number
+  /** Candidates dropped for scoring below the similarity threshold. */
+  droppedBelowThreshold: number
+  /** Candidates dropped because their page is no longer in the docs manifest. */
+  droppedStale: number
+}
+
 export class DocsSearchScopeError extends Error {
   readonly code = 'DOCS_SEARCH_SCOPE' as const
   constructor(message: string) {
@@ -94,11 +109,16 @@ function escapeLikePattern(value: string): string {
  * The index lags the VFS: a page added since the last index rebuild is readable
  * but not searchable, and a deleted one can still return chunks. Results whose
  * source no longer maps to a live `docs/` path are dropped.
+ *
+ * Because those drops happen after the SQL LIMIT, a caller can get fewer hits
+ * than it asked for — or none at all when every candidate was filtered. The
+ * returned {@link DocsSearchOutcome} reports that explicitly so an empty result
+ * is never mistaken for "the documentation does not cover this".
  */
 export async function searchDocs(
   query: string,
   options?: { path?: string; topK?: number }
-): Promise<DocsSearchResult[]> {
+): Promise<DocsSearchOutcome> {
   if (!query || typeof query !== 'string') throw new Error('query is required')
 
   const topK = Math.min(Math.max(Math.trunc(options?.topK ?? DEFAULT_TOP_K), 1), MAX_TOP_K)
@@ -107,7 +127,9 @@ export async function searchDocs(
   logger.info('Executing docs search', { query, topK, path: options?.path ?? null })
 
   const { embedding: queryEmbedding } = await generateSearchEmbedding(query)
-  if (!queryEmbedding || queryEmbedding.length === 0) return []
+  if (!queryEmbedding || queryEmbedding.length === 0) {
+    return { results: [], candidatesConsidered: 0, droppedBelowThreshold: 0, droppedStale: 0 }
+  }
 
   const rows = await db
     .select({
@@ -123,10 +145,18 @@ export async function searchDocs(
     .limit(topK)
 
   const results: DocsSearchResult[] = []
+  let droppedBelowThreshold = 0
+  let droppedStale = 0
   for (const row of rows) {
-    if (row.similarity < SIMILARITY_THRESHOLD) continue
+    if (row.similarity < SIMILARITY_THRESHOLD) {
+      droppedBelowThreshold++
+      continue
+    }
     const path = docsPathForSourceDocument(row.sourceDocument)
-    if (!path) continue
+    if (!path) {
+      droppedStale++
+      continue
+    }
     results.push({
       path,
       url: String(row.sourceLink || '#'),
@@ -138,7 +168,13 @@ export async function searchDocs(
 
   logger.info('Docs search complete', {
     count: results.length,
-    dropped: rows.length - results.length,
+    droppedBelowThreshold,
+    droppedStale,
   })
-  return results
+  return {
+    results,
+    candidatesConsidered: rows.length,
+    droppedBelowThreshold,
+    droppedStale,
+  }
 }

apps/sim/lib/copilot/tools/server/docs/search-docs.ts

@@ -1,3 +1,4 @@
+import type { DocsSearchResult } from '@/lib/copilot/docs/docs-search'
 import { searchDocs } from '@/lib/copilot/docs/docs-search'
 import { SearchDocs } from '@/lib/copilot/generated/tool-catalog-v1'
 import type { BaseServerTool } from '@/lib/copilot/tools/server/base-tool'
@@ -9,9 +10,39 @@ interface SearchDocsParams {
 }
 
 interface SearchDocsOutput {
-  results: Awaited<ReturnType<typeof searchDocs>>
+  results: DocsSearchResult[]
   query: string
   totalResults: number
+  /**
+   * Present only when the vector search matched chunks that were then filtered
+   * out. Without it an empty result set reads as "the docs do not cover this",
+   * which sends the caller off to guess instead of rephrasing or falling back
+   * to glob.
+   */
+  note?: string
+}
+
+/**
+ * Explain a short or empty result set in terms the caller can act on. Returns
+ * undefined when nothing was dropped — the common case needs no commentary.
+ */
+function shortfallNote(outcome: Awaited<ReturnType<typeof searchDocs>>): string | undefined {
+  const { results, candidatesConsidered, droppedBelowThreshold, droppedStale } = outcome
+  if (droppedBelowThreshold === 0 && droppedStale === 0) return undefined
+
+  const reasons: string[] = []
+  if (droppedBelowThreshold > 0)
+    reasons.push(`${droppedBelowThreshold} scored too low to be relevant`)
+  if (droppedStale > 0) {
+    reasons.push(
+      `${droppedStale} point at pages no longer in the docs (the search index lags the site)`
+    )
+  }
+  const dropped = reasons.join(' and ')
+
+  return results.length === 0
+    ? `No relevant matches. The search index returned ${candidatesConsidered} candidate(s), but ${dropped} — this does NOT mean the docs lack this topic. Rephrase the query, widen it by dropping the path scope, or browse with glob("docs/**").`
+    : `Returned ${results.length} of ${candidatesConsidered} candidate(s); ${dropped}. Rephrase or widen the query if these look off-topic.`
 }
 
 /**
@@ -22,7 +53,13 @@ interface SearchDocsOutput {
 export const searchDocsServerTool: BaseServerTool<SearchDocsParams, SearchDocsOutput> = {
   name: SearchDocs.id,
   async execute(params: SearchDocsParams): Promise<SearchDocsOutput> {
-    const results = await searchDocs(params.query, { path: params.path, topK: params.topK })
-    return { results, query: params.query, totalResults: results.length }
+    const outcome = await searchDocs(params.query, { path: params.path, topK: params.topK })
+    const note = shortfallNote(outcome)
+    return {
+      results: outcome.results,
+      query: params.query,
+      totalResults: outcome.results.length,
+      ...(note ? { note } : {}),
+    }
   },
 }