From 8069af12ff381b22ed7abb488eca10b532eb457e Mon Sep 17 00:00:00 2001 From: Baptiste Leproux Date: Thu, 13 Aug 2026 17:23:26 +0200 Subject: [PATCH 1/9] feat(search): client side fts --- app/app.vue | 29 +- app/composables/useDocsContent.ts | 3 - app/composables/useLocalSearch.ts | 140 ++++ app/error.vue | 11 +- app/utils/search-sections.ts | 23 - modules/config.ts | 11 +- nuxt.config.ts | 2 + package.json | 1 + pnpm-lock.yaml | 643 +++++++++--------- server/api/content/[...path].get.ts | 6 +- .../api/content/blob/[sha]/[...path].get.ts | 16 + server/api/content/head.get.ts | 13 + .../content/tree/[branch]/[...path].get.ts | 2 + server/api/revalidate.post.ts | 6 +- server/utils/content.ts | 22 +- 15 files changed, 536 insertions(+), 392 deletions(-) create mode 100644 app/composables/useLocalSearch.ts delete mode 100644 app/utils/search-sections.ts create mode 100644 server/api/content/head.get.ts diff --git a/app/app.vue b/app/app.vue index 62ef764..1253904 100644 --- a/app/app.vue +++ b/app/app.vue @@ -1,6 +1,5 @@ + + diff --git a/app/composables/useLocalSearch.ts b/app/composables/useLocalSearch.ts deleted file mode 100644 index 22ec947..0000000 --- a/app/composables/useLocalSearch.ts +++ /dev/null @@ -1,140 +0,0 @@ -import type { CacheArtifact, RelationalDatabase, SearchOptions, SearchResult } from 'comark-content' -import type { ActiveContent } from './useDocsContent' -import { prefixLink } from '../utils/routing' - -type LocalSearchStatus = 'idle' | 'loading' | 'ready' | 'error' - -/** The subset of the hydrated instance the palette needs (plugin methods, so typed by hand). */ -interface LocalSearchInstance { - init: () => Promise - search: (sources: string[], query: string, opts?: SearchOptions) => Promise -} - -const COMMIT_SHA = /^[0-9a-f]{40}$/i - -type LocalSearchScope = 'prod' | 'preview' - -/** - * At most two databases ever exist, one per scope: - * - prod: pinned to the head commit the page was rendered at (built once, kept for the session) - * - preview: one at a time, keyed by `previewBase` — visiting another `/tree` or `/blob` ref - * rebuilds the instance but reuses the preview database (its FTS rows are cleared per source) - */ -let prodInstance: Promise | undefined -let previewInstance: Promise | undefined -let previewBase: string | undefined -let previewDatabase: RelationalDatabase | undefined -const status = ref('idle') - -function resolveApiBase(active: ActiveContent, headSha: string): string { - if (active.mode === 'tree') return `/api/content/tree/${encodeURIComponent(active.ref!)}` - if (active.mode === 'blob') return `/api/content/blob/${active.ref}` - return COMMIT_SHA.test(headSha) ? `/api/content/blob/${headSha}` : '/api/content' -} - -async function createInstance(apiBase: string, scope: LocalSearchScope): Promise { - // Dynamic imports so sqlite-wasm and the FTS plugin only ever load in the browser, on demand. - const [{ comarkContent }, sqliteWasm, sqliteFullTextSearch] = await Promise.all([ - import('comark-content'), - import('comark-content/database/sqlite-wasm').then((m) => m.default), - import('comark-content/plugins/sqlite-full-text-search').then((m) => m.default), - ]) - - const database = scope === 'preview' ? (previewDatabase ??= sqliteWasm()) : sqliteWasm() - const content = comarkContent({ - cache: { - loadManifest: () => $fetch(`${apiBase}/manifest.json`), - loadSnapshot: (source: string) => $fetch(`${apiBase}/snapshot/${source}.json`), - }, - plugins: [sqliteFullTextSearch({ database })], - }) as unknown as LocalSearchInstance - - // Warm up the instance - await content.init() - await content.search(['content'], '') - - return content -} - -async function buildInstance( - active: ActiveContent, - headSha: string, - scope: LocalSearchScope -): Promise { - status.value = 'loading' - try { - const content = await createInstance(resolveApiBase(active, headSha), scope) - status.value = 'ready' - return content - } catch (error) { - // Don't memoize a failed hydration — the next palette open should retry. - if (scope === 'prod') { - prodInstance = undefined - } else if (previewBase === active.base) { - previewInstance = undefined - previewBase = undefined - } - status.value = 'error' - throw error - } -} - -function getInstance(active: ActiveContent, headSha: string): Promise { - if (active.mode === 'prod') { - prodInstance ??= buildInstance(active, headSha, 'prod') - return prodInstance - } - - if (!previewInstance || previewBase !== active.base) { - previewBase = active.base - previewInstance = buildInstance(active, headSha, 'preview') - } - return previewInstance -} - -/** - * Client-side full-text search: a browser-standalone comark-content instance (sqlite-wasm FTS5) - * hydrated from the per-commit snapshot artifacts. BM25-ranked section results, zero server work - * per keystroke. `status` follows `UContentSearch`'s `search-status` contract; a failed hydration - * surfaces as `'error'` and the next palette open retries. - */ -export function useLocalSearch() { - const active = useDocsContent() - - // The head commit the page was rendered at, resolved during SSR (in-process call) and shipped - // in the payload — the client never refetches it. The pin can advance past the deploy SHA via - // the push webhook, so it must come from the server's `getHeadRef()`, not build-time env. - const { data: headSha } = useAsyncData( - 'content-head-sha', - () => $fetch<{ sha: string }>('/api/content/head').then(({ sha }) => sha), - { default: () => '' } - ) - - /** Kick off wasm + snapshot loading before the first keystroke needs it. */ - function warmup(): void { - getInstance(active.value, headSha.value).catch(() => {}) // surfaced through `status` - } - - if (import.meta.client) { - onNuxtReady(warmup) - } - - async function search(query: string, opts?: SearchOptions): Promise { - const instance = await getInstance(active.value, headSha.value) - const results = await instance.search(['content'], query, { - limit: 25, - snippet: { columns: ['content'] }, - ...opts, - }) - - // Preview modes: keep result links inside `/tree/` / `/blob/`, like `searchFiles`. - const base = active.value.base - if (!base) return results - return results.map((result) => { - const [path, hash] = result.id.split('#') - return { ...result, id: prefixLink(path!, base) + (hash ? `#${hash}` : '') } - }) - } - - return { search, status: readonly(status), warmup } -} diff --git a/app/composables/useSearch.ts b/app/composables/useSearch.ts new file mode 100644 index 0000000..cf0095e --- /dev/null +++ b/app/composables/useSearch.ts @@ -0,0 +1,92 @@ +import type { SearchOptions, SearchResult } from 'comark-content' +import type { SearchWorkerPayload, SearchWorkerResponse } from '../types/search-worker' + +type SearchStatus = 'idle' | 'loading' | 'ready' | 'error' + +const status = ref('idle') + +let worker: Worker | undefined +let nextId = 0 +const pending = new Map void, reject: (error: Error) => void }>() + +function getWorker(): Worker { + if (worker) return worker + + worker = new Worker(new URL('../workers/search.worker.ts', import.meta.url), { type: 'module' }) + + worker.onmessage = (event: MessageEvent) => { + const message = event.data + if (message.type === 'status') { + status.value = message.value + return + } + const settle = pending.get(message.id) + if (!settle) return + pending.delete(message.id) + if (message.type === 'result') settle.resolve(message.results) + else settle.reject(new Error(message.message)) + } + + worker.onerror = () => { + status.value = 'error' + for (const { reject } of pending.values()) reject(new Error('[search] the search worker failed to load')) + pending.clear() + } + + return worker +} + +function request(message: SearchWorkerPayload): Promise { + const id = ++nextId + return new Promise((resolve, reject) => { + pending.set(id, { resolve, reject }) + try { + getWorker().postMessage({ ...message, id }) + } catch (error) { + pending.delete(id) + reject(error instanceof Error ? error : new Error(String(error))) + } + }) +} + +/** + * Client-side full-text search over production content (sqlite-wasm FTS5) hydrated from the + * per-commit snapshot artifacts. + */ +export function useSearch() { + const { data: headSha } = useAsyncData( + 'content-head-sha', + () => $fetch<{ sha: string | null }>('/api/content/head').then(({ sha }) => sha), + { default: () => null } + ) + + /** + * Load the database ahead of the first keystroke. No-op once loading or ready; retries after a + * failure — the worker holds that guard, since this side's `status` lags a message behind. + */ + async function warmup(): Promise { + try { + if (!headSha.value && !import.meta.dev) { + throw new Error('[search] /api/content/head returned no commit pin') + } + + // Immutable per-commit artifacts, CDN-cached forever. Only unpinned in dev, per the guard above. + const apiBase = headSha.value ? `/api/content/blob/${headSha.value}` : '/api/content' + + await request({ type: 'warmup', apiBase, origin: location.origin }) + } catch (error) { + status.value = 'error' + console.error('[search] could not load the search database', error) + } + } + + if (import.meta.client) { + onNuxtReady(warmup) + } + + async function search(query: string, opts?: SearchOptions): Promise { + return request({ type: 'search', query, opts }) + } + + return { search, status: readonly(status), warmup } +} diff --git a/app/error.vue b/app/error.vue index f114b82..b9e4ea8 100644 --- a/app/error.vue +++ b/app/error.vue @@ -18,12 +18,6 @@ useSeoMeta({ const { data: navigation } = await useAsyncData('navigation', () => prodContent.navigation()) -const { search: localSearch, status: localSearchStatus, warmup } = useLocalSearch() -const searchOpen = useContentSearch().open -watch(searchOpen, (isOpen) => { - if (isOpen) warmup() -}) - provide('navigation', navigation) @@ -35,13 +29,6 @@ provide('navigation', navigation) - - - + diff --git a/app/types/search-worker.ts b/app/types/search-worker.ts new file mode 100644 index 0000000..812bc85 --- /dev/null +++ b/app/types/search-worker.ts @@ -0,0 +1,31 @@ +import type { SearchOptions, SearchResult } from 'comark-content' + +/** + * Protocol between `useSearch` and `app/workers/search.worker.ts`. + * + * Every request carries an `id` and gets exactly one `result`/`error` reply — `warmup` answers + * with an empty array — so the caller can drain its pending map uniformly. + */ +export type SearchWorkerPayload = + | { + type: 'warmup' + apiBase: string + origin: string + } + | { + type: 'search', + query: string, + opts?: SearchOptions + } + +/** + * Intersected rather than spread into each member: `Omit` would collapse to the + * union's common keys, dropping every payload field. + */ +export type SearchWorkerRequest = SearchWorkerPayload & { id: number } + +/** `status` arrives unsolicited: the worker owns the hydration lifecycle, the caller mirrors it. */ +export type SearchWorkerResponse = + | { type: 'status', value: 'loading' | 'ready' | 'error' } + | { type: 'result', id: number, results: SearchResult[] } + | { type: 'error', id: number, message: string } diff --git a/app/workers/search.worker.ts b/app/workers/search.worker.ts new file mode 100644 index 0000000..ace1f0c --- /dev/null +++ b/app/workers/search.worker.ts @@ -0,0 +1,89 @@ +/** + * Search worker: owns the browser-standalone `comark-content` instance (sqlite-wasm FTS5) + * hydrated from the per-commit snapshot artifacts. + * + * It lives off the main thread because sqlite-wasm's `oo1` binding is synchronous and the FTS + * plugin indexes one row per section — on the main thread the whole hydration collapses into a + * single long task (the `await`s between inserts only yield to the microtask queue, which drains + * before the browser can paint or handle input). + * + * Not a Nuxt-scanned directory, so nothing here is auto-imported. + */ +import { comarkContent } from 'comark-content' +import sqliteWasm from 'comark-content/database/sqlite-wasm' +import sqliteFullTextSearch from 'comark-content/plugins/sqlite-full-text-search' +import { ofetch } from 'ofetch' +import type { CacheArtifact, ComarkContent } from 'comark-content' +import type { SqliteFullTextSearchMethods } from 'comark-content/plugins/sqlite-full-text-search' +import type { SearchWorkerRequest, SearchWorkerResponse } from '../types/search-worker' + +type SearchInstance = ComarkContent & SqliteFullTextSearchMethods +type SearchStatus = 'idle' | 'loading' | 'ready' | 'error' + +let instance: SearchInstance | undefined +let status: SearchStatus = 'idle' + +function post(message: SearchWorkerResponse): void { + self.postMessage(message) +} + +/** Every transition is mirrored to the main thread; the worker owns the hydration lifecycle. */ +function setStatus(value: Exclude): void { + status = value + post({ type: 'status', value }) +} + +/** + * Loads the database. No-op once loading or ready; retries after a failure. + * + * The guard lives here rather than in `useSearch` because the main thread's copy of `status` lags + * a message behind, so two warmups fired in the same tick would both get through it. + */ +async function loadDatabase(apiBase: string, origin: string): Promise { + if (status === 'loading' || status === 'ready') return + + setStatus('loading') + try { + const fetchArtifact = (path: string) => ofetch(new URL(path, origin).href) + + const content = comarkContent({ + cache: { + loadManifest: () => fetchArtifact(`${apiBase}/manifest.json`), + loadSnapshot: (source: string) => fetchArtifact(`${apiBase}/snapshot/${source}.json`), + }, + plugins: [sqliteFullTextSearch({ database: sqliteWasm() })], + }) + + await content.init() + await content.search(['content'], '') // pulls the snapshot in and builds the FTS index + + instance = content + setStatus('ready') + } catch (error) { + setStatus('error') + throw error + } +} + +self.onmessage = async (event: MessageEvent) => { + const request = event.data + try { + if (request.type === 'warmup') { + await loadDatabase(request.apiBase, request.origin) + post({ type: 'result', id: request.id, results: [] }) + return + } + + const results = instance + ? await instance.search(['content'], request.query, { + limit: 25, + snippet: { columns: ['content'] }, + ...request.opts, + }) + : [] + post({ type: 'result', id: request.id, results }) + } catch (error) { + // Serialized rather than cloned: plugin errors can carry non-transferable properties. + post({ type: 'error', id: request.id, message: error instanceof Error ? error.message : String(error) }) + } +} diff --git a/modules/config.ts b/modules/config.ts index d6079c2..ca2873b 100644 --- a/modules/config.ts +++ b/modules/config.ts @@ -166,7 +166,7 @@ export default defineNuxtModule({ '/llms.txt': { isr }, '/llms-full.txt': { isr }, '/rss.xml': { isr }, - // Per-commit artifacts hydrating the client-side search database (see `useLocalSearch`) + // Per-commit artifacts hydrating the client-side search database (see `useSearch`) '/api/content/blob/*/manifest.json': { isr: true }, // Immutable since SHA-pinned '/api/content/blob/*/snapshot/*': { isr: true }, // Immutable since SHA-pinned '/api/content/tree/*/manifest.json': { isr }, diff --git a/nuxt.config.ts b/nuxt.config.ts index f724a19..5fc7da1 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -33,6 +33,7 @@ export default defineNuxtConfig({ resolve: { alias: { 'beautiful-mermaid': resolveModulePath('beautiful-mermaid', { from: import.meta.url }) }, }, + worker: { format: 'es' }, optimizeDeps: { include: [ 'beautiful-mermaid', diff --git a/server/api/content/[...path].get.ts b/server/api/content/[...path].get.ts index 6dfb0f3..cf2ca9c 100644 --- a/server/api/content/[...path].get.ts +++ b/server/api/content/[...path].get.ts @@ -1,6 +1,6 @@ /** * Single data endpoint: `content.handler()` dispatches `get`, `navigation`, `list`, `manifest` - * and `snapshot`. Cached per-URL — see `routeRules`. + * and `snapshot`. Must be cached per-URL by layer consumer. */ export default defineEventHandler(async (event) => { const content = await getProdContent() diff --git a/server/api/content/head.get.ts b/server/api/content/head.get.ts index 5324dfa..11c3e14 100644 --- a/server/api/content/head.get.ts +++ b/server/api/content/head.get.ts @@ -1,13 +1,8 @@ /** - * The commit SHA production content is currently pinned to. The client-side search database - * (see `useLocalSearch`) uses it to hydrate from the immutable `/api/content/blob//*` - * artifacts instead of the live endpoints, so snapshot downloads are CDN-cached forever. - * - * `getProdContent()` refreshes the head against the branch tip (60s shared ref cache) before - * `getHeadRef()` is read. In dev this returns the branch name, which callers must treat as - * "no immutable pin available". + * The commit SHA production content is pinned to, or `null` in dev */ export default defineEventHandler(async () => { await getProdContent() - return { sha: getHeadRef() } + + return { sha: import.meta.dev ? null : getHeadRef() } }) From d26bc6ff24a615ed331311844e46315ad58dd3de Mon Sep 17 00:00:00 2001 From: Baptiste Leproux Date: Wed, 19 Aug 2026 16:08:12 +0200 Subject: [PATCH 5/9] use comark-cms latest --- package.json | 2 +- pnpm-lock.yaml | 10 +++++----- server/api/content/[...path].get.ts | 2 -- server/api/content/blob/[sha]/[...path].get.ts | 3 --- server/api/content/tree/[branch]/[...path].get.ts | 2 -- server/api/revalidate.post.ts | 10 +++------- server/utils/content.ts | 12 +++++------- 7 files changed, 14 insertions(+), 27 deletions(-) diff --git a/package.json b/package.json index 3d52923..8476aa3 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "ai": "^7.0.22", "beautiful-mermaid": "^1.1.3", "comark": "https://pkg.pr.new/comark@af8d3e8", - "comark-content": "https://pkg.pr.new/comark-content@6b8aae4", + "comark-content": "https://pkg.pr.new/comark-content@67c137f", "defu": "^6.1.7", "exsolve": "^1.1.0", "js-yaml": "^5.2.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5b81d1f..88180c8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -84,8 +84,8 @@ importers: specifier: https://pkg.pr.new/comark@af8d3e8 version: https://pkg.pr.new/comark@af8d3e8(beautiful-mermaid@1.1.3)(rangi@2.2.0)(shiki@4.3.1) comark-content: - specifier: https://pkg.pr.new/comark-content@6b8aae4 - version: https://pkg.pr.new/comark-content@6b8aae4(@vercel/functions@3.7.6(ws@8.21.1))(beautiful-mermaid@1.1.3)(db0@0.3.4)(ioredis@5.11.1)(rangi@2.2.0)(shiki@4.3.1) + specifier: https://pkg.pr.new/comark-content@67c137f + version: https://pkg.pr.new/comark-content@67c137f(@vercel/functions@3.7.6(ws@8.21.1))(beautiful-mermaid@1.1.3)(db0@0.3.4)(ioredis@5.11.1)(rangi@2.2.0)(shiki@4.3.1) defu: specifier: ^6.1.7 version: 6.1.7 @@ -3577,8 +3577,8 @@ packages: colortranslator@5.0.0: resolution: {integrity: sha512-Z3UPUKasUVDFCDYAjP2fmlVRf1jFHJv1izAmPjiOa0OCIw1W7iC8PZ2GsoDa8uZv+mKyWopxxStT9q05+27h7w==} - comark-content@https://pkg.pr.new/comark-content@6b8aae4: - resolution: {tarball: https://pkg.pr.new/comark-content@6b8aae4} + comark-content@https://pkg.pr.new/comark-content@67c137f: + resolution: {integrity: sha512-X6IbRRKi2IU8COgDCpoHtLZ8wiFprgQpIrm3UWyp9K3F/omo03w/lko+uGSE9SMoakFtfJS74pswUJDLcy05tw==, tarball: https://pkg.pr.new/comark-content@67c137f} version: 0.3.0 hasBin: true @@ -10557,7 +10557,7 @@ snapshots: colortranslator@5.0.0: {} - comark-content@https://pkg.pr.new/comark-content@6b8aae4(@vercel/functions@3.7.6(ws@8.21.1))(beautiful-mermaid@1.1.3)(db0@0.3.4)(ioredis@5.11.1)(rangi@2.2.0)(shiki@4.3.1): + comark-content@https://pkg.pr.new/comark-content@67c137f(@vercel/functions@3.7.6(ws@8.21.1))(beautiful-mermaid@1.1.3)(db0@0.3.4)(ioredis@5.11.1)(rangi@2.2.0)(shiki@4.3.1): dependencies: citty: 0.2.2 comark: 0.6.2(beautiful-mermaid@1.1.3)(rangi@2.2.0)(shiki@4.3.1) diff --git a/server/api/content/[...path].get.ts b/server/api/content/[...path].get.ts index cf2ca9c..c4f39d7 100644 --- a/server/api/content/[...path].get.ts +++ b/server/api/content/[...path].get.ts @@ -5,7 +5,5 @@ export default defineEventHandler(async (event) => { const content = await getProdContent() - await ensureSnapshotContent(content, getRouterParam(event, 'path') ?? '') - return content.handler(toWebRequest(event)) }) diff --git a/server/api/content/blob/[sha]/[...path].get.ts b/server/api/content/blob/[sha]/[...path].get.ts index a6ad6e8..b8ce7a1 100644 --- a/server/api/content/blob/[sha]/[...path].get.ts +++ b/server/api/content/blob/[sha]/[...path].get.ts @@ -21,7 +21,6 @@ export default defineEventHandler(async (event) => { if (sha === getHeadRef()) { const prod = await getProdContent() if (sha === getHeadRef()) { - await ensureSnapshotContent(prod, path) const request = toWebRequest(event) const url = new URL(request.url) url.pathname = url.pathname.replace(`/blob/${rawSha}`, '') @@ -31,7 +30,5 @@ export default defineEventHandler(async (event) => { const content = await getPreviewContent(sha, `/api/content/blob/${sha}`) - await ensureSnapshotContent(content, path) - return await content.handler(toWebRequest(event)) }) diff --git a/server/api/content/tree/[branch]/[...path].get.ts b/server/api/content/tree/[branch]/[...path].get.ts index e454ce5..9f0ef4e 100644 --- a/server/api/content/tree/[branch]/[...path].get.ts +++ b/server/api/content/tree/[branch]/[...path].get.ts @@ -16,7 +16,5 @@ export default defineEventHandler(async (event) => { const sha = await resolveSha(branch, { cacheMisses: true }) const content = await getPreviewContent(sha, `/api/content/tree/${encodeURIComponent(branch)}`) - await ensureSnapshotContent(content, path) - return await content.handler(toWebRequest(event)) }) diff --git a/server/api/revalidate.post.ts b/server/api/revalidate.post.ts index 188d984..bc2b26c 100644 --- a/server/api/revalidate.post.ts +++ b/server/api/revalidate.post.ts @@ -188,13 +188,9 @@ export default defineEventHandler(async (event) => { throw err }) - // Warm the per-SHA body cache so cold instances skip re-parsing from GitHub. - // `metaOnly` became `partial` in comark-content 0.2.0 with no alias and consumers straddle both, - // so send both keys — each version ignores the other's. Not inlined: as a literal, - // excess-property checking rejects whichever key the installed types don't declare. - const full = { partial: false, metaOnly: false } - await headContent.init(full).catch((err) => { - console.error(`${tag} cache warm failed`, err?.message ?? err) + // Warms the per-SHA body cache and persists the snapshot artifact + await warmSnapshot(headContent).catch((err) => { + console.error(`${tag} snapshot warm failed`, err?.message ?? err) }) await useStorage('cache:nuxt:payload').clear() diff --git a/server/utils/content.ts b/server/utils/content.ts index 9dd0cab..c573361 100644 --- a/server/utils/content.ts +++ b/server/utils/content.ts @@ -68,14 +68,12 @@ export async function createSourceContent( } /** - * Snapshot artifacts must dump the full corpus, but the default init is partial (frontmatter only) - * and `cache.snapshot()` only sees bodies already in the cache. Upgrade the instance before serving - * a snapshot — memoized, so bodies parse once per instance (the FTS serve handler has the same guard). + * Fully parse and persist the snapshot artifact into this instance's per-SHA cache. */ -export async function ensureSnapshotContent(content: ComarkContent, path: string): Promise { - if (path.startsWith('snapshot')) { - await content.init({ partial: false }) - } +export async function warmSnapshot(content: ComarkContent): Promise { + await content.init({ partial: false }) + const artifact = await content.cache.snapshot('content', { fresh: false }) + console.log(`[content] snapshot artifact ${artifact ? `${artifact.size} bytes` : 'not produced'}`) } /** Production branch, resolved per request: content pushes skip redeploys (`vercel.json` `ignoreCommand`). */ From 06e5eec209571563b4556c12fdbe37ab9a603813 Mon Sep 17 00:00:00 2001 From: Baptiste Leproux Date: Wed, 19 Aug 2026 17:59:59 +0200 Subject: [PATCH 6/9] app search nav groups --- app/components/AppSearch.vue | 52 ++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/app/components/AppSearch.vue b/app/components/AppSearch.vue index 0ffe5cc..1114812 100644 --- a/app/components/AppSearch.vue +++ b/app/components/AppSearch.vue @@ -1,17 +1,58 @@ @@ -21,6 +62,7 @@ watch(open, (isOpen) => { :search="search" :search-status="status" :navigation="navigation" + :groups="groups" :transition="false" :loading="status === 'loading'" /> From a783a636e27f66c8a3ff71d1e77270482760664a Mon Sep 17 00:00:00 2001 From: Baptiste Leproux Date: Wed, 19 Aug 2026 18:25:58 +0200 Subject: [PATCH 7/9] debug system --- app/composables/useSearch.ts | 19 +++++++++- app/types/search-worker.ts | 2 + app/workers/search-logger.ts | 72 ++++++++++++++++++++++++++++++++++++ app/workers/search.worker.ts | 45 +++++++++++++++++++--- 4 files changed, 131 insertions(+), 7 deletions(-) create mode 100644 app/workers/search-logger.ts diff --git a/app/composables/useSearch.ts b/app/composables/useSearch.ts index cf0095e..aaac194 100644 --- a/app/composables/useSearch.ts +++ b/app/composables/useSearch.ts @@ -9,6 +9,14 @@ let worker: Worker | undefined let nextId = 0 const pending = new Map void, reject: (error: Error) => void }>() +/** + * Hydration logging switch: `?debug=search` + */ +function searchDebug(): boolean { + if (!import.meta.client) return false + return new URLSearchParams(location.search).get('debug') === 'search' +} + function getWorker(): Worker { if (worker) return worker @@ -18,13 +26,17 @@ function getWorker(): Worker { const message = event.data if (message.type === 'status') { status.value = message.value + if (searchDebug()) console.info(`[search] status -> ${message.value}`) return } const settle = pending.get(message.id) if (!settle) return pending.delete(message.id) if (message.type === 'result') settle.resolve(message.results) - else settle.reject(new Error(message.message)) + else { + if (searchDebug()) console.error(`[search] request ${message.id} failed:`, message.message) + settle.reject(new Error(message.message)) + } } worker.onerror = () => { @@ -73,7 +85,10 @@ export function useSearch() { // Immutable per-commit artifacts, CDN-cached forever. Only unpinned in dev, per the guard above. const apiBase = headSha.value ? `/api/content/blob/${headSha.value}` : '/api/content' - await request({ type: 'warmup', apiBase, origin: location.origin }) + const debug = searchDebug() + if (debug) console.info(`[search] warmup from ${apiBase} (head ${headSha.value ?? 'unpinned'})`) + + await request({ type: 'warmup', apiBase, origin: location.origin, debug }) } catch (error) { status.value = 'error' console.error('[search] could not load the search database', error) diff --git a/app/types/search-worker.ts b/app/types/search-worker.ts index 812bc85..fd1038b 100644 --- a/app/types/search-worker.ts +++ b/app/types/search-worker.ts @@ -11,6 +11,8 @@ export type SearchWorkerPayload = type: 'warmup' apiBase: string origin: string + /** Turns on the worker's hydration logging. Resolved on the main thread, which owns `?debug=search`. */ + debug?: boolean } | { type: 'search', diff --git a/app/workers/search-logger.ts b/app/workers/search-logger.ts new file mode 100644 index 0000000..66790d6 --- /dev/null +++ b/app/workers/search-logger.ts @@ -0,0 +1,72 @@ +/** + * Logging for the search worker: the `?debug=search` switch, the phase-timing helpers, and the + * {@link Logger} handed to `comarkContent()` so the package's own diagnostics come out under this + * prefix. Separate from `search.worker.ts` to keep the hydration path free of instrumentation. + * + * Worker-side only. The main thread has its own `[search]` lines in `useSearch`, which is also where + * the switch is resolved — a worker cannot see the page URL, so the flag arrives with `warmup`. + */ +import type { ContentFile, Logger, RelationalDatabase } from 'comark-content' + +const PREFIX = '[search:worker]' + +let debug = false + +/** Called on every `warmup`; once on, it stays on for the life of the worker. */ +export function setDebug(value: boolean): void { + debug = debug || value +} + +export function isDebug(): boolean { + return debug +} + +export function log(...args: unknown[]): void { + if (debug) console.info(PREFIX, ...args) +} + +/** Milliseconds since `from`, for log lines. */ +export function since(from: number): string { + return `${(performance.now() - from).toFixed(1)}ms` +} + +/** + * Warn and error are deliberately ungated: the FTS plugin reports a missing snapshot through this + * channel, and that failure is otherwise indistinguishable from "the query matched nothing". + */ +export const logger: Logger = { + debug: (tag, ...args) => log(`${tag}:`, ...args), + info: (tag, ...args) => log(`${tag}:`, ...args), + warn: (tag, ...args) => console.warn(`${PREFIX} ${tag}:`, ...args), + error: (tag, ...args) => console.error(`${PREFIX} ${tag}:`, ...args), +} + +/** + * What a decoded artifact holds: a snapshot decodes to the source's items, the manifest to an object + * keyed by path. `with nodes` is the number that matters — the FTS plugin indexes + * `kind === 'document' && nodes?.length`, so a bodies-less (partial) snapshot builds an empty index. + */ +export function describeArtifact(decoded: unknown): string { + if (Array.isArray(decoded)) { + const items = decoded as ContentFile[] + const documents = items.filter((item) => item.meta.kind === 'document') + const withNodes = documents.filter((item) => item.nodes?.length) + return `${items.length} item(s), ${documents.length} document(s), ${withNodes.length} with nodes` + } + const items = (decoded as { items?: Record } | null)?.items + return `${items ? Object.keys(items).length : 0} manifest item(s)` +} + +/** + * Rows in the FTS plugin's index — the one number that separates "nothing was indexed" from "the + * query found nothing", since `search()` catches SQL errors and returns `[]` either way. Reads the + * plugin's private table, so it is a diagnostic, not something to build on. + */ +export async function indexedRows(database: RelationalDatabase, source: string): Promise { + try { + const rows = await database.all<{ n: number }>('SELECT count(*) as n FROM __fts_search WHERE source = ?', [source]) + return rows?.[0]?.n ?? 'unknown' + } catch (error) { + return `unknown (${error instanceof Error ? error.message : String(error)})` + } +} diff --git a/app/workers/search.worker.ts b/app/workers/search.worker.ts index ace1f0c..093144b 100644 --- a/app/workers/search.worker.ts +++ b/app/workers/search.worker.ts @@ -9,10 +9,11 @@ * * Not a Nuxt-scanned directory, so nothing here is auto-imported. */ -import { comarkContent } from 'comark-content' +import { comarkContent, readArtifact } from 'comark-content' import sqliteWasm from 'comark-content/database/sqlite-wasm' import sqliteFullTextSearch from 'comark-content/plugins/sqlite-full-text-search' import { ofetch } from 'ofetch' +import { describeArtifact, indexedRows, isDebug, log, logger, setDebug, since } from './search-logger' import type { CacheArtifact, ComarkContent } from 'comark-content' import type { SqliteFullTextSearchMethods } from 'comark-content/plugins/sqlite-full-text-search' import type { SearchWorkerRequest, SearchWorkerResponse } from '../types/search-worker' @@ -40,27 +41,58 @@ function setStatus(value: Exclude): void { * a message behind, so two warmups fired in the same tick would both get through it. */ async function loadDatabase(apiBase: string, origin: string): Promise { - if (status === 'loading' || status === 'ready') return + if (status === 'loading' || status === 'ready') { + log(`warmup ignored — already ${status}`) + return + } setStatus('loading') + const started = performance.now() try { - const fetchArtifact = (path: string) => ofetch(new URL(path, origin).href) + const fetchArtifact = async (path: string): Promise => { + const url = new URL(path, origin).href + const fetchStarted = performance.now() + try { + const artifact = await ofetch(url) + if (isDebug()) { + let contents: string + try { + contents = describeArtifact(await readArtifact(artifact)) + } catch (error) { + contents = `undecodable: ${error instanceof Error ? error.message : String(error)}` + } + log(`fetched ${path} in ${since(fetchStarted)} — ${artifact?.size ?? 0} bytes, ${contents}`) + } + return artifact + } catch (error) { + log(`failed ${path} after ${since(fetchStarted)}`, error) + throw error + } + } + // Held rather than inlined into the plugin so the row count below can query the index directly. + const database = sqliteWasm() const content = comarkContent({ cache: { loadManifest: () => fetchArtifact(`${apiBase}/manifest.json`), loadSnapshot: (source: string) => fetchArtifact(`${apiBase}/snapshot/${source}.json`), }, - plugins: [sqliteFullTextSearch({ database: sqliteWasm() })], + plugins: [sqliteFullTextSearch({ database })], + logger, }) await content.init() + + const indexStarted = performance.now() await content.search(['content'], '') // pulls the snapshot in and builds the FTS index + log(`index built in ${since(indexStarted)} — ${await indexedRows(database, 'content')} row(s)`) instance = content setStatus('ready') + log(`ready in ${since(started)}`) } catch (error) { setStatus('error') + log(`hydration failed after ${since(started)}`, error) throw error } } @@ -69,11 +101,13 @@ self.onmessage = async (event: MessageEvent) => { const request = event.data try { if (request.type === 'warmup') { + setDebug(request.debug === true) await loadDatabase(request.apiBase, request.origin) post({ type: 'result', id: request.id, results: [] }) return } + const queryStarted = performance.now() const results = instance ? await instance.search(['content'], request.query, { limit: 25, @@ -81,9 +115,10 @@ self.onmessage = async (event: MessageEvent) => { ...request.opts, }) : [] + if (!instance) log(`dropped query "${request.query}" — no instance yet (status ${status})`) + else log(`query "${request.query}" -> ${results.length} result(s) in ${since(queryStarted)}`) post({ type: 'result', id: request.id, results }) } catch (error) { - // Serialized rather than cloned: plugin errors can carry non-transferable properties. post({ type: 'error', id: request.id, message: error instanceof Error ? error.message : String(error) }) } } From d1b2cbc4f9435609d7b50b5eec8c4594f6af15e5 Mon Sep 17 00:00:00 2001 From: Baptiste Leproux Date: Wed, 19 Aug 2026 18:42:05 +0200 Subject: [PATCH 8/9] pnpm lock file --- pnpm-lock.yaml | 8 ++++---- server/api/revalidate.post.ts | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 88180c8..a6bc5a0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -380,13 +380,13 @@ packages: resolution: {integrity: sha512-3PxTH8itZzltK0U9jTwVVnjLXvnDYuq3m+QXsHkENxWiPRh4WaoLcs1SQjqgZ55kS+QyirpH5BVwzP2gMVG6EQ==} '@comark/nuxt@https://pkg.pr.new/@comark/nuxt@af8d3e8': - resolution: {tarball: https://pkg.pr.new/@comark/nuxt@af8d3e8} + resolution: {integrity: sha512-DB1uYeorYZOJGqEIqxkGzOtnW2Zq2t4Ov9dOmi/H9qAymwPXiON1esTAne72jaJ+pBnMKaFat5JDObbNe0PDUA==, tarball: https://pkg.pr.new/@comark/nuxt@af8d3e8} version: 0.6.2 peerDependencies: nuxt: ^4.0.0 '@comark/vue@https://pkg.pr.new/comarkdown/comark/@comark/vue@af8d3e8': - resolution: {tarball: https://pkg.pr.new/comarkdown/comark/@comark/vue@af8d3e8} + resolution: {integrity: sha512-Rsbr+USFlvSJOgDGOG61yQtS5V3Ms7maGofb9RnaxLHdVlBqewGlkhdS14xRz9ONOjCg717PumDMdz4IpundWA==, tarball: https://pkg.pr.new/comarkdown/comark/@comark/vue@af8d3e8} version: 0.6.2 peerDependencies: beautiful-mermaid: ^1.1.3 @@ -3600,7 +3600,7 @@ packages: optional: true comark@https://pkg.pr.new/comark@af8d3e8: - resolution: {tarball: https://pkg.pr.new/comark@af8d3e8} + resolution: {integrity: sha512-w4UJSGwzUf+W8qp2eGxf7dCwJgQPVJvTM8GLpibYHRelLImjwVEMA10pkGNQOumdi8QgTIDWWcba+JO1CFrTcg==, tarball: https://pkg.pr.new/comark@af8d3e8} version: 0.6.2 peerDependencies: beautiful-mermaid: ^1.1.3 @@ -3618,7 +3618,7 @@ packages: optional: true comark@https://pkg.pr.new/comarkdown/comark/comark@af8d3e8: - resolution: {tarball: https://pkg.pr.new/comarkdown/comark/comark@af8d3e8} + resolution: {integrity: sha512-w4UJSGwzUf+W8qp2eGxf7dCwJgQPVJvTM8GLpibYHRelLImjwVEMA10pkGNQOumdi8QgTIDWWcba+JO1CFrTcg==, tarball: https://pkg.pr.new/comarkdown/comark/comark@af8d3e8} version: 0.6.2 peerDependencies: beautiful-mermaid: ^1.1.3 diff --git a/server/api/revalidate.post.ts b/server/api/revalidate.post.ts index 8977ff4..214730e 100644 --- a/server/api/revalidate.post.ts +++ b/server/api/revalidate.post.ts @@ -150,9 +150,7 @@ export default defineEventHandler(async (event) => { // URL the browser loads (`…/_payload.json?`). const buildId = useRuntimeConfig(event).app.buildId - // Any content change invalidates the llms indexes and the feed. The search artifacts need no - // purge: the client hydrates from SHA-pinned `/api/content/blob//*` URLs, so a new head - // simply reads from new URLs and the old entries become unreachable. + // Any content change invalidates the llms indexes and the feed. const paths = new Set(['/llms.txt', '/llms-full.txt', '/rss.xml']) for (const f of changedFiles) { const pageUrl = pageUrlForPath(f) From 060607b5007b01d64d1383ff57e2fa71ba1c67d2 Mon Sep 17 00:00:00 2001 From: Baptiste Leproux Date: Wed, 19 Aug 2026 18:43:52 +0200 Subject: [PATCH 9/9] up --- app/components/AppSearch.vue | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/components/AppSearch.vue b/app/components/AppSearch.vue index 1114812..f12a890 100644 --- a/app/components/AppSearch.vue +++ b/app/components/AppSearch.vue @@ -5,11 +5,8 @@ const props = defineProps<{ navigation: NavigationItem[] }>() -// Setup runs on the server too (the `ClientOnly` is inside, around the palette), so `useSearch`'s -// head-sha `useAsyncData` still resolves during SSR and ships in the payload. const { search, status } = useSearch() - const appConfig = useAppConfig() interface PageItem {