diff --git a/app/app.vue b/app/app.vue index 62ef764..9347ad6 100644 --- a/app/app.vue +++ b/app/app.vue @@ -1,6 +1,5 @@ + + diff --git a/app/composables/useDocsContent.ts b/app/composables/useDocsContent.ts index 17ee0ca..3083ea3 100644 --- a/app/composables/useDocsContent.ts +++ b/app/composables/useDocsContent.ts @@ -1,12 +1,10 @@ import { createContentClient } from 'comark-content/client' -import { searchSectionsClient } from '../utils/search-sections' import type { ContentMode } from '../types/content' import { withLeadingSlash } from 'ufo' export const prodContent = createContentClient({ basePath: '/api/content', fetch: $fetch, - plugins: [searchSectionsClient()], }) const clients = new Map() @@ -17,7 +15,6 @@ function getClient(basePath: string) { client = createContentClient({ basePath, fetch: $fetch, - plugins: [searchSectionsClient()], }) clients.set(basePath, client) } diff --git a/app/composables/useSearch.ts b/app/composables/useSearch.ts new file mode 100644 index 0000000..cf0095e --- /dev/null +++ b/app/composables/useSearch.ts @@ -0,0 +1,92 @@ +import type { SearchOptions, SearchResult } from 'comark-content' +import type { SearchWorkerPayload, SearchWorkerResponse } from '../types/search-worker' + +type SearchStatus = 'idle' | 'loading' | 'ready' | 'error' + +const status = ref('idle') + +let worker: Worker | undefined +let nextId = 0 +const pending = new Map void, reject: (error: Error) => void }>() + +function getWorker(): Worker { + if (worker) return worker + + worker = new Worker(new URL('../workers/search.worker.ts', import.meta.url), { type: 'module' }) + + worker.onmessage = (event: MessageEvent) => { + const message = event.data + if (message.type === 'status') { + status.value = message.value + return + } + const settle = pending.get(message.id) + if (!settle) return + pending.delete(message.id) + if (message.type === 'result') settle.resolve(message.results) + else settle.reject(new Error(message.message)) + } + + worker.onerror = () => { + status.value = 'error' + for (const { reject } of pending.values()) reject(new Error('[search] the search worker failed to load')) + pending.clear() + } + + return worker +} + +function request(message: SearchWorkerPayload): Promise { + const id = ++nextId + return new Promise((resolve, reject) => { + pending.set(id, { resolve, reject }) + try { + getWorker().postMessage({ ...message, id }) + } catch (error) { + pending.delete(id) + reject(error instanceof Error ? error : new Error(String(error))) + } + }) +} + +/** + * Client-side full-text search over production content (sqlite-wasm FTS5) hydrated from the + * per-commit snapshot artifacts. + */ +export function useSearch() { + const { data: headSha } = useAsyncData( + 'content-head-sha', + () => $fetch<{ sha: string | null }>('/api/content/head').then(({ sha }) => sha), + { default: () => null } + ) + + /** + * Load the database ahead of the first keystroke. No-op once loading or ready; retries after a + * failure — the worker holds that guard, since this side's `status` lags a message behind. + */ + async function warmup(): Promise { + try { + if (!headSha.value && !import.meta.dev) { + throw new Error('[search] /api/content/head returned no commit pin') + } + + // Immutable per-commit artifacts, CDN-cached forever. Only unpinned in dev, per the guard above. + const apiBase = headSha.value ? `/api/content/blob/${headSha.value}` : '/api/content' + + await request({ type: 'warmup', apiBase, origin: location.origin }) + } catch (error) { + status.value = 'error' + console.error('[search] could not load the search database', error) + } + } + + if (import.meta.client) { + onNuxtReady(warmup) + } + + async function search(query: string, opts?: SearchOptions): Promise { + return request({ type: 'search', query, opts }) + } + + return { search, status: readonly(status), warmup } +} diff --git a/app/error.vue b/app/error.vue index 23b1c7c..b9e4ea8 100644 --- a/app/error.vue +++ b/app/error.vue @@ -17,9 +17,6 @@ useSeoMeta({ }) const { data: navigation } = await useAsyncData('navigation', () => prodContent.navigation()) -const { data: files } = useLazyAsyncData('search-sections', () => prodContent.searchSections(), { - server: false, -}) provide('navigation', navigation) @@ -32,11 +29,6 @@ provide('navigation', navigation) - - - + diff --git a/app/types/search-worker.ts b/app/types/search-worker.ts new file mode 100644 index 0000000..812bc85 --- /dev/null +++ b/app/types/search-worker.ts @@ -0,0 +1,31 @@ +import type { SearchOptions, SearchResult } from 'comark-content' + +/** + * Protocol between `useSearch` and `app/workers/search.worker.ts`. + * + * Every request carries an `id` and gets exactly one `result`/`error` reply — `warmup` answers + * with an empty array — so the caller can drain its pending map uniformly. + */ +export type SearchWorkerPayload = + | { + type: 'warmup' + apiBase: string + origin: string + } + | { + type: 'search', + query: string, + opts?: SearchOptions + } + +/** + * Intersected rather than spread into each member: `Omit` would collapse to the + * union's common keys, dropping every payload field. + */ +export type SearchWorkerRequest = SearchWorkerPayload & { id: number } + +/** `status` arrives unsolicited: the worker owns the hydration lifecycle, the caller mirrors it. */ +export type SearchWorkerResponse = + | { type: 'status', value: 'loading' | 'ready' | 'error' } + | { type: 'result', id: number, results: SearchResult[] } + | { type: 'error', id: number, message: string } diff --git a/app/utils/search-sections.ts b/app/utils/search-sections.ts deleted file mode 100644 index 4aa65d9..0000000 --- a/app/utils/search-sections.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { defineContentClientPlugin } from 'comark-content/client' -import { joinURL } from 'ufo' - -/** One search entry per document heading — consumed by `UContentSearch`. */ -export interface SearchSection { - id: string - title: string - titles: string[] - level: number - content: string -} - -interface SearchSectionsClientMethods { - searchSections(): Promise -} - -/** Client half of the `search-sections` serve handler (`server/utils/content.ts`); adds `content.searchSections()`. */ -export const searchSectionsClient = defineContentClientPlugin, SearchSectionsClientMethods>(() => ({ - name: 'search-sections', - setup: ({ options }) => ({ - searchSections: () => options.fetch(joinURL(options.baseURL, options.basePath, 'search-sections')), - }), -})) diff --git a/app/workers/search.worker.ts b/app/workers/search.worker.ts new file mode 100644 index 0000000..ace1f0c --- /dev/null +++ b/app/workers/search.worker.ts @@ -0,0 +1,89 @@ +/** + * Search worker: owns the browser-standalone `comark-content` instance (sqlite-wasm FTS5) + * hydrated from the per-commit snapshot artifacts. + * + * It lives off the main thread because sqlite-wasm's `oo1` binding is synchronous and the FTS + * plugin indexes one row per section — on the main thread the whole hydration collapses into a + * single long task (the `await`s between inserts only yield to the microtask queue, which drains + * before the browser can paint or handle input). + * + * Not a Nuxt-scanned directory, so nothing here is auto-imported. + */ +import { comarkContent } from 'comark-content' +import sqliteWasm from 'comark-content/database/sqlite-wasm' +import sqliteFullTextSearch from 'comark-content/plugins/sqlite-full-text-search' +import { ofetch } from 'ofetch' +import type { CacheArtifact, ComarkContent } from 'comark-content' +import type { SqliteFullTextSearchMethods } from 'comark-content/plugins/sqlite-full-text-search' +import type { SearchWorkerRequest, SearchWorkerResponse } from '../types/search-worker' + +type SearchInstance = ComarkContent & SqliteFullTextSearchMethods +type SearchStatus = 'idle' | 'loading' | 'ready' | 'error' + +let instance: SearchInstance | undefined +let status: SearchStatus = 'idle' + +function post(message: SearchWorkerResponse): void { + self.postMessage(message) +} + +/** Every transition is mirrored to the main thread; the worker owns the hydration lifecycle. */ +function setStatus(value: Exclude): void { + status = value + post({ type: 'status', value }) +} + +/** + * Loads the database. No-op once loading or ready; retries after a failure. + * + * The guard lives here rather than in `useSearch` because the main thread's copy of `status` lags + * a message behind, so two warmups fired in the same tick would both get through it. + */ +async function loadDatabase(apiBase: string, origin: string): Promise { + if (status === 'loading' || status === 'ready') return + + setStatus('loading') + try { + const fetchArtifact = (path: string) => ofetch(new URL(path, origin).href) + + const content = comarkContent({ + cache: { + loadManifest: () => fetchArtifact(`${apiBase}/manifest.json`), + loadSnapshot: (source: string) => fetchArtifact(`${apiBase}/snapshot/${source}.json`), + }, + plugins: [sqliteFullTextSearch({ database: sqliteWasm() })], + }) + + await content.init() + await content.search(['content'], '') // pulls the snapshot in and builds the FTS index + + instance = content + setStatus('ready') + } catch (error) { + setStatus('error') + throw error + } +} + +self.onmessage = async (event: MessageEvent) => { + const request = event.data + try { + if (request.type === 'warmup') { + await loadDatabase(request.apiBase, request.origin) + post({ type: 'result', id: request.id, results: [] }) + return + } + + const results = instance + ? await instance.search(['content'], request.query, { + limit: 25, + snippet: { columns: ['content'] }, + ...request.opts, + }) + : [] + post({ type: 'result', id: request.id, results }) + } catch (error) { + // Serialized rather than cloned: plugin errors can carry non-transferable properties. + post({ type: 'error', id: request.id, message: error instanceof Error ? error.message : String(error) }) + } +} diff --git a/modules/config.ts b/modules/config.ts index fea7649..ca2873b 100644 --- a/modules/config.ts +++ b/modules/config.ts @@ -159,17 +159,18 @@ export default defineNuxtModule({ '/logos': { isr }, // Previews are served live (SSR) off Runtime Cache; `/blob/**` is immutable commit HTML. '/tree/**': { isr, robots: 'noindex, nofollow' }, - '/blob/**': { isr: true, robots: 'noindex, nofollow' }, + '/blob/**': { isr: true, robots: 'noindex, nofollow' }, // Immutable since SHA-pinned // Raw markdown mirrors of every page, for agents. '/raw/**': { isr, robots: 'noindex' }, // Global content indexes, purged by the push webhook on content changes. '/llms.txt': { isr }, '/llms-full.txt': { isr }, '/rss.xml': { isr }, - // Fetched on every page hydration (see app.vue) and parses every doc body, so cache it. - '/api/content/blob/*/search-sections': { isr: true }, - '/api/content/tree/*/search-sections': { isr }, - '/api/content/search-sections': { isr }, + // Per-commit artifacts hydrating the client-side search database (see `useSearch`) + '/api/content/blob/*/manifest.json': { isr: true }, // Immutable since SHA-pinned + '/api/content/blob/*/snapshot/*': { isr: true }, // Immutable since SHA-pinned + '/api/content/tree/*/manifest.json': { isr }, + '/api/content/tree/*/snapshot/*': { isr }, '/api/code-explorer/**': { isr }, } diff --git a/nuxt.config.ts b/nuxt.config.ts index d43e6e9..5fc7da1 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -33,6 +33,7 @@ export default defineNuxtConfig({ resolve: { alias: { 'beautiful-mermaid': resolveModulePath('beautiful-mermaid', { from: import.meta.url }) }, }, + worker: { format: 'es' }, optimizeDeps: { include: [ 'beautiful-mermaid', @@ -42,6 +43,8 @@ export default defineNuxtConfig({ 'js-yaml', 'markdown-exit', ], + // Pre-bundling would break the wasm/worker assets sqlite loads relative to its module URL. + exclude: ['@sqlite.org/sqlite-wasm'], }, }, nitro: { diff --git a/package.json b/package.json index 5bf2129..3d52923 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "@octokit/webhooks-methods": "^6.0.0", "@opentelemetry/api": "^1.9.1", "@resvg/resvg-js": "^2.6.2", + "@sqlite.org/sqlite-wasm": "3.53.0-build1", "@vercel/analytics": "^2.0.1", "@vercel/functions": "^3.7.6", "@vercel/otel": "^2.1.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5277785..5b81d1f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,6 +56,9 @@ importers: '@resvg/resvg-js': specifier: ^2.6.2 version: 2.6.2 + '@sqlite.org/sqlite-wasm': + specifier: 3.53.0-build1 + version: 3.53.0-build1 '@vercel/analytics': specifier: ^2.0.1 version: 2.0.1(nuxt@4.5.1(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.7))(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.7))(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(@types/node@26.1.1)(@vercel/functions@3.7.6(ws@8.21.1))(@vue/compiler-sfc@3.5.40)(cac@7.0.0)(db0@0.3.4)(esbuild@0.28.1)(eslint@10.8.0(jiti@2.7.0))(ioredis@5.11.1)(lightningcss@1.33.0)(magicast@0.5.3)(optionator@0.9.4)(oxc-parser@0.140.0)(rollup-plugin-visualizer@7.0.1(rolldown@1.2.0)(rollup@4.62.2))(rollup@4.62.2)(srvx@0.11.22)(terser@5.49.0)(typescript@6.0.3)(vite@8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(yaml@2.9.0))(vue-tsc@3.3.8(typescript@6.0.3))(yaml@2.9.0))(vue@3.5.40(typescript@6.0.3)) @@ -377,13 +380,13 @@ packages: resolution: {integrity: sha512-3PxTH8itZzltK0U9jTwVVnjLXvnDYuq3m+QXsHkENxWiPRh4WaoLcs1SQjqgZ55kS+QyirpH5BVwzP2gMVG6EQ==} '@comark/nuxt@https://pkg.pr.new/@comark/nuxt@af8d3e8': - resolution: {integrity: sha512-DB1uYeorYZOJGqEIqxkGzOtnW2Zq2t4Ov9dOmi/H9qAymwPXiON1esTAne72jaJ+pBnMKaFat5JDObbNe0PDUA==, tarball: https://pkg.pr.new/@comark/nuxt@af8d3e8} + resolution: {tarball: https://pkg.pr.new/@comark/nuxt@af8d3e8} version: 0.6.2 peerDependencies: nuxt: ^4.0.0 '@comark/vue@https://pkg.pr.new/comarkdown/comark/@comark/vue@af8d3e8': - resolution: {integrity: sha512-Rsbr+USFlvSJOgDGOG61yQtS5V3Ms7maGofb9RnaxLHdVlBqewGlkhdS14xRz9ONOjCg717PumDMdz4IpundWA==, tarball: https://pkg.pr.new/comarkdown/comark/@comark/vue@af8d3e8} + resolution: {tarball: https://pkg.pr.new/comarkdown/comark/@comark/vue@af8d3e8} version: 0.6.2 peerDependencies: beautiful-mermaid: ^1.1.3 @@ -2264,6 +2267,10 @@ packages: '@speed-highlight/core@1.2.17': resolution: {integrity: sha512-Z92FwKpCtfaW1V0jTU/fh3QzYEZN8wDwrzRIBoADCJfn4mJCNcJN/XegifX7BDrQ8/h9Xh/JnbyMchL0FqXrkg==} + '@sqlite.org/sqlite-wasm@3.53.0-build1': + resolution: {integrity: sha512-PfWPWN2n+/37doa8oh2/oUXk4OOsRYZsxc1W1sDXIGb/Pu5Yrb+f2eyYpgQMGITVX7HVgxhs9P18Rc6I97ym/g==} + engines: {node: '>=22'} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -3571,7 +3578,7 @@ packages: resolution: {integrity: sha512-Z3UPUKasUVDFCDYAjP2fmlVRf1jFHJv1izAmPjiOa0OCIw1W7iC8PZ2GsoDa8uZv+mKyWopxxStT9q05+27h7w==} comark-content@https://pkg.pr.new/comark-content@6b8aae4: - resolution: {integrity: sha512-xAVSgpAUw8HXop9QoqU1PpRkBmsHRu/r/3fp4IjpGW0LrffScF+VXRo7C6BG91GR3xIJlkS4t1nywPOlWR0ssQ==, tarball: https://pkg.pr.new/comark-content@6b8aae4} + resolution: {tarball: https://pkg.pr.new/comark-content@6b8aae4} version: 0.3.0 hasBin: true @@ -3593,7 +3600,7 @@ packages: optional: true comark@https://pkg.pr.new/comark@af8d3e8: - resolution: {integrity: sha512-w4UJSGwzUf+W8qp2eGxf7dCwJgQPVJvTM8GLpibYHRelLImjwVEMA10pkGNQOumdi8QgTIDWWcba+JO1CFrTcg==, tarball: https://pkg.pr.new/comark@af8d3e8} + resolution: {tarball: https://pkg.pr.new/comark@af8d3e8} version: 0.6.2 peerDependencies: beautiful-mermaid: ^1.1.3 @@ -3611,7 +3618,7 @@ packages: optional: true comark@https://pkg.pr.new/comarkdown/comark/comark@af8d3e8: - resolution: {integrity: sha512-w4UJSGwzUf+W8qp2eGxf7dCwJgQPVJvTM8GLpibYHRelLImjwVEMA10pkGNQOumdi8QgTIDWWcba+JO1CFrTcg==, tarball: https://pkg.pr.new/comarkdown/comark/comark@af8d3e8} + resolution: {tarball: https://pkg.pr.new/comarkdown/comark/comark@af8d3e8} version: 0.6.2 peerDependencies: beautiful-mermaid: ^1.1.3 @@ -9223,6 +9230,8 @@ snapshots: '@speed-highlight/core@1.2.17': {} + '@sqlite.org/sqlite-wasm@3.53.0-build1': {} + '@standard-schema/spec@1.1.0': {} '@stylistic/eslint-plugin@5.10.0(eslint@10.8.0(jiti@2.7.0))': diff --git a/server/api/content/[...path].get.ts b/server/api/content/[...path].get.ts index e10db39..cf2ca9c 100644 --- a/server/api/content/[...path].get.ts +++ b/server/api/content/[...path].get.ts @@ -1,9 +1,11 @@ /** - * Single data endpoint: `content.handler()` dispatches `get`, `navigation`, `list` and custom handlers - * (e.g. `search-sections`). Cached per-URL — see `routeRules`. + * Single data endpoint: `content.handler()` dispatches `get`, `navigation`, `list`, `manifest` + * and `snapshot`. Must be cached per-URL by layer consumer. */ export default defineEventHandler(async (event) => { const content = await getProdContent() + await ensureSnapshotContent(content, getRouterParam(event, 'path') ?? '') + return content.handler(toWebRequest(event)) }) diff --git a/server/api/content/blob/[sha]/[...path].get.ts b/server/api/content/blob/[sha]/[...path].get.ts index fa15393..a6ad6e8 100644 --- a/server/api/content/blob/[sha]/[...path].get.ts +++ b/server/api/content/blob/[sha]/[...path].get.ts @@ -15,7 +15,23 @@ export default defineEventHandler(async (event) => { throw createError({ statusCode: 400, statusMessage: 'Invalid commit SHA' }) } + // Head-of-branch requests reuse the shared prod instance (same source ref, same per-SHA cache + // namespace) instead of minting a duplicate preview instance that would pin an LRU slot with a + // clone of production. Re-checked after `getProdContent()`, which may advance the head. + if (sha === getHeadRef()) { + const prod = await getProdContent() + if (sha === getHeadRef()) { + await ensureSnapshotContent(prod, path) + const request = toWebRequest(event) + const url = new URL(request.url) + url.pathname = url.pathname.replace(`/blob/${rawSha}`, '') + return await prod.handler(new Request(url, request)) + } + } + const content = await getPreviewContent(sha, `/api/content/blob/${sha}`) + await ensureSnapshotContent(content, path) + return await content.handler(toWebRequest(event)) }) diff --git a/server/api/content/head.get.ts b/server/api/content/head.get.ts new file mode 100644 index 0000000..11c3e14 --- /dev/null +++ b/server/api/content/head.get.ts @@ -0,0 +1,8 @@ +/** + * The commit SHA production content is pinned to, or `null` in dev + */ +export default defineEventHandler(async () => { + await getProdContent() + + return { sha: import.meta.dev ? null : getHeadRef() } +}) diff --git a/server/api/content/tree/[branch]/[...path].get.ts b/server/api/content/tree/[branch]/[...path].get.ts index 9f0ef4e..e454ce5 100644 --- a/server/api/content/tree/[branch]/[...path].get.ts +++ b/server/api/content/tree/[branch]/[...path].get.ts @@ -16,5 +16,7 @@ export default defineEventHandler(async (event) => { const sha = await resolveSha(branch, { cacheMisses: true }) const content = await getPreviewContent(sha, `/api/content/tree/${encodeURIComponent(branch)}`) + await ensureSnapshotContent(content, path) + return await content.handler(toWebRequest(event)) }) diff --git a/server/api/revalidate.post.ts b/server/api/revalidate.post.ts index ace2e2b..188d984 100644 --- a/server/api/revalidate.post.ts +++ b/server/api/revalidate.post.ts @@ -146,8 +146,10 @@ export default defineEventHandler(async (event) => { // URL the browser loads (`…/_payload.json?`). const buildId = useRuntimeConfig(event).app.buildId - // Any content change invalidates the llms indexes, the feed, and the body-derived search index. - const paths = new Set(['/llms.txt', '/llms-full.txt', '/rss.xml', '/api/content/search-sections']) + // Any content change invalidates the llms indexes and the feed. The search artifacts need no + // purge: the client hydrates from SHA-pinned `/api/content/blob//*` URLs, so a new head + // simply reads from new URLs and the old entries become unreachable. + const paths = new Set(['/llms.txt', '/llms-full.txt', '/rss.xml']) for (const f of changedFiles) { const pageUrl = pageUrlForPath(f) if (pageUrl) { diff --git a/server/utils/content.ts b/server/utils/content.ts index 57622d2..9dd0cab 100644 --- a/server/utils/content.ts +++ b/server/utils/content.ts @@ -1,4 +1,4 @@ -import { defineContentPlugin, type ComarkContent, type CacheOptions, comarkContent } from 'comark-content'; +import { type ComarkContent, type CacheOptions, comarkContent } from 'comark-content'; import fs from 'comark-content/sources/fs' import github from 'comark-content/sources/github' import rangi from 'comark/plugins/rangi' @@ -26,14 +26,6 @@ const comarkPlugins = [ }), ] -// Bound to THIS instance so a preview content instance serves its own version's sections, not production's. -const searchSectionsPlugin = defineContentPlugin(() => ({ - name: 'search-sections', - setup(ctx) { - ctx.addServeHandler('search-sections', async () => Response.json(await buildSearchSections(ctx as unknown as ComarkContent))) - }, -}))() - /** * Create a new content instance reading content at `ref` (a commit SHA or branch). `remote` forces the * GitHub source, `cache` overrides comark's (in-memory by default), `watch` is dev file watching. @@ -55,7 +47,6 @@ export async function createSourceContent( }, plugins: [ yaml(), // enable .navigation.yml to be detected - searchSectionsPlugin, tracer && tracingOtel({ tracer }), ], cache: opts.cache, @@ -76,6 +67,17 @@ export async function createSourceContent( return instance } +/** + * Snapshot artifacts must dump the full corpus, but the default init is partial (frontmatter only) + * and `cache.snapshot()` only sees bodies already in the cache. Upgrade the instance before serving + * a snapshot — memoized, so bodies parse once per instance (the FTS serve handler has the same guard). + */ +export async function ensureSnapshotContent(content: ComarkContent, path: string): Promise { + if (path.startsWith('snapshot')) { + await content.init({ partial: false }) + } +} + /** Production branch, resolved per request: content pushes skip redeploys (`vercel.json` `ignoreCommand`). */ export function targetBranch(): string { return process.env.VERCEL_GIT_COMMIT_REF || useRuntimeConfig().docs.github.branch || 'main'