From 6f616ec9bac628332938ac2990f180a923d78fee Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 9 Apr 2026 13:20:53 +0000 Subject: [PATCH] feat: multi-provider embeddings, distillation vector search, and cross-project recall - Abstract embedding provider interface with Voyage AI and OpenAI support. Config gets a 'provider' field (default: voyage, backward-compatible). Each provider reads its own env var (VOYAGE_API_KEY, OPENAI_API_KEY). - Extend vector search to distillations: schema migration 9 adds embedding BLOB to distillations table, fire-and-forget embed on store, brute-force cosine search feeds into recall RRF alongside FTS results. - Cross-project knowledge discovery in recall tool: when scope is 'all', searches knowledge entries from other projects and surfaces them tagged with the source project name. --- src/config.ts | 18 +- src/db.ts | 18 +- src/distillation.ts | 16 +- src/embedding.ts | 381 +++++++++++++++++++++++++++++++++-------- src/index.ts | 7 +- src/ltm.ts | 46 +++++ src/reflect.ts | 100 +++++++++-- test/db.test.ts | 2 +- test/embedding.test.ts | 3 + test/index.test.ts | 3 +- 10 files changed, 498 insertions(+), 96 deletions(-) diff --git a/src/config.ts b/src/config.ts index 2c0b873..bf3a715 100644 --- a/src/config.ts +++ b/src/config.ts @@ -66,21 +66,29 @@ export const LoreConfig = z.object({ * When enabled, the configured model generates 2–3 alternative query phrasings * before search, improving recall for ambiguous queries. */ queryExpansion: z.boolean().default(false), - /** Vector embedding search via Voyage AI. - * Automatically enabled when VOYAGE_API_KEY env var is set. + /** Vector embedding search. + * Supports multiple providers: "voyage" (Voyage AI, VOYAGE_API_KEY), + * "openai" (OpenAI, OPENAI_API_KEY). + * Automatically enabled when the configured provider's API key env var is set. * Set enabled: false to explicitly disable even with the key present. */ embeddings: z .object({ /** Enable/disable vector embedding search. Default: true. - * Set to false to explicitly disable even when VOYAGE_API_KEY is set. */ + * Set to false to explicitly disable even when the API key is set. */ enabled: z.boolean().default(true), - /** Voyage AI model ID. Default: voyage-code-3. */ + /** Embedding provider. Default: "voyage". + * Each provider reads its own env var for the API key: + * - "voyage": VOYAGE_API_KEY (default model: voyage-code-3, 1024 dims) + * - "openai": OPENAI_API_KEY (default model: text-embedding-3-small, 1536 dims) */ + provider: z.enum(["voyage", "openai"]).default("voyage"), + /** Model ID for the embedding provider. Default depends on provider. */ model: z.string().default("voyage-code-3"), /** Embedding dimensions. Default: 1024. */ dimensions: z.number().min(256).max(2048).default(1024), }) .default({ enabled: true, + provider: "voyage", model: "voyage-code-3", dimensions: 1024, }), @@ -89,7 +97,7 @@ export const LoreConfig = z.object({ ftsWeights: { title: 6.0, content: 2.0, category: 3.0 }, recallLimit: 10, queryExpansion: false, - embeddings: { enabled: true, model: "voyage-code-3", dimensions: 1024 }, + embeddings: { enabled: true, provider: "voyage" as const, model: "voyage-code-3", dimensions: 1024 }, }), crossProject: z.boolean().default(false), agentsFile: z diff --git a/src/db.ts b/src/db.ts index 5843071..4fe5f29 100644 --- a/src/db.ts +++ b/src/db.ts @@ -2,7 +2,7 @@ import { Database } from "bun:sqlite"; import { join, dirname } from "path"; import { mkdirSync } from "fs"; -const SCHEMA_VERSION = 8; +const SCHEMA_VERSION = 9; const MIGRATIONS: string[] = [ ` @@ -220,6 +220,14 @@ const MIGRATIONS: string[] = [ value TEXT NOT NULL ); `, + ` + -- Version 9: Embedding BLOB column for distillation vector search. + -- Same pattern as knowledge embeddings (version 8). Enables semantic + -- search over distilled session summaries via cosine similarity. + -- No backfill — entries get embedded lazily on next distillation + -- or via explicit backfill when embeddings are first enabled. + ALTER TABLE distillations ADD COLUMN embedding BLOB; + `, ]; function dataDir() { @@ -316,6 +324,14 @@ export function projectId(path: string): string | undefined { return row?.id; } +/** Look up a project's display name by its internal ID. */ +export function projectName(id: string): string | null { + const row = db() + .query("SELECT name FROM projects WHERE id = ?") + .get(id) as { name: string } | null; + return row?.name ?? null; +} + /** * Returns true if Lore has never been used before (no projects in the DB). * Must be called before ensureProject() to get an accurate result. diff --git a/src/distillation.ts b/src/distillation.ts index ad4d924..0411acc 100644 --- a/src/distillation.ts +++ b/src/distillation.ts @@ -2,6 +2,7 @@ import type { createOpencodeClient } from "@opencode-ai/sdk"; import { db, ensureProject } from "./db"; import { config } from "./config"; import * as temporal from "./temporal"; +import * as embedding from "./embedding"; import * as log from "./log"; import { DISTILLATION_SYSTEM, @@ -402,7 +403,7 @@ async function distillSegment(input: { const result = parseDistillationResult(responsePart.text); if (!result) return null; - storeDistillation({ + const distillId = storeDistillation({ projectPath: input.projectPath, sessionID: input.sessionID, observations: result.observations, @@ -410,6 +411,12 @@ async function distillSegment(input: { generation: 0, }); temporal.markDistilled(input.messages.map((m) => m.id)); + + // Fire-and-forget: embed the distillation for vector search + if (embedding.isAvailable()) { + embedding.embedDistillation(distillId, result.observations); + } + return result; } @@ -458,7 +465,7 @@ async function metaDistill(input: { // Store the meta-distillation at generation N+1 const maxGen = Math.max(...existing.map((d) => d.generation)); const allSourceIDs = existing.flatMap((d) => d.source_ids); - storeDistillation({ + const metaId = storeDistillation({ projectPath: input.projectPath, sessionID: input.sessionID, observations: result.observations, @@ -466,6 +473,11 @@ async function metaDistill(input: { generation: maxGen + 1, }); + // Fire-and-forget: embed the meta-distillation for vector search + if (embedding.isAvailable()) { + embedding.embedDistillation(metaId, result.observations); + } + // Archive the gen-0 distillations that were merged into gen-1+. // They remain searchable via recall but excluded from the in-context prefix. archiveDistillations(existing.map((d) => d.id)); diff --git a/src/embedding.ts b/src/embedding.ts index a7de592..2102262 100644 --- a/src/embedding.ts +++ b/src/embedding.ts @@ -1,84 +1,223 @@ /** - * Voyage AI embedding integration for vector search. + * Embedding integration for vector search. * - * Provides embedding generation via Voyage AI's REST API, pure-JS cosine - * similarity, and vector search over the knowledge table. All operations - * are gated behind `search.embeddings.enabled` config + `VOYAGE_API_KEY` - * env var — falls back silently to FTS-only when unavailable. + * Supports multiple embedding providers (Voyage AI, OpenAI) behind a common + * interface. Provides embedding generation, pure-JS cosine similarity, and + * vector search over the knowledge and distillation tables. All operations + * are gated behind `search.embeddings.enabled` config + the provider's API + * key env var — falls back silently to FTS-only when unavailable. */ import { db } from "./db"; import { config } from "./config"; import * as log from "./log"; -const VOYAGE_API_URL = "https://api.voyageai.com/v1/embeddings"; - // --------------------------------------------------------------------------- -// Availability +// Provider interface // --------------------------------------------------------------------------- -function getApiKey(): string | undefined { - return process.env.VOYAGE_API_KEY; -} - -/** Returns true if embedding is available. - * Active when VOYAGE_API_KEY is set, unless explicitly disabled via - * `search.embeddings.enabled: false` in .lore.json. */ -export function isAvailable(): boolean { - if (config().search.embeddings.enabled === false) return false; - return !!getApiKey(); +export interface EmbeddingProvider { + embed(texts: string[], inputType: "document" | "query"): Promise; + readonly maxBatchSize: number; } // --------------------------------------------------------------------------- -// Voyage AI API +// Voyage AI provider // --------------------------------------------------------------------------- +const VOYAGE_API_URL = "https://api.voyageai.com/v1/embeddings"; + type VoyageResponse = { data: Array<{ embedding: number[]; index: number }>; model: string; usage: { total_tokens: number }; }; +class VoyageProvider implements EmbeddingProvider { + readonly maxBatchSize = 128; + private apiKey: string; + private model: string; + private dimensions: number; + + constructor(apiKey: string, model: string, dimensions: number) { + this.apiKey = apiKey; + this.model = model; + this.dimensions = dimensions; + } + + async embed(texts: string[], inputType: "document" | "query"): Promise { + const res = await fetch(VOYAGE_API_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${this.apiKey}`, + }, + body: JSON.stringify({ + input: texts, + model: this.model, + input_type: inputType, + output_dimension: this.dimensions, + }), + }); + + if (!res.ok) { + const body = await res.text().catch(() => ""); + throw new Error(`Voyage API ${res.status}: ${body}`); + } + + const json = (await res.json()) as VoyageResponse; + const sorted = [...json.data].sort((a, b) => a.index - b.index); + return sorted.map((d) => new Float32Array(d.embedding)); + } +} + +// --------------------------------------------------------------------------- +// OpenAI provider +// --------------------------------------------------------------------------- + +const OPENAI_API_URL = "https://api.openai.com/v1/embeddings"; + +type OpenAIResponse = { + data: Array<{ embedding: number[]; index: number }>; + model: string; + usage: { prompt_tokens: number; total_tokens: number }; +}; + +class OpenAIProvider implements EmbeddingProvider { + readonly maxBatchSize = 2048; + private apiKey: string; + private model: string; + private dimensions: number; + + constructor(apiKey: string, model: string, dimensions: number) { + this.apiKey = apiKey; + this.model = model; + this.dimensions = dimensions; + } + + async embed(texts: string[], _inputType: "document" | "query"): Promise { + const body: Record = { + input: texts, + model: this.model, + }; + // OpenAI supports dimensions parameter for text-embedding-3-* models + if (this.model.startsWith("text-embedding-3")) { + body.dimensions = this.dimensions; + } + + const res = await fetch(OPENAI_API_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${this.apiKey}`, + }, + body: JSON.stringify(body), + }); + + if (!res.ok) { + const responseBody = await res.text().catch(() => ""); + throw new Error(`OpenAI API ${res.status}: ${responseBody}`); + } + + const json = (await res.json()) as OpenAIResponse; + const sorted = [...json.data].sort((a, b) => a.index - b.index); + return sorted.map((d) => new Float32Array(d.embedding)); + } +} + +// --------------------------------------------------------------------------- +// Provider resolution +// --------------------------------------------------------------------------- + +/** Default models per provider — used when config doesn't override. */ +const PROVIDER_DEFAULTS: Record = { + voyage: { model: "voyage-code-3", dimensions: 1024 }, + openai: { model: "text-embedding-3-small", dimensions: 1536 }, +}; + +/** Env var name for each provider's API key. */ +const PROVIDER_ENV_KEYS: Record = { + voyage: "VOYAGE_API_KEY", + openai: "OPENAI_API_KEY", +}; + +function getProviderApiKey(provider: string): string | undefined { + const envKey = PROVIDER_ENV_KEYS[provider]; + return envKey ? process.env[envKey] : undefined; +} + +let cachedProvider: EmbeddingProvider | null | undefined; + +function getProvider(): EmbeddingProvider | null { + if (cachedProvider !== undefined) return cachedProvider; + + const cfg = config().search.embeddings; + if (cfg.enabled === false) { + cachedProvider = null; + return null; + } + + const providerName = cfg.provider; + const apiKey = getProviderApiKey(providerName); + if (!apiKey) { + cachedProvider = null; + return null; + } + + const defaults = PROVIDER_DEFAULTS[providerName]; + const model = cfg.model === defaults?.model ? cfg.model : cfg.model; + const dimensions = cfg.dimensions; + + switch (providerName) { + case "voyage": + cachedProvider = new VoyageProvider(apiKey, model, dimensions); + break; + case "openai": + cachedProvider = new OpenAIProvider(apiKey, model, dimensions); + break; + default: + log.info(`unknown embedding provider: ${providerName}`); + cachedProvider = null; + } + + return cachedProvider; +} + +/** Reset cached provider — called when config changes. */ +export function resetProvider(): void { + cachedProvider = undefined; +} + +// --------------------------------------------------------------------------- +// Availability +// --------------------------------------------------------------------------- + +/** Returns true if embedding is available. + * Active when the configured provider's API key is set, unless explicitly + * disabled via `search.embeddings.enabled: false` in .lore.json. */ +export function isAvailable(): boolean { + return getProvider() !== null; +} + +// --------------------------------------------------------------------------- +// Public embed API +// --------------------------------------------------------------------------- + /** - * Call Voyage AI embeddings API. + * Generate embeddings for the given texts using the configured provider. * - * @param texts Array of texts to embed (max 128 per call) + * @param texts Array of texts to embed * @param inputType "document" for storage, "query" for search * @returns Float32Array per input text - * @throws On API errors or missing API key + * @throws On API errors or missing provider */ export async function embed( texts: string[], inputType: "document" | "query", ): Promise { - const apiKey = getApiKey(); - if (!apiKey) throw new Error("VOYAGE_API_KEY not set"); - - const cfg = config().search.embeddings; - - const res = await fetch(VOYAGE_API_URL, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - input: texts, - model: cfg.model, - input_type: inputType, - output_dimension: cfg.dimensions, - }), - }); - - if (!res.ok) { - const body = await res.text().catch(() => ""); - throw new Error(`Voyage API ${res.status}: ${body}`); - } - - const json = (await res.json()) as VoyageResponse; - // Sort by index to match input order (API may reorder) - const sorted = [...json.data].sort((a, b) => a.index - b.index); - return sorted.map((d) => new Float32Array(d.embedding)); + const provider = getProvider(); + if (!provider) throw new Error("No embedding provider available"); + return provider.embed(texts, inputType); } // --------------------------------------------------------------------------- @@ -121,7 +260,7 @@ export function fromBlob(blob: Buffer | Uint8Array): Float32Array { } // --------------------------------------------------------------------------- -// Vector search +// Vector search — knowledge // --------------------------------------------------------------------------- type VectorHit = { id: string; similarity: number }; @@ -150,6 +289,36 @@ export function vectorSearch( return scored.slice(0, limit); } +// --------------------------------------------------------------------------- +// Vector search — distillations +// --------------------------------------------------------------------------- + +/** + * Search non-archived distillations with embeddings by cosine similarity. + * Returns top-k entries sorted by similarity descending. + * Pure brute-force — fine for ~50 entries. + */ +export function vectorSearchDistillations( + queryEmbedding: Float32Array, + limit = 10, +): VectorHit[] { + const rows = db() + .query( + "SELECT id, embedding FROM distillations WHERE embedding IS NOT NULL AND archived = 0", + ) + .all() as Array<{ id: string; embedding: Buffer }>; + + const scored: VectorHit[] = []; + for (const row of rows) { + const vec = fromBlob(row.embedding); + const sim = cosineSimilarity(queryEmbedding, vec); + scored.push({ id: row.id, similarity: sim }); + } + + scored.sort((a, b) => b.similarity - a.similarity); + return scored.slice(0, limit); +} + // --------------------------------------------------------------------------- // Fire-and-forget embedding // --------------------------------------------------------------------------- @@ -172,7 +341,27 @@ export function embedKnowledgeEntry( .run(toBlob(vec), id); }) .catch((err) => { - log.info("embedding failed for entry", id, ":", err); + log.info("embedding failed for knowledge entry", id, ":", err); + }); +} + +/** + * Embed a distillation and store the result in the DB. + * Fire-and-forget — errors are logged, never thrown. + * The distillation remains searchable via FTS even if embedding fails. + */ +export function embedDistillation( + id: string, + observations: string, +): void { + embed([observations], "document") + .then(([vec]) => { + db() + .query("UPDATE distillations SET embedding = ? WHERE id = ?") + .run(toBlob(vec), id); + }) + .catch((err) => { + log.info("embedding failed for distillation", id, ":", err); }); } @@ -181,13 +370,13 @@ export function embedKnowledgeEntry( // --------------------------------------------------------------------------- /** - * Build a config fingerprint from model + dimensions. - * Used to detect when the embedding config changes (model swap, dimension change) - * so we can clear stale embeddings and re-embed. + * Build a config fingerprint from provider + model + dimensions. + * Used to detect when the embedding config changes (provider swap, model swap, + * dimension change) so we can clear stale embeddings and re-embed. */ function configFingerprint(): string { const cfg = config().search.embeddings; - return `${cfg.model}:${cfg.dimensions}`; + return `${cfg.provider}:${cfg.model}:${cfg.dimensions}`; } const EMBEDDING_CONFIG_KEY = "lore:embedding_config"; @@ -200,7 +389,7 @@ const EMBEDDING_CONFIG_KEY = "lore:embedding_config"; * Returns true if embeddings were cleared (full re-embed needed). */ export function checkConfigChange(): boolean { - // Read stored fingerprint from schema_version metadata (reuse the table) + // Read stored fingerprint from kv_meta const stored = db() .query("SELECT value FROM kv_meta WHERE key = ?") .get(EMBEDDING_CONFIG_KEY) as { value: string } | null; @@ -209,15 +398,20 @@ export function checkConfigChange(): boolean { if (stored && stored.value === current) return false; - // Config changed (or first run) — clear all embeddings + // Config changed (or first run) — clear all embeddings in both tables if (stored) { - const count = db() + const knowledgeCount = db() .query("SELECT COUNT(*) as n FROM knowledge WHERE embedding IS NOT NULL") .get() as { n: number }; - if (count.n > 0) { + const distillCount = db() + .query("SELECT COUNT(*) as n FROM distillations WHERE embedding IS NOT NULL") + .get() as { n: number }; + const total = knowledgeCount.n + distillCount.n; + if (total > 0) { db().query("UPDATE knowledge SET embedding = NULL").run(); + db().query("UPDATE distillations SET embedding = NULL").run(); log.info( - `embedding config changed (${stored.value} → ${current}), cleared ${count.n} stale embeddings`, + `embedding config changed (${stored.value} → ${current}), cleared ${total} stale embeddings`, ); } } @@ -233,32 +427,34 @@ export function checkConfigChange(): boolean { } // --------------------------------------------------------------------------- -// Backfill +// Backfill — knowledge // --------------------------------------------------------------------------- /** * Embed all knowledge entries that are missing embeddings. * Called on startup when embeddings are first enabled. - * Also handles config changes: if model/dimensions changed, clears + * Also handles config changes: if provider/model/dimensions changed, clears * stale embeddings first, then re-embeds all entries. * Returns the number of entries embedded. */ export async function backfillEmbeddings(): Promise { - // Detect model/dimension changes and clear stale embeddings + // Detect config changes and clear stale embeddings checkConfigChange(); + const provider = getProvider(); + if (!provider) return 0; + const rows = db() .query("SELECT id, title, content FROM knowledge WHERE embedding IS NULL AND confidence > 0.2") .all() as Array<{ id: string; title: string; content: string }>; if (!rows.length) return 0; - // Batch embed (Voyage supports up to 128 per call) - const BATCH_SIZE = 128; + const batchSize = provider.maxBatchSize; let embedded = 0; - for (let i = 0; i < rows.length; i += BATCH_SIZE) { - const batch = rows.slice(i, i + BATCH_SIZE); + for (let i = 0; i < rows.length; i += batchSize) { + const batch = rows.slice(i, i + batchSize); const texts = batch.map((r) => `${r.title}\n${r.content}`); try { @@ -281,3 +477,52 @@ export async function backfillEmbeddings(): Promise { } return embedded; } + +// --------------------------------------------------------------------------- +// Backfill — distillations +// --------------------------------------------------------------------------- + +/** + * Embed all non-archived distillations that are missing embeddings. + * Called on startup alongside knowledge backfill. + * Returns the number of distillations embedded. + */ +export async function backfillDistillationEmbeddings(): Promise { + const provider = getProvider(); + if (!provider) return 0; + + const rows = db() + .query( + "SELECT id, observations FROM distillations WHERE embedding IS NULL AND archived = 0 AND observations != ''", + ) + .all() as Array<{ id: string; observations: string }>; + + if (!rows.length) return 0; + + const batchSize = provider.maxBatchSize; + let embedded = 0; + + for (let i = 0; i < rows.length; i += batchSize) { + const batch = rows.slice(i, i + batchSize); + const texts = batch.map((r) => r.observations); + + try { + const vectors = await embed(texts, "document"); + const update = db().prepare( + "UPDATE distillations SET embedding = ? WHERE id = ?", + ); + + for (let j = 0; j < batch.length; j++) { + update.run(toBlob(vectors[j]), batch[j].id); + embedded++; + } + } catch (err) { + log.info(`distillation embedding backfill batch ${i}-${i + batch.length} failed:`, err); + } + } + + if (embedded > 0) { + log.info(`embedded ${embedded} distillations`); + } + return embedded; +} diff --git a/src/index.ts b/src/index.ts index 9d9c6ce..3be99a2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -735,9 +735,12 @@ End with "I'm ready to continue." so the agent knows to pick up where it left of // Background: backfill embeddings for entries that don't have one yet. // Fires once when embeddings are first enabled — subsequent entries - // get embedded on create/update via ltm.ts hooks. + // get embedded on create/update via ltm.ts and distillation.ts hooks. if (embedding.isAvailable()) { - embedding.backfillEmbeddings().catch((err) => { + Promise.all([ + embedding.backfillEmbeddings(), + embedding.backfillDistillationEmbeddings(), + ]).catch((err) => { log.info("embedding backfill failed:", err); }); } diff --git a/src/ltm.ts b/src/ltm.ts index 6db2bf3..756d5ec 100644 --- a/src/ltm.ts +++ b/src/ltm.ts @@ -514,6 +514,52 @@ export function searchScored(input: { } } +/** + * Search knowledge entries from OTHER projects — entries that are project-specific + * (cross_project=0) and belong to a different project_id than the given one. + * Used by the recall tool in "all" scope to surface relevant knowledge from + * the user's other projects ("tunnel" discovery across projects). + */ +export function searchScoredOtherProjects(input: { + query: string; + excludeProjectPath: string; + limit?: number; +}): ScoredKnowledgeEntry[] { + const limit = input.limit ?? 10; + const q = ftsQuery(input.query); + if (q === EMPTY_QUERY) return []; + + const excludePid = ensureProject(input.excludeProjectPath); + const { title, content, category } = ftsWeights(); + + // Find entries from other projects that are NOT cross-project (those are + // already included in the normal search via the cross_project=1 filter). + // Also exclude entries with no project_id (global) — already included. + const ftsSQL = `SELECT ${KNOWLEDGE_COLS_K}, bm25(knowledge_fts, ?, ?, ?) as rank FROM knowledge k + JOIN knowledge_fts f ON k.rowid = f.rowid + WHERE knowledge_fts MATCH ? + AND k.project_id IS NOT NULL + AND k.project_id != ? + AND k.cross_project = 0 + AND k.confidence > 0.2 + ORDER BY rank LIMIT ?`; + + const ftsParams = [title, content, category, q, excludePid, limit]; + + try { + const results = db().query(ftsSQL).all(...ftsParams) as ScoredKnowledgeEntry[]; + if (results.length) return results; + + // AND returned nothing — try OR fallback + const qOr = ftsQueryOr(input.query); + if (qOr === EMPTY_QUERY) return []; + const ftsParamsOr = [title, content, category, qOr, excludePid, limit]; + return db().query(ftsSQL).all(...ftsParamsOr) as ScoredKnowledgeEntry[]; + } catch { + return []; + } +} + export function get(id: string): KnowledgeEntry | null { return db() .query(`SELECT ${KNOWLEDGE_COLS} FROM knowledge WHERE id = ?`) diff --git a/src/reflect.ts b/src/reflect.ts index 559478a..b63bc4e 100644 --- a/src/reflect.ts +++ b/src/reflect.ts @@ -4,7 +4,7 @@ import * as temporal from "./temporal"; import * as ltm from "./ltm"; import * as log from "./log"; import * as embedding from "./embedding"; -import { db, ensureProject } from "./db"; +import { db, ensureProject, projectName } from "./db"; import { ftsQuery, ftsQueryOr, EMPTY_QUERY, reciprocalRankFusion, expandQuery } from "./search"; import { serialize, inline, h, p, ul, lip, liph, t, root } from "./markdown"; import type { LoreConfig } from "./config"; @@ -146,6 +146,7 @@ function formatResults(input: { type TaggedResult = | { source: "knowledge"; item: ltm.ScoredKnowledgeEntry } + | { source: "cross-knowledge"; item: ltm.ScoredKnowledgeEntry; projectLabel: string } | { source: "distillation"; item: ScoredDistillation } | { source: "temporal"; item: temporal.ScoredTemporalMessage }; @@ -165,6 +166,14 @@ function formatFusedResults( ), ); } + case "cross-knowledge": { + const k = tagged.item; + return liph( + t( + `**[knowledge/${k.category} from: ${tagged.projectLabel}]** ${inline(k.title)}: ${inline(k.content)}`, + ), + ); + } case "distillation": { const d = tagged.item; const preview = @@ -311,30 +320,89 @@ export function createRecallTool( ); } - // Vector search: embed query and find similar knowledge entries - if (embedding.isAvailable() && knowledgeEnabled && scope !== "session") { + // Vector search: embed query and find similar knowledge entries + distillations + if (embedding.isAvailable() && scope !== "session") { try { const [queryVec] = await embedding.embed([args.query], "query"); - const vectorHits = embedding.vectorSearch(queryVec, limit); - const vectorTagged: TaggedResult[] = []; - for (const hit of vectorHits) { - const entry = ltm.get(hit.id); - if (entry) { - vectorTagged.push({ - source: "knowledge", - item: { ...entry, rank: -hit.similarity }, + + // Knowledge vector search + if (knowledgeEnabled) { + const vectorHits = embedding.vectorSearch(queryVec, limit); + const vectorTagged: TaggedResult[] = []; + for (const hit of vectorHits) { + const entry = ltm.get(hit.id); + if (entry) { + vectorTagged.push({ + source: "knowledge", + item: { ...entry, rank: -hit.similarity }, + }); + } + } + if (vectorTagged.length) { + // Same `k:` key prefix as BM25 knowledge — RRF merges, not duplicates + allRrfLists.push({ + items: vectorTagged, + key: (r) => `k:${r.item.id}`, + }); + } + } + + // Distillation vector search + if (scope !== "knowledge") { + const distVectorHits = embedding.vectorSearchDistillations(queryVec, limit); + const distVectorTagged: TaggedResult[] = distVectorHits + .map((hit): TaggedResult | null => { + // Look up the distillation to get its full fields + const row = db() + .query( + "SELECT id, observations, generation, created_at, session_id FROM distillations WHERE id = ?", + ) + .get(hit.id) as Distillation | null; + if (!row) return null; + return { + source: "distillation", + item: { ...row, rank: -hit.similarity }, + }; + }) + .filter((r): r is TaggedResult => r !== null); + if (distVectorTagged.length) { + // Same `d:` key prefix as BM25 distillations — RRF merges, not duplicates + allRrfLists.push({ + items: distVectorTagged, + key: (r) => `d:${r.item.id}`, }); } } - if (vectorTagged.length) { - // Same `k:` key prefix as BM25 knowledge — RRF merges, not duplicates + } catch (err) { + log.info("recall: vector search failed:", err); + } + } + + // Cross-project knowledge discovery: find relevant entries from other projects. + // Only runs in "all" scope — "project", "session", and "knowledge" scopes are + // intentionally limited to the current project context. + if (knowledgeEnabled && scope === "all") { + try { + const crossProjectResults = ltm.searchScoredOtherProjects({ + query: args.query, + excludeProjectPath: projectPath, + limit, + }); + if (crossProjectResults.length) { allRrfLists.push({ - items: vectorTagged, - key: (r) => `k:${r.item.id}`, + items: crossProjectResults.map((item: ltm.ScoredKnowledgeEntry) => { + const label = (item.project_id ? projectName(item.project_id) : null) ?? "other"; + return { + source: "cross-knowledge" as const, + item, + projectLabel: label, + } as TaggedResult; + }), + key: (r) => `xk:${r.item.id}`, }); } } catch (err) { - log.info("recall: vector search failed:", err); + log.info("recall: cross-project knowledge search failed:", err); } } diff --git a/test/db.test.ts b/test/db.test.ts index fe4679e..10df76f 100644 --- a/test/db.test.ts +++ b/test/db.test.ts @@ -21,7 +21,7 @@ describe("db", () => { const row = db().query("SELECT version FROM schema_version").get() as { version: number; }; - expect(row.version).toBe(8); + expect(row.version).toBe(9); }); test("distillation_fts virtual table exists", () => { diff --git a/test/embedding.test.ts b/test/embedding.test.ts index 79f0ee3..600bbfc 100644 --- a/test/embedding.test.ts +++ b/test/embedding.test.ts @@ -7,6 +7,7 @@ import { isAvailable, vectorSearch, checkConfigChange, + resetProvider, } from "../src/embedding"; describe("cosineSimilarity", () => { @@ -86,8 +87,10 @@ describe("isAvailable", () => { // In test environment, VOYAGE_API_KEY should not be set const original = process.env.VOYAGE_API_KEY; delete process.env.VOYAGE_API_KEY; + resetProvider(); // Clear cached provider so isAvailable re-evaluates expect(isAvailable()).toBe(false); if (original) process.env.VOYAGE_API_KEY = original; + resetProvider(); // Restore cached provider state }); }); diff --git a/test/index.test.ts b/test/index.test.ts index a944985..4878783 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -808,7 +808,8 @@ function restoreDistillationTables() { token_count INTEGER DEFAULT 0, created_at INTEGER NOT NULL DEFAULT (unixepoch()), observations TEXT NOT NULL DEFAULT '', - archived INTEGER NOT NULL DEFAULT 0 + archived INTEGER NOT NULL DEFAULT 0, + embedding BLOB ) `); // Post-migration indexes: compound indexes from v6, single-column idx_distillation_project dropped