From 94ee66d4e80e3c1e6157eb10bbf432c0fa95c288 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 7 Jul 2026 14:14:34 -0600 Subject: [PATCH] feat(rag): expose knowledge improvement lifecycle --- README.md | 39 +++ docs/eval/rag-eval-roadmap.md | 9 +- src/autodata/powered.ts | 4 +- src/index.ts | 1 + src/lint.ts | 4 +- src/rag-improvement-loop.ts | 484 +++++++++++++++++++++++++++++ src/wikilinks.ts | 4 +- tests/rag-improvement-loop.test.ts | 162 ++++++++++ 8 files changed, 702 insertions(+), 5 deletions(-) create mode 100644 src/rag-improvement-loop.ts create mode 100644 tests/rag-improvement-loop.test.ts diff --git a/README.md b/README.md index 3713c6e..3da17f3 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,8 @@ Two ways in, depending on what you're doing: - *"Grow the KB as a researcher"* → [`runKnowledgeResearchLoop`](#research-loop) (deterministic mechanics; your agent owns judgment), [`runTwoAgentResearchLoop`](#two-agent-research-loop) (researcher proposes, verifier checks + fills gaps, offline), or the sandbox [researcher profile](#researcher-profile) for `runLoop`. - *"Spawn one researcher per sub-topic and stop when the KB is ready"* → [`runResearchSupervisor`](#research-supervisor) (a supervisor brain sizes the topology over a `Scope`; LIVE, needs creds). - *"Tune retrieval for a knowledge base"* → `runRetrievalImprovementLoop` in the [Agent-Eval integration](#agent-eval-integration) section. + - *"Improve the whole RAG knowledge base"* → `runRagKnowledgeImprovementLoop` in the [Agent-Eval integration](#agent-eval-integration) section. + It exposes retrieval tuning, gap diagnosis, knowledge acquisition/update, answer-quality checks, and promotion as one typed lifecycle. - *"Does this candidate KB actually improve task success?"* → run an [agent-eval improvement loop](#agent-eval-integration) over KB variants, then `knowledgeReleaseReport` for the promotion decision. - *"Keep live authorities fresh"* → [pluggable sources](#pluggable-knowledge-sources) + `detectChanges` → eval re-runs. @@ -92,6 +94,11 @@ from `@tangle-network/agent-knowledge`. - `createKnowledgeControlLoopAdapter()` maps those mechanics into `agent-eval`'s `runAgentControlLoop()` so products can plug in their own proposer, reviewer, and driver policies. +- `runRagKnowledgeImprovementLoop()` coordinates the whole RAG improvement + lifecycle. Retrieval tuning, diagnosis, acquisition, KB update, + answer-quality eval, and promotion are separate typed phases so products can + plug in browser agents, coding agents, connectors, or deterministic policies + without this package hardcoding an agent runner. - Zod schemas define the stable wire shape. - Graph/search/lint are deterministic and fast. - `searchKnowledge` returns hits with three score fields. `score` and @@ -116,6 +123,38 @@ readiness/eval machinery without making `agent-knowledge` own the database. ## Agent-Eval Integration +Use `runRagKnowledgeImprovementLoop` when the product question is broader than retrieval: +can the system find the gaps, gather or update knowledge, prove generated answers still behave, and decide whether to promote? +`agent-knowledge` owns the knowledge/eval contract; the caller supplies the research, coding, connector, and answer-eval hooks. + +```ts +import { runRagKnowledgeImprovementLoop } from '@tangle-network/agent-knowledge' + +const result = await runRagKnowledgeImprovementLoop({ + goal: 'Improve the support RAG KB', + retrieval: { + baseline: { k: 5, hybrid: false }, + scenarios: trainRetrievalScenarios, + holdoutScenarios, + index, + searchSpace: { k: [5, 10, 20], hybrid: [false, true] }, + targetRecall: 0.9, + }, + diagnose: async ({ retrieval }) => diagnoseRagGaps(retrieval), + acquireKnowledge: async ({ findings }) => researchMissingSources(findings), + knowledgeResearch: { root: './kb' }, + evaluateAnswers: async ({ knowledgeUpdate }) => runAnswerEval(knowledgeUpdate), + promote: async ({ retrieval, answerQuality }) => + decidePromotion({ retrieval, answerQuality }), + requiredPhases: ['retrieval-tuning', 'knowledge-update', 'answer-quality', 'promotion'], +}) + +console.log(result.promotion) +``` + +If a required phase is missing its hook, the loop throws. +That keeps the public API from reporting a fake “RAG improved” result when the caller only wired retrieval or only wired a researcher. + Use retrieval eval when the question is whether a retrieval/RAG config can find the right knowledge before an agent reasons over it. The labels should name stable pages, source records, anchors, or source spans, not ephemeral chunk IDs. The completion roadmap is in [`docs/eval/rag-eval-roadmap.md`](docs/eval/rag-eval-roadmap.md). diff --git a/docs/eval/rag-eval-roadmap.md b/docs/eval/rag-eval-roadmap.md index c5d0471..b948cdc 100644 --- a/docs/eval/rag-eval-roadmap.md +++ b/docs/eval/rag-eval-roadmap.md @@ -20,9 +20,12 @@ SOTA RAG evaluation requires retrieval quality, context quality, generated-answe Done: - `runRetrievalImprovementLoop()` auto-searches retrieval configs through `agent-eval`. +- `runRagKnowledgeImprovementLoop()` exposes the whole RAG lifecycle as typed phases: + retrieval tuning, gap diagnosis, knowledge acquisition, knowledge update, answer-quality eval, and promotion. - Retrieval scenarios can label pages, page paths, sources, source anchors, and source spans. - The retrieval judge reports recall, MRR, nDCG, precision@k, cost, and held-out promotion. - The loop is tested with a real `agent-eval` run where `{ k: 2 }` beats `{ k: 1 }`. +- The lifecycle loop is tested both with pluggable phase hooks and with a real local KB update through `runKnowledgeResearchLoop()`. Not done: @@ -31,6 +34,8 @@ Not done: - Citation support and claim-level groundedness. - Abstention and unanswerable-question scoring. - Slice-level reporting for freshness, distractors, multi-hop, and long-tail cases. +- Packaged runtime adapters for browser/coding/research agents. + The lifecycle API accepts those agents as hooks today; it does not hardcode provider-specific workers. ## Completion Criteria @@ -101,6 +106,7 @@ Never tune on holdout. Ship criteria: - `runRetrievalImprovementLoop()` gates retrieval config changes. +- `runRagKnowledgeImprovementLoop()` is the default front door when retrieval changes, source acquisition, KB updates, answer checks, and promotion must run together. - Answer-quality eval gates prompt and synthesis changes. - Reports persist run id, commit, config hash, dataset hash, metric versions, cost, latency, and traces. - A promoted candidate must improve the target metric without violating faithfulness, abstention, cost, or latency limits. @@ -111,4 +117,5 @@ Ship criteria: 2. Add context relevance and context sufficiency judges over retrieved hits. 3. Add forbidden/stale source targets to retrieval scenarios. 4. Add slice-level aggregation helpers for the required six eval slices. -5. Add a CLI command that runs the retrieval loop and writes a reproducible report under `.agent-knowledge/eval/`. +5. Add packaged adapters that turn `agent-runtime` browser, research, and coding agents into `runRagKnowledgeImprovementLoop()` hooks. +6. Add a CLI command that runs the lifecycle loop and writes a reproducible report under `.agent-knowledge/eval/`. diff --git a/src/autodata/powered.ts b/src/autodata/powered.ts index 3140e8e..37f9d2f 100644 --- a/src/autodata/powered.ts +++ b/src/autodata/powered.ts @@ -438,11 +438,11 @@ async function main(): Promise { printReport(stats, spendUsd) // Emit the machine-readable result alongside the prose, for the doc + any re-analysis. - console.log('RESULT_JSON ' + JSON.stringify({ ...stats, spendUsd })) + console.log(`RESULT_JSON ${JSON.stringify({ ...stats, spendUsd })}`) } // Only auto-run when invoked directly (keeps `analyzeTrails` importable + unit-testable). -if (process.argv[1] && process.argv[1].endsWith('powered.ts')) { +if (process.argv[1]?.endsWith('powered.ts')) { main().catch((err) => { console.error(err) process.exit(1) diff --git a/src/index.ts b/src/index.ts index 7c55af8..e6c3e10 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,6 +21,7 @@ export * from './material-facts-metric' export * from './memory/index' export * from './proposals' export * from './propose-from-finding' +export * from './rag-improvement-loop' export * from './release' export * from './research-driving-driver' export * from './research-loop' diff --git a/src/lint.ts b/src/lint.ts index d7ccce5..8170107 100644 --- a/src/lint.ts +++ b/src/lint.ts @@ -141,8 +141,10 @@ function extractSourceRefs(text: string): Array<{ sourceId: string; anchorId?: s const refs: Array<{ sourceId: string; anchorId?: string }> = [] const regex = /\[\^([A-Za-z0-9_-]+)(?:#([A-Za-z0-9_.:-]+))?\]/g let match: RegExpExecArray | null - while ((match = regex.exec(text)) !== null) { + match = regex.exec(text) + while (match !== null) { refs.push({ sourceId: match[1]!, anchorId: match[2] }) + match = regex.exec(text) } return refs } diff --git a/src/rag-improvement-loop.ts b/src/rag-improvement-loop.ts new file mode 100644 index 0000000..b6a5915 --- /dev/null +++ b/src/rag-improvement-loop.ts @@ -0,0 +1,484 @@ +import type { JsonValue } from '@tangle-network/agent-eval/campaign' +import { + type KnowledgeResearchLoopDecision, + type KnowledgeResearchLoopResult, + type RunKnowledgeResearchLoopOptions, + runKnowledgeResearchLoop, +} from './research-loop' +import { + type RunRetrievalImprovementLoopOptions, + type RunRetrievalImprovementLoopResult, + runRetrievalImprovementLoop, +} from './retrieval-eval' + +export type RagKnowledgeImprovementPhase = + | 'retrieval-tuning' + | 'gap-diagnosis' + | 'knowledge-acquisition' + | 'knowledge-update' + | 'answer-quality' + | 'promotion' + +export type RagKnowledgeImprovementPhaseStatus = 'completed' | 'skipped' | 'failed' + +export type RagGapKind = + | 'missing-source' + | 'stale-source' + | 'retrieval-miss' + | 'retrieval-noise' + | 'chunking-mismatch' + | 'missing-multihop-evidence' + | 'generator-unsupported-claim' + | 'citation-mismatch' + | 'incorrect-abstention' + | 'unknown' + +export type RagGapSeverity = 'info' | 'warning' | 'error' | 'critical' + +export interface RagGapFinding { + id: string + kind: RagGapKind + severity: RagGapSeverity + message: string + scenarioId?: string + evidence?: Record +} + +export interface RagKnowledgeImprovementPhaseResult { + phase: RagKnowledgeImprovementPhase + status: RagKnowledgeImprovementPhaseStatus + summary: string + startedAt: string + finishedAt: string + metadata?: Record +} + +export interface RagPhaseInputBase { + goal: string + phases: readonly RagKnowledgeImprovementPhaseResult[] + signal?: AbortSignal +} + +export interface RagDiagnosisInput extends RagPhaseInputBase { + retrieval?: RunRetrievalImprovementLoopResult +} + +export interface RagKnowledgeAcquisitionInput extends RagPhaseInputBase { + retrieval?: RunRetrievalImprovementLoopResult + findings: readonly RagGapFinding[] +} + +export interface RagKnowledgeUpdateInput extends RagPhaseInputBase { + retrieval?: RunRetrievalImprovementLoopResult + findings: readonly RagGapFinding[] + acquisition?: KnowledgeResearchLoopDecision +} + +export interface RagKnowledgeUpdateResult { + applied: boolean + summary: string + research?: KnowledgeResearchLoopResult + metadata?: Record +} + +export interface RagAnswerQualityInput extends RagPhaseInputBase { + retrieval?: RunRetrievalImprovementLoopResult + findings: readonly RagGapFinding[] + acquisition?: KnowledgeResearchLoopDecision + knowledgeUpdate?: RagKnowledgeUpdateResult +} + +export interface RagAnswerQualityResult { + passed: boolean + metrics: Record + findings?: readonly RagGapFinding[] + metadata?: Record +} + +export interface RagPromotionInput extends RagPhaseInputBase { + retrieval?: RunRetrievalImprovementLoopResult + findings: readonly RagGapFinding[] + acquisition?: KnowledgeResearchLoopDecision + knowledgeUpdate?: RagKnowledgeUpdateResult + answerQuality?: RagAnswerQualityResult +} + +export interface RagPromotionResult { + promoted: boolean + reason: string + metadata?: Record +} + +export interface RagKnowledgeResearchOptions + extends Omit { + goal?: string + step?: RunKnowledgeResearchLoopOptions['step'] +} + +export interface RunRagKnowledgeImprovementLoopOptions { + goal: string + retrieval?: RunRetrievalImprovementLoopOptions + diagnose?: (input: RagDiagnosisInput) => MaybePromise + acquireKnowledge?: ( + input: RagKnowledgeAcquisitionInput, + ) => MaybePromise + knowledgeResearch?: RagKnowledgeResearchOptions + updateKnowledge?: (input: RagKnowledgeUpdateInput) => MaybePromise + evaluateAnswers?: (input: RagAnswerQualityInput) => MaybePromise + promote?: (input: RagPromotionInput) => MaybePromise + enabledPhases?: readonly RagKnowledgeImprovementPhase[] + requiredPhases?: readonly RagKnowledgeImprovementPhase[] + signal?: AbortSignal + now?: () => Date +} + +export interface RunRagKnowledgeImprovementLoopResult { + goal: string + phases: readonly RagKnowledgeImprovementPhaseResult[] + retrieval?: RunRetrievalImprovementLoopResult + findings: readonly RagGapFinding[] + acquisition?: KnowledgeResearchLoopDecision + knowledgeUpdate?: RagKnowledgeUpdateResult + answerQuality?: RagAnswerQualityResult + promotion?: RagPromotionResult +} + +type MaybePromise = T | Promise + +export async function runRagKnowledgeImprovementLoop( + options: RunRagKnowledgeImprovementLoopOptions, +): Promise { + assertConfiguredRequiredPhases(options) + const now = options.now ?? (() => new Date()) + const phases: RagKnowledgeImprovementPhaseResult[] = [] + let retrieval: RunRetrievalImprovementLoopResult | undefined + let findings: RagGapFinding[] = [] + let acquisition: KnowledgeResearchLoopDecision | undefined + let knowledgeUpdate: RagKnowledgeUpdateResult | undefined + let answerQuality: RagAnswerQualityResult | undefined + let promotion: RagPromotionResult | undefined + + if (phaseEnabled(options, 'retrieval-tuning')) { + if (options.retrieval) { + retrieval = await runPhase( + phases, + now, + 'retrieval-tuning', + async () => { + assertNotAborted(options.signal) + return runRetrievalImprovementLoop(options.retrieval!) + }, + summarizeRetrievalResult, + ) + } else { + skipPhase(phases, now, 'retrieval-tuning', 'no retrieval options provided') + } + } + + if (phaseEnabled(options, 'gap-diagnosis')) { + if (options.diagnose) { + findings = [ + ...(await runPhase( + phases, + now, + 'gap-diagnosis', + async () => { + assertNotAborted(options.signal) + return options.diagnose!({ + goal: options.goal, + phases, + signal: options.signal, + retrieval, + }) + }, + (diagnosed) => `${diagnosed.length} finding(s)`, + )), + ] + } else { + skipPhase(phases, now, 'gap-diagnosis', 'no diagnosis hook provided') + } + } + + if (phaseEnabled(options, 'knowledge-acquisition')) { + if (options.acquireKnowledge) { + acquisition = await runPhase( + phases, + now, + 'knowledge-acquisition', + async () => { + assertNotAborted(options.signal) + return options.acquireKnowledge!({ + goal: options.goal, + phases, + signal: options.signal, + retrieval, + findings, + }) + }, + summarizeAcquisitionDecision, + ) + } else { + skipPhase(phases, now, 'knowledge-acquisition', 'no acquisition hook provided') + } + } + + if (phaseEnabled(options, 'knowledge-update')) { + if (options.updateKnowledge) { + knowledgeUpdate = await runPhase( + phases, + now, + 'knowledge-update', + async () => { + assertNotAborted(options.signal) + return options.updateKnowledge!({ + goal: options.goal, + phases, + signal: options.signal, + retrieval, + findings, + acquisition, + }) + }, + (result) => result.summary, + ) + } else if (options.knowledgeResearch) { + knowledgeUpdate = await runPhase( + phases, + now, + 'knowledge-update', + async () => { + assertNotAborted(options.signal) + return runKnowledgeResearchUpdate(options, acquisition) + }, + (result) => result.summary, + ) + } else { + skipPhase(phases, now, 'knowledge-update', 'no update hook or research loop provided') + } + } + + if (phaseEnabled(options, 'answer-quality')) { + if (options.evaluateAnswers) { + answerQuality = await runPhase( + phases, + now, + 'answer-quality', + async () => { + assertNotAborted(options.signal) + return options.evaluateAnswers!({ + goal: options.goal, + phases, + signal: options.signal, + retrieval, + findings, + acquisition, + knowledgeUpdate, + }) + }, + summarizeAnswerQuality, + ) + findings = [...findings, ...(answerQuality.findings ?? [])] + } else { + skipPhase(phases, now, 'answer-quality', 'no answer-quality hook provided') + } + } + + if (phaseEnabled(options, 'promotion')) { + if (options.promote) { + promotion = await runPhase( + phases, + now, + 'promotion', + async () => { + assertNotAborted(options.signal) + return options.promote!({ + goal: options.goal, + phases, + signal: options.signal, + retrieval, + findings, + acquisition, + knowledgeUpdate, + answerQuality, + }) + }, + (result) => `${result.promoted ? 'promoted' : 'held'}: ${result.reason}`, + ) + } else { + skipPhase(phases, now, 'promotion', 'no promotion hook provided') + } + } + + return { + goal: options.goal, + phases, + retrieval, + findings, + acquisition, + knowledgeUpdate, + answerQuality, + promotion, + } +} + +async function runKnowledgeResearchUpdate( + options: RunRagKnowledgeImprovementLoopOptions, + acquisition: KnowledgeResearchLoopDecision | undefined, +): Promise { + const research = options.knowledgeResearch + if (!research) { + throw new Error('knowledgeResearch options are required to run the knowledge update phase') + } + const { goal, step, ...rest } = research + const researchStep = step ?? acquisitionBackedResearchStep(acquisition) + const maxIterations = step ? rest.maxIterations : 1 + const result = await runKnowledgeResearchLoop({ + ...rest, + goal: goal ?? options.goal, + maxIterations, + signal: options.signal, + step: researchStep, + }) + return { + applied: result.steps.some((stepResult) => { + return stepResult.addedSources.length > 0 || Boolean(stepResult.applied) + }), + summary: `${result.iterations} research iteration(s); done=${String(result.done)}`, + research: result, + } +} + +function acquisitionBackedResearchStep( + acquisition: KnowledgeResearchLoopDecision | undefined, +): RunKnowledgeResearchLoopOptions['step'] { + if (!acquisition) { + throw new Error( + 'knowledgeResearch requires either a step hook or a knowledge-acquisition result to apply', + ) + } + return () => ({ ...acquisition, done: acquisition.done ?? true }) +} + +async function runPhase( + phases: RagKnowledgeImprovementPhaseResult[], + now: () => Date, + phase: RagKnowledgeImprovementPhase, + action: () => MaybePromise, + summarize: (result: T) => string, +): Promise { + const startedAt = now().toISOString() + try { + const result = await action() + phases.push({ + phase, + status: 'completed', + summary: summarize(result), + startedAt, + finishedAt: now().toISOString(), + }) + return result + } catch (error) { + phases.push({ + phase, + status: 'failed', + summary: (error as Error).message, + startedAt, + finishedAt: now().toISOString(), + }) + throw error + } +} + +function skipPhase( + phases: RagKnowledgeImprovementPhaseResult[], + now: () => Date, + phase: RagKnowledgeImprovementPhase, + summary: string, +): void { + const timestamp = now().toISOString() + phases.push({ phase, status: 'skipped', summary, startedAt: timestamp, finishedAt: timestamp }) +} + +function summarizeRetrievalResult(result: RunRetrievalImprovementLoopResult): string { + return `${result.candidates.length} candidate(s); winner=${JSON.stringify(result.winnerConfig)}` +} + +function summarizeAcquisitionDecision(decision: KnowledgeResearchLoopDecision): string { + const sourcePathCount = decision.sourcePaths?.length ?? 0 + const sourceTextCount = decision.sourceTexts?.length ?? 0 + const proposal = decision.proposalText ? 'proposal' : 'no proposal' + return `${sourcePathCount} path source(s), ${sourceTextCount} text source(s), ${proposal}` +} + +function summarizeAnswerQuality(result: RagAnswerQualityResult): string { + const metrics = Object.entries(result.metrics) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, value]) => `${key}=${formatMetric(value)}`) + .join(', ') + return `${result.passed ? 'passed' : 'failed'}${metrics ? `; ${metrics}` : ''}` +} + +function formatMetric(value: number): string { + return Number.isFinite(value) ? value.toFixed(3) : String(value) +} + +function phaseEnabled( + options: RunRagKnowledgeImprovementLoopOptions, + phase: RagKnowledgeImprovementPhase, +): boolean { + return !options.enabledPhases || options.enabledPhases.includes(phase) +} + +function assertConfiguredRequiredPhases(options: RunRagKnowledgeImprovementLoopOptions): void { + for (const phase of options.requiredPhases ?? []) { + if (!phaseEnabled(options, phase)) { + throw new Error(`required phase ${phase} is not enabled`) + } + if (!phaseConfigured(options, phase)) { + throw new Error(requiredPhaseMessage(phase)) + } + } +} + +function phaseConfigured( + options: RunRagKnowledgeImprovementLoopOptions, + phase: RagKnowledgeImprovementPhase, +): boolean { + switch (phase) { + case 'retrieval-tuning': + return Boolean(options.retrieval) + case 'gap-diagnosis': + return Boolean(options.diagnose) + case 'knowledge-acquisition': + return Boolean(options.acquireKnowledge) + case 'knowledge-update': + return Boolean(options.updateKnowledge ?? options.knowledgeResearch) + case 'answer-quality': + return Boolean(options.evaluateAnswers) + case 'promotion': + return Boolean(options.promote) + } +} + +function requiredPhaseMessage(phase: RagKnowledgeImprovementPhase): string { + switch (phase) { + case 'retrieval-tuning': + return 'required phase retrieval-tuning requires retrieval options' + case 'gap-diagnosis': + return 'required phase gap-diagnosis requires a diagnose hook' + case 'knowledge-acquisition': + return 'required phase knowledge-acquisition requires an acquireKnowledge hook' + case 'knowledge-update': + return 'required phase knowledge-update requires updateKnowledge or knowledgeResearch' + case 'answer-quality': + return 'required phase answer-quality requires an evaluateAnswers hook' + case 'promotion': + return 'required phase promotion requires a promote hook' + } +} + +function assertNotAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) { + throw new Error('RAG knowledge improvement loop aborted') + } +} diff --git a/src/wikilinks.ts b/src/wikilinks.ts index 7b7cc2c..0c04588 100644 --- a/src/wikilinks.ts +++ b/src/wikilinks.ts @@ -4,8 +4,10 @@ export function extractWikilinks(content: string): string[] { const links: string[] = [] const regex = new RegExp(WIKILINK_REGEX.source, 'g') let match: RegExpExecArray | null - while ((match = regex.exec(content)) !== null) { + match = regex.exec(content) + while (match !== null) { links.push(match[1]!.trim()) + match = regex.exec(content) } return [...new Set(links)] } diff --git a/tests/rag-improvement-loop.test.ts b/tests/rag-improvement-loop.test.ts new file mode 100644 index 0000000..c0502c6 --- /dev/null +++ b/tests/rag-improvement-loop.test.ts @@ -0,0 +1,162 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { afterEach, describe, expect, it } from 'vitest' +import { type RetrievalEvalScenario, runRagKnowledgeImprovementLoop } from '../src/index' + +const tempRoots: string[] = [] + +afterEach(async () => { + await Promise.all(tempRoots.map((root) => rm(root, { recursive: true, force: true }))) + tempRoots.length = 0 +}) + +describe('RAG knowledge improvement loop', () => { + it('exposes retrieval, diagnosis, acquisition, update, answer eval, and promotion phases', async () => { + const calls: string[] = [] + const trainScenario: RetrievalEvalScenario = { + id: 'q-train', + kind: 'retrieval-eval', + query: 'needs second result', + expected: { kind: 'page', pageId: 'gold' }, + } + const holdoutScenario: RetrievalEvalScenario = { + id: 'q-holdout', + kind: 'retrieval-eval', + query: 'held out needs second result', + expected: { kind: 'page', pageId: 'gold' }, + } + + const result = await runRagKnowledgeImprovementLoop({ + goal: 'Improve support RAG', + retrieval: { + baseline: { k: 1 }, + scenarios: [trainScenario], + holdoutScenarios: [holdoutScenario], + searchSpace: { k: [1, 2] }, + retrieve: async ({ k }) => ({ + hits: [ + { pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 }, + ...(k >= 2 ? [{ pageId: 'gold', path: 'knowledge/gold.md', rank: 2 }] : []), + ], + }), + targetRecall: 1, + deltaThreshold: 0.01, + populationSize: 1, + maxGenerations: 1, + runDir: 'memory://rag-lifecycle-retrieval-test', + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + }, + diagnose({ retrieval }) { + calls.push('diagnose') + expect(retrieval?.winnerConfig).toMatchObject({ k: 2 }) + return [ + { + id: 'missing-refund-policy', + kind: 'missing-source', + severity: 'error', + message: 'Refund policy source is missing.', + }, + ] + }, + acquireKnowledge({ findings }) { + calls.push('acquire') + expect(findings).toHaveLength(1) + return { + sourceTexts: [ + { + uri: 'research://refund-policy', + title: 'Refund Policy', + text: 'Refunds are available within 30 days.', + }, + ], + proposalText: 'proposed KB update', + done: true, + } + }, + updateKnowledge({ acquisition }) { + calls.push('update') + expect(acquisition?.sourceTexts).toHaveLength(1) + return { applied: true, summary: 'external vector DB updated' } + }, + evaluateAnswers({ knowledgeUpdate }) { + calls.push('answer') + expect(knowledgeUpdate?.applied).toBe(true) + return { passed: true, metrics: { faithfulness: 1, answer_relevance: 0.95 } } + }, + promote({ answerQuality }) { + calls.push('promote') + expect(answerQuality?.passed).toBe(true) + return { promoted: true, reason: 'retrieval and answer checks passed' } + }, + }) + + expect(calls).toEqual(['diagnose', 'acquire', 'update', 'answer', 'promote']) + expect(result.retrieval?.winnerConfig).toMatchObject({ k: 2 }) + expect(result.findings).toHaveLength(1) + expect(result.knowledgeUpdate?.applied).toBe(true) + expect(result.answerQuality?.passed).toBe(true) + expect(result.promotion?.promoted).toBe(true) + expect(result.phases.map((phase) => `${phase.phase}:${phase.status}`)).toEqual([ + 'retrieval-tuning:completed', + 'gap-diagnosis:completed', + 'knowledge-acquisition:completed', + 'knowledge-update:completed', + 'answer-quality:completed', + 'promotion:completed', + ]) + }) + + it('can apply acquired source text and write blocks through the existing research loop', async () => { + const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-rag-loop-')) + tempRoots.push(root) + + const result = await runRagKnowledgeImprovementLoop({ + goal: 'Fill refund-policy knowledge', + enabledPhases: ['knowledge-acquisition', 'knowledge-update'], + requiredPhases: ['knowledge-update'], + acquireKnowledge: () => ({ + sourceTexts: [ + { + uri: 'research://refund-policy', + title: 'Refund Policy Source', + text: 'Customers can request refunds within 30 days.', + }, + ], + proposalText: [ + '---FILE: knowledge/support/refund-policy.md---', + '---', + 'id: refund-policy', + 'title: Refund Policy', + '---', + '# Refund Policy', + 'Customers can request refunds within 30 days.', + '---END FILE---', + ].join('\n'), + done: true, + }), + knowledgeResearch: { + root, + sourceOptions: { now: () => new Date('2026-01-01T00:00:00.000Z') }, + }, + }) + + expect(result.knowledgeUpdate?.applied).toBe(true) + expect(result.knowledgeUpdate?.research?.iterations).toBe(1) + expect(result.knowledgeUpdate?.research?.index.pages.map((page) => page.path)).toContain( + 'knowledge/support/refund-policy.md', + ) + expect(result.knowledgeUpdate?.research?.index.sources).toHaveLength(1) + }) + + it('fails loudly when a required phase has no implementation hook', async () => { + await expect( + runRagKnowledgeImprovementLoop({ + goal: 'Do not fake answer eval', + requiredPhases: ['answer-quality'], + }), + ).rejects.toThrow(/answer-quality requires an evaluateAnswers hook/) + }) +})