From 5b562cf201e4317d4a65bd9de5e246c8c8f9938d Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 7 Jul 2026 18:33:23 -0600 Subject: [PATCH] feat(improvement): add resumable knowledge base loop --- README.md | 37 +- package.json | 6 +- pnpm-lock.yaml | 109 +++-- src/index.ts | 1 + src/kb-improvement.ts | 906 +++++++++++++++++++++++++++++++++++ tests/kb-improvement.test.ts | 374 +++++++++++++++ 6 files changed, 1379 insertions(+), 54 deletions(-) create mode 100644 src/kb-improvement.ts create mode 100644 tests/kb-improvement.test.ts diff --git a/README.md b/README.md index c9996c5..56853f6 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,9 @@ Two ways in, depending on what you're doing: - *"Grow the KB as a researcher"* → [`runKnowledgeResearchLoop`](#research-loop) (deterministic mechanics; your agent owns judgment), [`runTwoAgentResearchLoop`](#two-agent-research-loop) (researcher proposes, verifier checks + fills gaps, offline), or the sandbox [researcher profile](#researcher-profile) for `runLoop`. - *"Spawn one researcher per sub-topic and stop when the KB is ready"* → [`runResearchSupervisor`](#research-supervisor) (a supervisor brain sizes the topology over a `Scope`; LIVE, needs creds). - *"Tune retrieval for a knowledge base"* → `runRetrievalImprovementLoop` in the [Agent-Eval integration](#agent-eval-integration) section. - - *"Improve the whole RAG knowledge base"* → `runRagKnowledgeImprovementLoop` in the [Agent-Eval integration](#agent-eval-integration) section. + - *"Run an operator-grade KB improvement cycle"* → `improveKnowledgeBase` in the [Agent-Eval integration](#agent-eval-integration) section. + It creates a candidate KB, lets agents or deterministic hooks improve it, runs configured evals, can be resumed, and promotes only when the live KB has not changed underneath it. + - *"Expose the lower-level RAG lifecycle phases"* → `runRagKnowledgeImprovementLoop` in the [Agent-Eval integration](#agent-eval-integration) section. It exposes retrieval tuning, gap diagnosis, knowledge acquisition/update, answer-quality checks, and promotion as one typed lifecycle. - *"Evaluate RAG answers or a wiki/KB"* → `ragAnswerQualityJudge`, `createRagAnswerQualityHook`, and `scoreKnowledgeBaseIndex` in the [Agent-Eval integration](#agent-eval-integration) section. - *"Does this candidate KB actually improve task success?"* → run an [agent-eval improvement loop](#agent-eval-integration) over KB variants, then `knowledgeReleaseReport` for the promotion decision. @@ -100,6 +102,11 @@ from `@tangle-network/agent-knowledge`. answer-quality eval, and promotion are separate typed phases so products can plug in browser agents, coding agents, connectors, or deterministic policies without this package hardcoding an agent runner. +- `improveKnowledgeBase()` wraps that lifecycle with durable candidate state, + a per-run lock, resume support, isolated candidate workspaces, KB quality + scoring, and conflict-safe promotion. + Use it when running agents in loops against a real KB rather than only + exposing phase hooks. - RAG answer evaluation follows the common open-source shape used by Ragas, DeepEval, TruLens, and RAGChecker: context quality, answer relevance, support/faithfulness, citations, abstention, and failure diagnosis. @@ -184,6 +191,34 @@ const result = await runRagKnowledgeImprovementLoop({ console.log(result.promotion) ``` +Use `improveKnowledgeBase` when a program should own the candidate workspace and promotion mechanics. +The wrapper composes the same RAG lifecycle, but adds resumable state under `.agent-knowledge/improvements//`, a lock lease for parallel operators, and a base-hash check before copying the candidate over the live KB. + +```ts +import { improveKnowledgeBase } from '@tangle-network/agent-knowledge' + +const result = await improveKnowledgeBase({ + root: './kb', + goal: 'Improve support refund-policy knowledge', + readinessSpecs, + retrieval: { + baseline: { k: 5 }, + scenarios: trainRetrievalScenarios, + holdoutScenarios, + searchSpace: { k: [5, 10, 20] }, + targetRecall: 0.9, + }, + step: async ({ readiness }) => runResearchAgent({ missing: readiness?.report }), + evaluateAnswers, + requiredPhases: ['knowledge-update', 'retrieval-tuning', 'answer-quality'], +}) + +console.log(result.promoted, result.candidate?.evaluation) +``` + +Pass `promote: false` to leave the candidate workspace open for another agent or a human edit. +Calling `improveKnowledgeBase` again with the same `runId` re-evaluates that candidate and promotes it only if the original live KB hash still matches. + If a required phase is missing its hook, the loop throws. That keeps the public API from reporting a fake “RAG improved” result when the caller only wired retrieval or only wired a researcher. diff --git a/package.json b/package.json index a92e158..4433d5c 100644 --- a/package.json +++ b/package.json @@ -74,14 +74,14 @@ "autodata": "tsx src/autodata/run.ts" }, "dependencies": { - "@tangle-network/agent-eval": "^0.100.0", - "@tangle-network/agent-runtime": "^0.77.0", + "@tangle-network/agent-eval": "^0.107.0", + "@tangle-network/agent-runtime": "^0.89.0", "zod": "^4.3.6" }, "devDependencies": { "@biomejs/biome": "^2.4.15", "@neo4j-labs/agent-memory": "0.4.0", - "@tangle-network/sandbox": "^0.8.0", + "@tangle-network/sandbox": "^0.9.7", "@types/node": "^25.6.0", "tsup": "^8.0.0", "tsx": "^4.22.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cc04568..6aaf726 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,11 +9,11 @@ importers: .: dependencies: '@tangle-network/agent-eval': - specifier: ^0.100.0 - version: 0.100.0(typescript@5.9.3) + specifier: ^0.107.0 + version: 0.107.0(typescript@5.9.3) '@tangle-network/agent-runtime': - specifier: ^0.77.0 - version: 0.77.0(@tangle-network/agent-eval@0.100.0(typescript@5.9.3))(@tangle-network/agent-interface@0.10.1)(@tangle-network/sandbox@0.8.2(viem@2.48.8(typescript@5.9.3)(zod@4.4.2))) + specifier: ^0.89.0 + version: 0.89.0(@tangle-network/agent-eval@0.107.0(typescript@5.9.3))(@tangle-network/agent-interface@0.17.1)(@tangle-network/sandbox@0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3))) zod: specifier: ^4.3.6 version: 4.4.2 @@ -25,8 +25,8 @@ importers: specifier: 0.4.0 version: 0.4.0 '@tangle-network/sandbox': - specifier: ^0.8.0 - version: 0.8.2(viem@2.48.8(typescript@5.9.3)(zod@4.4.2)) + specifier: ^0.9.7 + version: 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3)) '@types/node': specifier: ^25.6.0 version: 25.6.0 @@ -617,47 +617,42 @@ packages: '@scure/bip39@2.2.0': resolution: {integrity: sha512-T/Bj/YvYMNkIPq6EENO6/rcs2e7qTNuyoUXf0KBFDmp0ZDu0H2X4Lq6yC3i0c8PcWkov5EbW+yQZZbdMmk154A==} - '@tangle-network/agent-eval@0.100.0': - resolution: {integrity: sha512-yBupVJJAqHozhe1BL5xBuDObjvNsoY+XmJo7qfpw/w7rehAXbKliBb4k3XS1G55+GaYPjFA+xwPzlEDQISpMRw==} + '@tangle-network/agent-core@0.3.8': + resolution: {integrity: sha512-bZfVpdiFjXbcQwSxSQABdtXEmUuapWChCcXrHkP6fAnwqEy9hWEfeZLlMZUy+1XaOfpTaMLUOEAwYhXfN018wQ==} + + '@tangle-network/agent-eval@0.107.0': + resolution: {integrity: sha512-wkeDD47MWz5703CU0ACiI6OWS1im8gFDScgNhNiEXvn9iqModeqM1rcoL/8ww2frRdCWMF/trLQNcnoGUdm7eA==} engines: {node: '>=20'} hasBin: true '@tangle-network/agent-interface@0.10.1': resolution: {integrity: sha512-yehY/0EgKvu8lG6jIVoZCtMPLkj8VEWwasuAtuph2RaB9MKE5wuxRF647O6jw8KufNZ3aQ2UVVWpZ19dGCbs6w==} - '@tangle-network/agent-interface@0.8.0': - resolution: {integrity: sha512-okz9LGKwPNKODNyT9Y7+T+sQsJ4g6oTy/hpWpxR6r2BI7pS6WqIdgCOQcx98+WtlPoibkY3ewRRAb8YJMrPHog==} + '@tangle-network/agent-interface@0.13.0': + resolution: {integrity: sha512-CeTPGRLoXqpt0h+BCyFgZPkfU1zyRpWmqfD+85i/uk+uvbqxkfI+JprfKVf3tBsQuCgJPSjPt5qjdW8n3h2BVg==} + + '@tangle-network/agent-interface@0.17.1': + resolution: {integrity: sha512-B7dRJTo0HSUtgBCB1VMwkTFYkLUaRr/4BcRglrQuGhGUwOzKv1RYyMejOVh5M3a5AagY9N79f7GYbjcA3UmnIA==} - '@tangle-network/agent-runtime@0.77.0': - resolution: {integrity: sha512-cykjXYVO9WZcibCnFHnmOPsTMGgFr2Zr8lHla9PhkNY/B5mLwHkqBvGTP9msqJTzcKEXdvsvjM4a3nN1Z4/oXA==} + '@tangle-network/agent-runtime@0.89.0': + resolution: {integrity: sha512-tjQ/uJORuLOAE/E99w0NxWb5O0uP/nwescMRyIqxsqLXzry7WhNh9DK/BHKTEfb6XMvcIaT92J/3T1ZJTtXZ9Q==} engines: {node: '>=20'} hasBin: true peerDependencies: - '@tangle-network/agent-eval': '>=0.97.0 <1.0.0' - '@tangle-network/agent-interface': '>=0.10.0 <1.0.0' - '@tangle-network/agent-knowledge': '>=1.7.0 <2.0.0' + '@tangle-network/agent-eval': '>=0.101.0 <1.0.0' + '@tangle-network/agent-interface': '>=0.14.0 <1.0.0' '@tangle-network/sandbox': '>=0.8.0 <1.0.0' playwright: ^1.40.0 peerDependenciesMeta: '@tangle-network/agent-interface': optional: true - '@tangle-network/agent-knowledge': - optional: true '@tangle-network/sandbox': optional: true playwright: optional: true - '@tangle-network/sandbox@0.1.2': - resolution: {integrity: sha512-6TPH9QgCgou9Bhc1kzLNL4/PRiT1mjId6NONY5Le/KT2kh77cXH8KN3TTY/cU+/eW+WM5FYJOy32FWl2HShXbw==} - peerDependencies: - viem: ^2.0.0 - peerDependenciesMeta: - viem: - optional: true - - '@tangle-network/sandbox@0.8.2': - resolution: {integrity: sha512-MG3dj7SnF7vI8CagW1OwpkJSUq3IREpADBWp6knOukKxSYYCMGwJ0nPZz+O2eotI+Nl2A2LIGiHqPB82jgOvjw==} + '@tangle-network/sandbox@0.9.7': + resolution: {integrity: sha512-9pCwJ5MlF7RUpp0AQKQDFyR0yu+E0udEhWkqhrlb/RuoJxlt72zVPuzO4FnMb1MZTkfjStmomC3k5xQyqi1YSA==} peerDependencies: '@mastra/core': ^1.36.0 '@modelcontextprotocol/sdk': ^1.29.0 @@ -680,8 +675,8 @@ packages: resolution: {integrity: sha512-+TAF9s5t1jOWGyGHvKhIWe2FYmG7puVaxmmg0Et67ylAjGa7GqUAvISXGjG/6dzld7A170V0kQHK0WVdh2Wh0Q==} engines: {node: '>=18'} - '@tangle-network/tcloud@0.4.6': - resolution: {integrity: sha512-ABjo9CBvFmKlEMWGazY7nPOlCI0lXa9nuM+qmVEa3RXLy+0q0RT33mA6ekqjA4Ieb8irym93uAan5P/iwdmxXg==} + '@tangle-network/tcloud@0.4.14': + resolution: {integrity: sha512-jWYt//cGdLBDOv0luLH6xAGS4gbuOt8uHIkaCWwDDpQ1zp0FUPATHIrA3RMuF0qtQq9Vq00IhLrmCnHdHBP+dg==} engines: {node: '>=18'} hasBin: true @@ -1169,17 +1164,17 @@ snapshots: '@adraffy/ens-normalize@1.11.1': {} - '@asteasolutions/zod-to-openapi@8.5.0(zod@4.4.2)': + '@asteasolutions/zod-to-openapi@8.5.0(zod@4.4.3)': dependencies: openapi3-ts: 4.5.0 - zod: 4.4.2 + zod: 4.4.3 - '@ax-llm/ax@19.0.45(zod@4.4.2)': + '@ax-llm/ax@19.0.45(zod@4.4.3)': dependencies: '@opentelemetry/api': 1.9.1 dayjs: 1.11.20 optionalDependencies: - zod: 4.4.2 + zod: 4.4.3 '@biomejs/biome@2.4.15': optionalDependencies: @@ -1509,17 +1504,26 @@ snapshots: '@noble/hashes': 2.2.0 '@scure/base': 2.2.0 - '@tangle-network/agent-eval@0.100.0(typescript@5.9.3)': + '@tangle-network/agent-core@0.3.8': + dependencies: + '@tangle-network/agent-interface': 0.17.1 + zod: 4.4.3 + + '@tangle-network/agent-eval@0.107.0(typescript@5.9.3)': dependencies: - '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.2) - '@ax-llm/ax': 19.0.45(zod@4.4.2) + '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) + '@ax-llm/ax': 19.0.45(zod@4.4.3) '@hono/node-server': 2.0.1(hono@4.12.16) '@tangle-network/agent-interface': 0.10.1 - '@tangle-network/tcloud': 0.4.6(typescript@5.9.3)(zod@4.4.2) + '@tangle-network/tcloud': 0.4.14(typescript@5.9.3)(zod@4.4.3) hono: 4.12.16 - zod: 4.4.2 + zod: 4.4.3 transitivePeerDependencies: + - '@mastra/core' + - '@modelcontextprotocol/sdk' + - ai - bufferutil + - openai - typescript - utf-8-validate @@ -1527,39 +1531,44 @@ snapshots: dependencies: zod: 4.4.3 - '@tangle-network/agent-interface@0.8.0': + '@tangle-network/agent-interface@0.13.0': dependencies: zod: 4.4.3 - '@tangle-network/agent-runtime@0.77.0(@tangle-network/agent-eval@0.100.0(typescript@5.9.3))(@tangle-network/agent-interface@0.10.1)(@tangle-network/sandbox@0.8.2(viem@2.48.8(typescript@5.9.3)(zod@4.4.2)))': + '@tangle-network/agent-interface@0.17.1': dependencies: - '@tangle-network/agent-eval': 0.100.0(typescript@5.9.3) - optionalDependencies: - '@tangle-network/agent-interface': 0.10.1 - '@tangle-network/sandbox': 0.8.2(viem@2.48.8(typescript@5.9.3)(zod@4.4.2)) + zod: 4.4.3 - '@tangle-network/sandbox@0.1.2(viem@2.48.8(typescript@5.9.3)(zod@4.4.2))': + '@tangle-network/agent-runtime@0.89.0(@tangle-network/agent-eval@0.107.0(typescript@5.9.3))(@tangle-network/agent-interface@0.17.1)(@tangle-network/sandbox@0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3)))': + dependencies: + '@tangle-network/agent-eval': 0.107.0(typescript@5.9.3) optionalDependencies: - viem: 2.48.8(typescript@5.9.3)(zod@4.4.2) + '@tangle-network/agent-interface': 0.17.1 + '@tangle-network/sandbox': 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3)) - '@tangle-network/sandbox@0.8.2(viem@2.48.8(typescript@5.9.3)(zod@4.4.2))': + '@tangle-network/sandbox@0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3))': dependencies: - '@tangle-network/agent-interface': 0.8.0 + '@tangle-network/agent-core': 0.3.8 + '@tangle-network/agent-interface': 0.13.0 optionalDependencies: viem: 2.48.8(typescript@5.9.3)(zod@4.4.2) '@tangle-network/tcloud-attestation@0.1.1': {} - '@tangle-network/tcloud@0.4.6(typescript@5.9.3)(zod@4.4.2)': + '@tangle-network/tcloud@0.4.14(typescript@5.9.3)(zod@4.4.3)': dependencies: '@scure/bip32': 2.2.0 '@scure/bip39': 2.2.0 - '@tangle-network/sandbox': 0.1.2(viem@2.48.8(typescript@5.9.3)(zod@4.4.2)) + '@tangle-network/sandbox': 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3)) '@tangle-network/tcloud-attestation': 0.1.1 commander: 14.0.3 viem: 2.48.8(typescript@5.9.3)(zod@4.4.2) transitivePeerDependencies: + - '@mastra/core' + - '@modelcontextprotocol/sdk' + - ai - bufferutil + - openai - typescript - utf-8-validate - zod diff --git a/src/index.ts b/src/index.ts index d6bace8..e033813 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,6 +15,7 @@ export * from './indexer' export * from './inspect' export * from './investment-thesis-set' export * from './investment-thesis-task' +export * from './kb-improvement' export * from './kb-store' export * from './lint' export * from './material-facts-metric' diff --git a/src/kb-improvement.ts b/src/kb-improvement.ts new file mode 100644 index 0000000..5bc6336 --- /dev/null +++ b/src/kb-improvement.ts @@ -0,0 +1,906 @@ +import { cp, mkdir, open, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises' +import { dirname, join, relative } from 'node:path' +import type { + BuildEvalKnowledgeBundleOptions, + EvalKnowledgeBundleBuildResult, + KnowledgeReadinessSpec, +} from './eval-readiness' +import { sha256, stableId } from './ids' +import { buildKnowledgeIndex, writeKnowledgeIndex } from './indexer' +import { + type KnowledgeBaseQualityOptions, + type KnowledgeBaseQualityReport, + scoreKnowledgeBaseIndex, +} from './rag-eval' +import { + type RagAnswerQualityResult, + type RagKnowledgeImprovementPhase, + type RagKnowledgeImprovementPhaseResult, + type RagKnowledgeResearchOptions, + type RagPromotionResult, + type RunRagKnowledgeImprovementLoopOptions, + type RunRagKnowledgeImprovementLoopResult, + runRagKnowledgeImprovementLoop, +} from './rag-improvement-loop' +import { readinessFor } from './readiness-helpers' +import type { RunKnowledgeResearchLoopOptions } from './research-loop' +import { type ResearchSupervisorOptions, runResearchSupervisor } from './research-supervisor' +import type { RetrievalConfig, RunRetrievalImprovementLoopOptions } from './retrieval-eval' +import { initKnowledgeBase, layoutFor } from './store' +import type { KnowledgeIndex } from './types' +import { + type ValidateKnowledgeOptions, + type ValidateKnowledgeResult, + validateKnowledgeIndex, +} from './validate' + +export type KnowledgeImprovementStatus = + | 'running' + | 'candidate-ready' + | 'promoted' + | 'rejected' + | 'blocked' + +export interface KnowledgeImprovementMetric { + score: number + passed: boolean + dimensions?: Record + notes?: string +} + +export interface KnowledgeImprovementEvaluationInput { + runId: string + iteration: number + root: string + baselineRoot: string + candidateRoot: string + baselineIndex: KnowledgeIndex + candidateIndex: KnowledgeIndex + baseHash: string + candidateHash: string + validation: ValidateKnowledgeResult + readiness?: EvalKnowledgeBundleBuildResult + kbQuality: KnowledgeBaseQualityReport + lifecycle?: RunRagKnowledgeImprovementLoopResult + signal?: AbortSignal +} + +export type KnowledgeImprovementEvaluator = ( + input: KnowledgeImprovementEvaluationInput, +) => Promise | KnowledgeImprovementMetric + +export interface KnowledgeImprovementLifecycleRecord { + stage: 'candidate-update' | 'candidate-evaluation' + phases: readonly RagKnowledgeImprovementPhaseResult[] + findingCount: number + retrievalWinnerConfig?: RetrievalConfig + answerQuality?: RagAnswerQualityResult + promotionDecision?: RagPromotionResult +} + +export interface KnowledgeImprovementCandidateRecord { + iteration: number + candidateId: string + candidateRoot: string + baseHash: string + candidateHash?: string + status: KnowledgeImprovementStatus + createdAt: string + updatedAt: string + validation?: ValidateKnowledgeResult + kbQuality?: KnowledgeBaseQualityReport + readinessBlockingMissing?: number + evaluation?: KnowledgeImprovementMetric + lifecycle?: readonly KnowledgeImprovementLifecycleRecord[] + retrievalWinnerConfig?: RetrievalConfig + answerQuality?: RagAnswerQualityResult + promotionDecision?: RagPromotionResult + notes?: string +} + +export interface KnowledgeImprovementRunState { + version: 1 + runId: string + root: string + goal: string + status: KnowledgeImprovementStatus + baseHash: string + createdAt: string + updatedAt: string + ownerId?: string + candidates: KnowledgeImprovementCandidateRecord[] + promotedCandidateId?: string + blockedReason?: string +} + +export interface KnowledgeImprovementResult { + runId: string + runDir: string + state: KnowledgeImprovementRunState + candidate?: KnowledgeImprovementCandidateRecord + lifecycle?: RunRagKnowledgeImprovementLoopResult + promoted: boolean + blocked: boolean +} + +export interface KnowledgeImprovementRetrievalOptions + extends Omit { + runDir?: RunRetrievalImprovementLoopOptions['runDir'] +} + +export interface KnowledgeImprovementOptions { + root: string + goal: string + runId?: string + runDir?: string + ownerId?: string + leaseTtlMs?: number + resume?: boolean + promote?: boolean + maxCandidates?: number + candidateResearchIterations?: number + strict?: ValidateKnowledgeOptions['strict'] + readinessSpecs?: KnowledgeReadinessSpec[] + readinessTaskId?: string + readiness?: Omit + kbQuality?: KnowledgeBaseQualityOptions + step?: RunKnowledgeResearchLoopOptions['step'] + knowledgeResearch?: Omit + supervisor?: Omit< + ResearchSupervisorOptions, + 'root' | 'goal' | 'readinessSpecs' | 'readinessTaskId' | 'readiness' + > + retrieval?: KnowledgeImprovementRetrievalOptions + diagnose?: NonNullable + acquireKnowledge?: NonNullable + updateKnowledge?: NonNullable + evaluateAnswers?: NonNullable + decidePromotion?: NonNullable + enabledPhases?: readonly RagKnowledgeImprovementPhase[] + requiredPhases?: readonly RagKnowledgeImprovementPhase[] + evaluate?: KnowledgeImprovementEvaluator + signal?: AbortSignal + now?: () => Date + onState?: (state: KnowledgeImprovementRunState) => Promise | void +} + +interface LeaseHandle { + ownerId: string + path: string + release(): Promise +} + +interface LeaseFile { + ownerId: string + acquiredAt: string + expiresAt: string + pid: number +} + +const DEFAULT_LEASE_TTL_MS = 15 * 60 * 1000 +const UPDATE_PHASES: readonly RagKnowledgeImprovementPhase[] = [ + 'knowledge-acquisition', + 'knowledge-update', +] +const EVALUATION_PHASES: readonly RagKnowledgeImprovementPhase[] = [ + 'retrieval-tuning', + 'gap-diagnosis', + 'answer-quality', + 'promotion', +] + +export function knowledgeImprovementRunId(root: string, goal: string): string { + return stableId('kimpr', `${root}:${goal}`) +} + +export function knowledgeImprovementRunDir(root: string, runId: string): string { + return join(layoutFor(root).cacheDir, 'improvements', runId) +} + +export async function loadKnowledgeImprovementState( + root: string, + runId: string, + runDir = knowledgeImprovementRunDir(root, runId), +): Promise { + try { + return JSON.parse(await readFile(statePath(runDir), 'utf8')) as KnowledgeImprovementRunState + } catch { + return null + } +} + +export async function improveKnowledgeBase( + options: KnowledgeImprovementOptions, +): Promise { + assertKnowledgeImprovementOptions(options) + + const now = options.now ?? (() => new Date()) + const runId = options.runId ?? knowledgeImprovementRunId(options.root, options.goal) + const runDir = options.runDir ?? knowledgeImprovementRunDir(options.root, runId) + await initKnowledgeBase(options.root) + await mkdir(runDir, { recursive: true }) + + const lease = await acquireRunLease(runDir, { + ownerId: options.ownerId ?? `pid-${process.pid}`, + ttlMs: options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS, + now, + }) + + try { + let state = + options.resume === false + ? null + : await loadKnowledgeImprovementState(options.root, runId, runDir) + if (!state) { + const baseHash = await hashKnowledgeBase(options.root) + state = { + version: 1, + runId, + root: options.root, + goal: options.goal, + status: 'running', + baseHash, + createdAt: now().toISOString(), + updatedAt: now().toISOString(), + ownerId: lease.ownerId, + candidates: [], + } + await saveState(runDir, state, options.onState) + await appendLedger(runDir, { type: 'run.created', runId, baseHash }) + } + + if (state.status === 'promoted') { + return { runId, runDir, state, promoted: true, blocked: false } + } + if (state.status === 'blocked') { + return { runId, runDir, state, promoted: false, blocked: true } + } + + const maxCandidates = Math.max(1, options.maxCandidates ?? 1) + let candidate = findActiveCandidate(state) + let lifecycle: RunRagKnowledgeImprovementLoopResult | undefined + + while ( + candidate?.status === 'running' || + (!candidate && state.candidates.length < maxCandidates) + ) { + if (!candidate) { + const currentHash = await hashKnowledgeBase(options.root) + if (currentHash !== state.baseHash) { + state = await blockRun( + runDir, + state, + `base changed before candidate creation: expected ${state.baseHash}, got ${currentHash}`, + options.onState, + now, + ) + return { runId, runDir, state, promoted: false, blocked: true } + } + candidate = await createCandidateWorkspace(runDir, state, options.root, now) + state.candidates.push(candidate) + state.status = 'running' + state.updatedAt = now().toISOString() + await saveState(runDir, state, options.onState) + await appendLedger(runDir, { + type: 'candidate.created', + runId, + candidateId: candidate.candidateId, + iteration: candidate.iteration, + }) + } + + lifecycle = await runCandidateLifecycle(runDir, candidate, options, now) + candidate.status = 'candidate-ready' + candidate.updatedAt = now().toISOString() + state.status = 'candidate-ready' + state.updatedAt = now().toISOString() + await saveState(runDir, state, options.onState) + await appendLedger(runDir, { + type: 'candidate.ready', + runId, + candidateId: candidate.candidateId, + }) + } + + if (!candidate) { + state.status = 'rejected' + state.updatedAt = now().toISOString() + await saveState(runDir, state, options.onState) + return { runId, runDir, state, promoted: false, blocked: false } + } + + candidate = await evaluateCandidate(runDir, state, candidate, lifecycle, options, now) + state.updatedAt = now().toISOString() + await saveState(runDir, state, options.onState) + + if (!candidate.evaluation?.passed) { + candidate.status = 'rejected' + state.status = 'rejected' + state.updatedAt = now().toISOString() + await saveState(runDir, state, options.onState) + await appendLedger(runDir, { + type: 'candidate.rejected', + runId, + candidateId: candidate.candidateId, + }) + return { runId, runDir, state, candidate, lifecycle, promoted: false, blocked: false } + } + + if (options.promote === false) { + state.status = 'candidate-ready' + state.updatedAt = now().toISOString() + await saveState(runDir, state, options.onState) + return { runId, runDir, state, candidate, lifecycle, promoted: false, blocked: false } + } + + const currentHash = await hashKnowledgeBase(options.root) + if (currentHash !== state.baseHash) { + state = await blockRun( + runDir, + state, + `base changed before promotion: expected ${state.baseHash}, got ${currentHash}`, + options.onState, + now, + ) + await appendLedger(runDir, { + type: 'promotion.blocked', + runId, + candidateId: candidate.candidateId, + reason: state.blockedReason, + }) + return { runId, runDir, state, candidate, lifecycle, promoted: false, blocked: true } + } + + await promoteCandidate(options.root, candidate.candidateRoot) + candidate.status = 'promoted' + candidate.updatedAt = now().toISOString() + state.status = 'promoted' + state.promotedCandidateId = candidate.candidateId + state.updatedAt = now().toISOString() + await saveState(runDir, state, options.onState) + await appendLedger(runDir, { + type: 'candidate.promoted', + runId, + candidateId: candidate.candidateId, + }) + return { runId, runDir, state, candidate, lifecycle, promoted: true, blocked: false } + } finally { + await lease.release() + } +} + +function assertKnowledgeImprovementOptions(options: KnowledgeImprovementOptions): void { + if (options.step && options.knowledgeResearch?.step) { + throw new Error('improveKnowledgeBase accepts either step or knowledgeResearch.step, not both') + } + const updateDrivers = [ + Boolean(options.step ?? options.knowledgeResearch), + Boolean(options.supervisor), + Boolean(options.updateKnowledge), + ].filter(Boolean).length + if (updateDrivers > 1) { + throw new Error( + 'improveKnowledgeBase accepts only one knowledge-update driver: knowledgeResearch, supervisor, or updateKnowledge', + ) + } + if (options.supervisor && (!options.readinessSpecs || options.readinessSpecs.length === 0)) { + throw new Error('improveKnowledgeBase supervisor mode requires readinessSpecs') + } +} + +async function runCandidateLifecycle( + runDir: string, + candidate: KnowledgeImprovementCandidateRecord, + options: KnowledgeImprovementOptions, + now: () => Date, +): Promise { + const lifecycles: RunRagKnowledgeImprovementLoopResult[] = [] + + if (shouldRunUpdateStage(options)) { + const updateLifecycle = await runRagKnowledgeImprovementLoop({ + goal: options.goal, + acquireKnowledge: options.acquireKnowledge, + knowledgeResearch: candidateKnowledgeResearchOptions(candidate.candidateRoot, options), + updateKnowledge: candidateUpdateHook(candidate.candidateRoot, options), + enabledPhases: selectedStagePhases(options, UPDATE_PHASES), + requiredPhases: selectedStageRequiredPhases(options, UPDATE_PHASES), + signal: options.signal, + now, + }) + lifecycles.push(updateLifecycle) + recordLifecycle(candidate, 'candidate-update', updateLifecycle) + } + + if (shouldRunEvaluationStage(options)) { + const candidateIndex = await buildKnowledgeIndex(candidate.candidateRoot) + const evaluationLifecycle = await runRagKnowledgeImprovementLoop({ + goal: options.goal, + retrieval: options.retrieval + ? { + ...options.retrieval, + index: candidateIndex, + runDir: options.retrieval.runDir ?? join(runDir, 'retrieval', candidate.candidateId), + } + : undefined, + diagnose: options.diagnose, + evaluateAnswers: options.evaluateAnswers, + promote: options.decidePromotion, + enabledPhases: selectedStagePhases(options, EVALUATION_PHASES), + requiredPhases: selectedStageRequiredPhases(options, EVALUATION_PHASES), + signal: options.signal, + now, + }) + lifecycles.push(evaluationLifecycle) + recordLifecycle(candidate, 'candidate-evaluation', evaluationLifecycle) + } + + return mergeLifecycleResults(options.goal, lifecycles) +} + +function candidateKnowledgeResearchOptions( + candidateRoot: string, + options: KnowledgeImprovementOptions, +): RagKnowledgeResearchOptions | undefined { + if (!options.step && !options.knowledgeResearch) return undefined + const { step: researchStep, ...rest } = options.knowledgeResearch ?? {} + const step = options.step ?? researchStep + return { + ...rest, + root: candidateRoot, + step, + maxIterations: + rest.maxIterations ?? (step ? (options.candidateResearchIterations ?? 3) : undefined), + strict: rest.strict ?? options.strict, + readinessSpecs: rest.readinessSpecs ?? options.readinessSpecs, + readinessTaskId: rest.readinessTaskId ?? options.readinessTaskId, + readiness: rest.readiness ?? options.readiness, + } +} + +function candidateUpdateHook( + candidateRoot: string, + options: KnowledgeImprovementOptions, +): RunRagKnowledgeImprovementLoopOptions['updateKnowledge'] { + if (options.updateKnowledge) return options.updateKnowledge + if (!options.supervisor) return undefined + return async () => { + await runResearchSupervisor({ + ...options.supervisor!, + root: candidateRoot, + goal: options.goal, + readinessSpecs: options.readinessSpecs ?? [], + readinessTaskId: options.readinessTaskId, + readiness: options.readiness, + }) + return { + applied: true, + summary: 'research supervisor completed', + metadata: { supervised: true }, + } + } +} + +function shouldRunUpdateStage(options: KnowledgeImprovementOptions): boolean { + const phases = selectedStagePhases(options, UPDATE_PHASES) + if (phases.length === 0) return false + return Boolean( + options.acquireKnowledge || + options.step || + options.knowledgeResearch || + options.supervisor || + options.updateKnowledge || + selectedStageRequiredPhases(options, UPDATE_PHASES).length > 0, + ) +} + +function shouldRunEvaluationStage(options: KnowledgeImprovementOptions): boolean { + const phases = selectedStagePhases(options, EVALUATION_PHASES) + if (phases.length === 0) return false + return Boolean( + options.retrieval || + options.diagnose || + options.evaluateAnswers || + options.decidePromotion || + selectedStageRequiredPhases(options, EVALUATION_PHASES).length > 0, + ) +} + +function selectedStagePhases( + options: Pick, + stagePhases: readonly RagKnowledgeImprovementPhase[], +): RagKnowledgeImprovementPhase[] { + const requested = options.enabledPhases ?? stagePhases + return requested.filter((phase) => stagePhases.includes(phase)) +} + +function selectedStageRequiredPhases( + options: Pick, + stagePhases: readonly RagKnowledgeImprovementPhase[], +): RagKnowledgeImprovementPhase[] { + return (options.requiredPhases ?? []).filter((phase) => stagePhases.includes(phase)) +} + +function recordLifecycle( + candidate: KnowledgeImprovementCandidateRecord, + stage: KnowledgeImprovementLifecycleRecord['stage'], + lifecycle: RunRagKnowledgeImprovementLoopResult, +): void { + const record: KnowledgeImprovementLifecycleRecord = { + stage, + phases: lifecycle.phases, + findingCount: lifecycle.findings.length, + retrievalWinnerConfig: lifecycle.retrieval?.winnerConfig, + answerQuality: lifecycle.answerQuality, + promotionDecision: lifecycle.promotion, + } + candidate.lifecycle = [...(candidate.lifecycle ?? []), record] + candidate.retrievalWinnerConfig = + lifecycle.retrieval?.winnerConfig ?? candidate.retrievalWinnerConfig + candidate.answerQuality = lifecycle.answerQuality ?? candidate.answerQuality + candidate.promotionDecision = lifecycle.promotion ?? candidate.promotionDecision +} + +function mergeLifecycleResults( + goal: string, + lifecycles: readonly RunRagKnowledgeImprovementLoopResult[], +): RunRagKnowledgeImprovementLoopResult | undefined { + if (lifecycles.length === 0) return undefined + return { + goal, + phases: lifecycles.flatMap((lifecycle) => lifecycle.phases), + retrieval: lastDefined(lifecycles.map((lifecycle) => lifecycle.retrieval)), + findings: lifecycles.flatMap((lifecycle) => lifecycle.findings), + acquisition: lastDefined(lifecycles.map((lifecycle) => lifecycle.acquisition)), + knowledgeUpdate: lastDefined(lifecycles.map((lifecycle) => lifecycle.knowledgeUpdate)), + answerQuality: lastDefined(lifecycles.map((lifecycle) => lifecycle.answerQuality)), + promotion: lastDefined(lifecycles.map((lifecycle) => lifecycle.promotion)), + } +} + +function lastDefined(values: readonly (T | undefined)[]): T | undefined { + for (let index = values.length - 1; index >= 0; index -= 1) { + if (values[index] !== undefined) return values[index] + } + return undefined +} + +async function evaluateCandidate( + runDir: string, + state: KnowledgeImprovementRunState, + candidate: KnowledgeImprovementCandidateRecord, + lifecycle: RunRagKnowledgeImprovementLoopResult | undefined, + options: KnowledgeImprovementOptions, + now: () => Date, +): Promise { + const [baselineIndex, candidateIndex] = await Promise.all([ + buildKnowledgeIndex(options.root), + buildKnowledgeIndex(candidate.candidateRoot), + ]) + const validation = validateKnowledgeIndex(candidateIndex, { strict: options.strict }) + const readiness = readinessFor(options, candidateIndex) + const kbQuality = scoreKnowledgeBaseIndex(candidateIndex, { + strict: options.strict, + ...options.kbQuality, + }) + const candidateHash = await hashKnowledgeBase(candidate.candidateRoot) + const metric = + options.evaluate?.({ + runId: state.runId, + iteration: candidate.iteration, + root: options.root, + baselineRoot: options.root, + candidateRoot: candidate.candidateRoot, + baselineIndex, + candidateIndex, + baseHash: state.baseHash, + candidateHash, + validation, + readiness, + kbQuality, + lifecycle, + signal: options.signal, + }) ?? + defaultKnowledgeImprovementMetric( + validation, + readiness, + options.readinessSpecs, + kbQuality, + lifecycle, + ) + candidate.validation = validation + candidate.kbQuality = kbQuality + candidate.readinessBlockingMissing = readiness?.report.blockingMissingRequirements.length + candidate.evaluation = applyLifecycleFailures(normalizeMetric(await metric), lifecycle) + candidate.candidateHash = candidateHash + candidate.updatedAt = now().toISOString() + await appendLedger(runDir, { + type: 'candidate.evaluated', + runId: state.runId, + candidateId: candidate.candidateId, + score: candidate.evaluation.score, + passed: candidate.evaluation.passed, + }) + return candidate +} + +function defaultKnowledgeImprovementMetric( + validation: ValidateKnowledgeResult, + readiness: EvalKnowledgeBundleBuildResult | undefined, + readinessSpecs: readonly KnowledgeReadinessSpec[] | undefined, + kbQuality: KnowledgeBaseQualityReport, + lifecycle: RunRagKnowledgeImprovementLoopResult | undefined, +): KnowledgeImprovementMetric { + const blockingMissing = readiness?.report.blockingMissingRequirements.length ?? 0 + const blockingTotal = readinessSpecs?.filter((spec) => spec.importance === 'blocking').length ?? 0 + const blockingReadiness = + blockingTotal === 0 ? 1 : Math.max(0, blockingTotal - blockingMissing) / blockingTotal + const answerQuality = lifecycle?.answerQuality + ? average(Object.values(lifecycle.answerQuality.metrics).filter(Number.isFinite)) + : 1 + const promotionDecision = lifecycle?.promotion ? (lifecycle.promotion.promoted ? 1 : 0) : 1 + const dimensions = { + validation: validation.ok ? 1 : 0, + kb_quality: kbQuality.ok ? 1 : 0, + blocking_readiness: blockingReadiness, + answer_quality: answerQuality, + promotion_decision: promotionDecision, + } + const failedReasons = [ + validation.ok ? undefined : 'candidate validation failed', + kbQuality.ok ? undefined : 'candidate KB quality check failed', + blockingMissing === 0 + ? undefined + : `${blockingMissing}/${blockingTotal} blocking knowledge requirements still missing`, + lifecycle?.answerQuality && !lifecycle.answerQuality.passed + ? 'answer quality failed' + : undefined, + lifecycle?.promotion && !lifecycle.promotion.promoted + ? `promotion decision held: ${lifecycle.promotion.reason}` + : undefined, + ].filter((reason): reason is string => Boolean(reason)) + return { + score: average(Object.values(dimensions)), + passed: failedReasons.length === 0, + dimensions, + notes: + failedReasons.length === 0 ? 'candidate passed configured checks' : failedReasons.join('; '), + } +} + +function applyLifecycleFailures( + metric: KnowledgeImprovementMetric, + lifecycle: RunRagKnowledgeImprovementLoopResult | undefined, +): KnowledgeImprovementMetric { + const reasons = [ + metric.notes, + lifecycle?.answerQuality && !lifecycle.answerQuality.passed + ? 'answer quality failed' + : undefined, + lifecycle?.promotion && !lifecycle.promotion.promoted + ? `promotion decision held: ${lifecycle.promotion.reason}` + : undefined, + ].filter((reason): reason is string => Boolean(reason)) + const forcedFailure = + Boolean(lifecycle?.answerQuality && !lifecycle.answerQuality.passed) || + Boolean(lifecycle?.promotion && !lifecycle.promotion.promoted) + return { + ...metric, + passed: metric.passed && !forcedFailure, + notes: reasons.length > 0 ? reasons.join('; ') : metric.notes, + } +} + +function normalizeMetric(metric: KnowledgeImprovementMetric): KnowledgeImprovementMetric { + if (!Number.isFinite(metric.score) || metric.score < 0 || metric.score > 1) { + throw new Error(`knowledge improvement score must be in [0, 1], got ${String(metric.score)}`) + } + return { ...metric, passed: Boolean(metric.passed) } +} + +async function createCandidateWorkspace( + runDir: string, + state: KnowledgeImprovementRunState, + root: string, + now: () => Date, +): Promise { + const iteration = state.candidates.length + 1 + const candidateId = stableId('kcand', `${state.runId}:${iteration}:${now().toISOString()}`) + const candidateRoot = join(runDir, 'candidates', candidateId, 'workspace') + await copyKnowledgeWorkspace(root, candidateRoot) + const createdAt = now().toISOString() + return { + iteration, + candidateId, + candidateRoot, + baseHash: state.baseHash, + status: 'running', + createdAt, + updatedAt: createdAt, + } +} + +function findActiveCandidate( + state: KnowledgeImprovementRunState, +): KnowledgeImprovementCandidateRecord | undefined { + return [...state.candidates] + .reverse() + .find((candidate) => candidate.status === 'candidate-ready' || candidate.status === 'running') +} + +async function copyKnowledgeWorkspace(sourceRoot: string, targetRoot: string): Promise { + await rm(targetRoot, { recursive: true, force: true }) + await initKnowledgeBase(targetRoot) + await copyIfExists(join(sourceRoot, 'knowledge'), join(targetRoot, 'knowledge')) + await copyIfExists(join(sourceRoot, 'raw'), join(targetRoot, 'raw')) + await copyIfExists( + join(layoutFor(sourceRoot).cacheDir, 'sources.json'), + join(layoutFor(targetRoot).cacheDir, 'sources.json'), + ) + await writeKnowledgeIndex(targetRoot) +} + +async function promoteCandidate(root: string, candidateRoot: string): Promise { + await copyIfExists(join(candidateRoot, 'knowledge'), join(root, 'knowledge'), { replace: true }) + await copyIfExists(join(candidateRoot, 'raw'), join(root, 'raw'), { replace: true }) + await copyIfExists( + join(layoutFor(candidateRoot).cacheDir, 'sources.json'), + join(layoutFor(root).cacheDir, 'sources.json'), + { replace: true }, + ) + await writeKnowledgeIndex(root) +} + +async function copyIfExists( + source: string, + target: string, + options: { replace?: boolean } = {}, +): Promise { + try { + await stat(source) + } catch { + return + } + if (options.replace) await rm(target, { recursive: true, force: true }) + await mkdir(dirname(target), { recursive: true }) + await cp(source, target, { recursive: true }) +} + +export async function hashKnowledgeBase(root: string): Promise { + const entries: Array<{ path: string; hash: string }> = [] + for (const rel of ['knowledge', 'raw']) { + entries.push(...(await hashTreeEntries(root, rel))) + } + const sourceRegistry = relative(root, layoutFor(root).sourceRegistryPath).replace(/\\/g, '/') + entries.push(...(await hashFileEntry(root, sourceRegistry))) + entries.sort((a, b) => a.path.localeCompare(b.path)) + return sha256(JSON.stringify(entries)) +} + +async function hashTreeEntries( + root: string, + relDir: string, +): Promise> { + const abs = join(root, relDir) + try { + const s = await stat(abs) + if (!s.isDirectory()) return [] + } catch { + return [] + } + const out: Array<{ path: string; hash: string }> = [] + const entries = await readdir(abs, { withFileTypes: true }) + for (const entry of entries) { + const rel = join(relDir, entry.name).replace(/\\/g, '/') + if (entry.isDirectory()) out.push(...(await hashTreeEntries(root, rel))) + else if (entry.isFile()) out.push(...(await hashFileEntry(root, rel))) + } + return out +} + +async function hashFileEntry( + root: string, + rel: string, +): Promise> { + try { + const bytes = await readFile(join(root, rel)) + return [{ path: rel, hash: sha256(bytes.toString('base64')) }] + } catch { + return [] + } +} + +async function acquireRunLease( + runDir: string, + options: { ownerId: string; ttlMs: number; now: () => Date }, +): Promise { + const path = join(runDir, 'run.lock') + const now = options.now() + const expiresAt = new Date(now.getTime() + options.ttlMs) + const payload: LeaseFile = { + ownerId: options.ownerId, + acquiredAt: now.toISOString(), + expiresAt: expiresAt.toISOString(), + pid: process.pid, + } + try { + const handle = await open(path, 'wx') + try { + await handle.writeFile(`${JSON.stringify(payload, null, 2)}\n`, 'utf8') + } finally { + await handle.close() + } + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + if (code !== 'EEXIST') throw error + const existing = await readLease(path) + if (existing && new Date(existing.expiresAt).getTime() > now.getTime()) { + throw new Error( + `knowledge improvement run is locked by ${existing.ownerId} until ${existing.expiresAt}`, + ) + } + await rm(path, { force: true }) + return acquireRunLease(runDir, options) + } + return { + ownerId: options.ownerId, + path, + async release() { + const current = await readLease(path) + if (!current || current.ownerId === options.ownerId) await rm(path, { force: true }) + }, + } +} + +async function readLease(path: string): Promise { + try { + return JSON.parse(await readFile(path, 'utf8')) as LeaseFile + } catch { + return null + } +} + +async function blockRun( + runDir: string, + state: KnowledgeImprovementRunState, + reason: string, + onState: KnowledgeImprovementOptions['onState'], + now: () => Date, +): Promise { + state.status = 'blocked' + state.blockedReason = reason + state.updatedAt = now().toISOString() + await saveState(runDir, state, onState) + return state +} + +async function saveState( + runDir: string, + state: KnowledgeImprovementRunState, + onState?: KnowledgeImprovementOptions['onState'], +): Promise { + await writeJsonAtomic(statePath(runDir), state) + await onState?.(state) +} + +async function appendLedger(runDir: string, value: Record): Promise { + const row = { at: new Date().toISOString(), ...value } + await mkdir(runDir, { recursive: true }) + await writeFile(join(runDir, 'events.jsonl'), `${JSON.stringify(row)}\n`, { flag: 'a' }) +} + +async function writeJsonAtomic(path: string, value: unknown): Promise { + await mkdir(dirname(path), { recursive: true }) + const tmp = `${path}.${process.pid}.${Date.now()}.tmp` + await writeFile(tmp, `${JSON.stringify(value, null, 2)}\n`, 'utf8') + await rename(tmp, path) +} + +function statePath(runDir: string): string { + return join(runDir, 'state.json') +} + +function average(values: readonly number[]): number { + const finite = values.filter(Number.isFinite) + if (finite.length === 0) return 0 + return finite.reduce((sum, value) => sum + value, 0) / finite.length +} diff --git a/tests/kb-improvement.test.ts b/tests/kb-improvement.test.ts new file mode 100644 index 0000000..47b1236 --- /dev/null +++ b/tests/kb-improvement.test.ts @@ -0,0 +1,374 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + buildEvalKnowledgeBundle, + buildKnowledgeIndex, + defineReadinessSpec, + hashKnowledgeBase, + improveKnowledgeBase, + initKnowledgeBase, + knowledgeImprovementRunDir, + sha256, + stableId, +} from '../src/index' + +async function withKb(fn: (root: string) => Promise): Promise { + const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-improve-')) + try { + await initKnowledgeBase(root) + await fn(root) + } finally { + await rm(root, { recursive: true, force: true }) + } +} + +const refundSpec = defineReadinessSpec({ + id: 'refund-policy', + description: 'Refund policy support knowledge', + query: 'refund policy customer billing refund', + requiredFor: ['support-agent'], + importance: 'blocking', + minHits: 1, + minSources: 1, +}) + +function refundSource() { + const text = 'The billing support refund policy allows refunds within 30 days with receipt proof.' + const uri = 'research://refund-policy' + return { + uri, + text, + title: 'Refund Policy Source', + id: stableId('src', `${sha256(text)}:${uri}`), + } +} + +function refundProposal(sourceId: string, extra = ''): string { + return [ + '---FILE: knowledge/support/refund-policy.md---', + '---', + 'id: refund-policy', + 'title: Refund Policy', + 'sources:', + ` - ${sourceId}`, + '---', + '# Refund Policy', + 'Billing support can grant a customer refund within 30 days when receipt proof is present.', + extra, + '---END FILE---', + ].join('\n') +} + +describe('improveKnowledgeBase', () => { + it('promotes a candidate KB after readiness separates weak from strong', async () => { + await withKb(async (root) => { + const emptyIndex = await buildKnowledgeIndex(root) + const emptyReadiness = buildEvalKnowledgeBundle({ + taskId: 'support-agent', + index: emptyIndex, + specs: [refundSpec], + }) + expect(emptyReadiness.report.blockingMissingRequirements).toHaveLength(1) + + const source = refundSource() + const result = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'refund-improvement', + readinessSpecs: [refundSpec], + strict: true, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + }) + + expect(result.promoted).toBe(true) + expect(result.state.status).toBe('promoted') + expect(result.candidate?.evaluation?.score).toBe(1) + expect( + result.candidate?.lifecycle?.some((record) => + record.phases.some( + (phase) => phase.phase === 'knowledge-update' && phase.status === 'completed', + ), + ), + ).toBe(true) + + const promotedIndex = await buildKnowledgeIndex(root) + expect(promotedIndex.pages.map((page) => page.id)).toContain('refund-policy') + const promotedReadiness = buildEvalKnowledgeBundle({ + taskId: 'support-agent', + index: promotedIndex, + specs: [refundSpec], + }) + expect(promotedReadiness.report.blockingMissingRequirements).toHaveLength(0) + }) + }) + + it('resumes a candidate workspace after an external edit and promotes the edited artifact', async () => { + await withKb(async (root) => { + const source = refundSource() + const first = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'resume-edit', + readinessSpecs: [refundSpec], + strict: true, + promote: false, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + }) + + expect(first.promoted).toBe(false) + expect(first.state.status).toBe('candidate-ready') + const candidateRoot = first.candidate?.candidateRoot + expect(candidateRoot).toBeDefined() + + const editedPage = join(candidateRoot!, 'knowledge', 'support', 'refund-escalation.md') + await mkdir(dirname(editedPage), { recursive: true }) + await writeFile( + editedPage, + [ + '---', + 'id: refund-escalation', + 'title: Refund Escalation', + 'sources:', + ` - ${source.id}`, + '---', + '# Refund Escalation', + 'Escalate refund requests that lack receipt proof.', + ].join('\n'), + ) + + const resumed = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'resume-edit', + readinessSpecs: [refundSpec], + strict: true, + }) + + expect(resumed.promoted).toBe(true) + const promoted = await readFile( + join(root, 'knowledge', 'support', 'refund-escalation.md'), + 'utf8', + ) + expect(promoted).toContain('Escalate refund requests') + }) + }) + + it('retries a running candidate after a crashed operator run', async () => { + await withKb(async (root) => { + const source = refundSource() + await expect( + improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'crash-resume', + readinessSpecs: [refundSpec], + strict: true, + step: () => { + throw new Error('research worker crashed') + }, + }), + ).rejects.toThrow(/research worker crashed/) + + const resumed = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'crash-resume', + readinessSpecs: [refundSpec], + strict: true, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + }) + + expect(resumed.promoted).toBe(true) + expect(resumed.state.candidates).toHaveLength(1) + expect(resumed.candidate?.iteration).toBe(1) + }) + }) + + it('blocks promotion when the live KB changed after candidate creation', async () => { + await withKb(async (root) => { + const source = refundSource() + const first = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'conflict', + readinessSpecs: [refundSpec], + strict: true, + promote: false, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + }) + expect(first.state.status).toBe('candidate-ready') + + const concurrentPage = join(root, 'knowledge', 'support', 'concurrent.md') + await mkdir(dirname(concurrentPage), { recursive: true }) + await writeFile( + concurrentPage, + [ + '---', + 'id: concurrent', + 'title: Concurrent Edit', + '---', + '# Concurrent Edit', + 'A different agent edited the live KB.', + ].join('\n'), + ) + + const blocked = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'conflict', + readinessSpecs: [refundSpec], + strict: true, + }) + + expect(blocked.blocked).toBe(true) + expect(blocked.state.status).toBe('blocked') + expect(blocked.state.blockedReason).toMatch(/base changed before promotion/) + expect(await readFile(concurrentPage, 'utf8')).toContain('different agent') + }) + }) + + it('fails fast when another active operator owns the run lease', async () => { + await withKb(async (root) => { + const runDir = knowledgeImprovementRunDir(root, 'locked') + await mkdir(runDir, { recursive: true }) + await writeFile( + join(runDir, 'run.lock'), + `${JSON.stringify({ + ownerId: 'agent-a', + acquiredAt: '2026-01-01T00:00:00.000Z', + expiresAt: '2999-01-01T00:00:00.000Z', + pid: 123, + })}\n`, + ) + + await expect( + improveKnowledgeBase({ + root, + goal: 'Locked run', + runId: 'locked', + readinessSpecs: [refundSpec], + }), + ).rejects.toThrow(/locked by agent-a/) + }) + }) + + it('runs retrieval tuning against the updated candidate KB', async () => { + await withKb(async (root) => { + const source = refundSource() + const result = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'retrieval', + readinessSpecs: [refundSpec], + strict: true, + promote: false, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + retrieval: { + baseline: { k: 1 }, + scenarios: [ + { + id: 'q-train', + kind: 'retrieval-eval', + query: 'refund policy', + expected: { kind: 'page', pageId: 'refund-policy' }, + }, + ], + holdoutScenarios: [ + { + id: 'q-holdout', + kind: 'retrieval-eval', + query: 'billing refund', + expected: { kind: 'page', pageId: 'refund-policy' }, + }, + ], + searchSpace: { k: [1, 2] }, + retrieve: async ({ k }) => ({ + hits: [ + { pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 }, + ...(k >= 2 + ? [{ pageId: 'refund-policy', path: 'knowledge/support/refund-policy.md', rank: 2 }] + : []), + ], + }), + targetRecall: 1, + deltaThreshold: 0.01, + populationSize: 1, + maxGenerations: 1, + expectUsage: 'off', + }, + }) + + expect(result.candidate?.retrievalWinnerConfig).toMatchObject({ k: 2 }) + expect(result.lifecycle?.phases.map((phase) => `${phase.phase}:${phase.status}`)).toContain( + 'retrieval-tuning:completed', + ) + }) + }) + + it('rejects a candidate when answer quality fails', async () => { + await withKb(async (root) => { + const source = refundSource() + const result = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'answer-quality-failure', + readinessSpecs: [refundSpec], + strict: true, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + evaluateAnswers: () => ({ + passed: false, + metrics: { faithfulness: 0, answer_relevance: 0.8 }, + findings: [ + { + id: 'refund-answer:faithfulness', + kind: 'generator-unsupported-claim', + severity: 'error', + message: 'The answer made an unsupported claim.', + }, + ], + }), + }) + + expect(result.promoted).toBe(false) + expect(result.state.status).toBe('rejected') + expect(result.candidate?.evaluation?.passed).toBe(false) + expect(result.candidate?.evaluation?.notes).toContain('answer quality failed') + }) + }) + + it('hashes only promoted KB surfaces, not improvement run scratch space', async () => { + await withKb(async (root) => { + const before = await hashKnowledgeBase(root) + const runDir = knowledgeImprovementRunDir(root, 'scratch') + await mkdir(runDir, { recursive: true }) + await writeFile(join(runDir, 'scratch.txt'), 'scratch state') + await expect(hashKnowledgeBase(root)).resolves.toBe(before) + }) + }) +})