diff --git a/README.md b/README.md index 5837236..3b7d1d4 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,9 @@ from `@tangle-network/agent-knowledge`. scoring, and conflict-safe promotion. Use it when running agents in loops against a real KB rather than only exposing phase hooks. +- `evaluateKnowledgeBaseReadiness()` checks one KB root without running an + improvement loop: it rebuilds the index, validates/lints it, scores KB + quality, and evaluates configured readiness specs. - RAG answer evaluation follows the common open-source shape used by Ragas, DeepEval, TruLens, and RAGChecker: context quality, answer relevance, support/faithfulness, citations, abstention, and failure diagnosis. diff --git a/package.json b/package.json index 808007d..fa6255e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-knowledge", - "version": "1.10.0", + "version": "1.11.0", "description": "Source-grounded, eval-gated knowledge growth primitives for agents.", "homepage": "https://github.com/tangle-network/agent-knowledge#readme", "repository": { diff --git a/src/index.ts b/src/index.ts index 0618d7b..6e451ea 100644 --- a/src/index.ts +++ b/src/index.ts @@ -25,6 +25,7 @@ export * from './proposals' export * from './propose-from-finding' export * from './rag-eval' export * from './rag-improvement-loop' +export * from './readiness-check' export * from './release' export * from './research-driving-driver' export * from './research-loop' diff --git a/src/readiness-check.ts b/src/readiness-check.ts new file mode 100644 index 0000000..5cd8537 --- /dev/null +++ b/src/readiness-check.ts @@ -0,0 +1,87 @@ +import type { + BuildEvalKnowledgeBundleOptions, + EvalKnowledgeBundleBuildResult, + KnowledgeReadinessSpec, +} from './eval-readiness' +import { buildKnowledgeIndex } from './indexer' +import { + type KnowledgeBaseQualityOptions, + type KnowledgeBaseQualityReport, + scoreKnowledgeBaseIndex, +} from './rag-eval' +import { readinessFor } from './readiness-helpers' +import type { KnowledgeIndex } from './types' +import { + type ValidateKnowledgeOptions, + type ValidateKnowledgeResult, + validateKnowledgeIndex, +} from './validate' + +export interface EvaluateKnowledgeBaseReadinessOptions { + root: string + goal: string + readinessSpecs?: readonly KnowledgeReadinessSpec[] + readinessTaskId?: string + readiness?: Omit + strict?: ValidateKnowledgeOptions['strict'] + kbQuality?: KnowledgeBaseQualityOptions +} + +export interface KnowledgeBaseReadinessEvaluation { + ready: boolean + summary: string + index: KnowledgeIndex + validation: ValidateKnowledgeResult + readiness?: EvalKnowledgeBundleBuildResult + kbQuality: KnowledgeBaseQualityReport + dimensions: { + validation: number + kb_quality: number + blocking_readiness: number + } +} + +export async function evaluateKnowledgeBaseReadiness( + options: EvaluateKnowledgeBaseReadinessOptions, +): Promise { + const index = await buildKnowledgeIndex(options.root) + const validation = validateKnowledgeIndex(index, { strict: options.strict }) + const readiness = readinessFor( + { + ...options, + readinessSpecs: options.readinessSpecs ? [...options.readinessSpecs] : undefined, + }, + index, + ) + const kbQuality = scoreKnowledgeBaseIndex(index, { + strict: options.strict, + ...options.kbQuality, + }) + const blockingMissing = readiness?.report.blockingMissingRequirements.length ?? 0 + const blockingTotal = + options.readinessSpecs?.filter((spec) => spec.importance === 'blocking').length ?? 0 + const blockingReadiness = + blockingTotal === 0 ? 1 : Math.max(0, blockingTotal - blockingMissing) / blockingTotal + const ready = validation.ok && kbQuality.ok && blockingMissing === 0 + const failures = [ + validation.ok ? undefined : `${validation.findings.length} validation finding(s)`, + kbQuality.ok ? undefined : `${kbQuality.findings.length} KB quality finding(s)`, + blockingMissing === 0 + ? undefined + : `${blockingMissing}/${blockingTotal} blocking readiness requirement(s) missing`, + ].filter((failure): failure is string => Boolean(failure)) + + return { + ready, + summary: ready ? 'knowledge base passed readiness checks' : failures.join('; '), + index, + validation, + readiness, + kbQuality, + dimensions: { + validation: validation.ok ? 1 : 0, + kb_quality: kbQuality.ok ? 1 : 0, + blocking_readiness: blockingReadiness, + }, + } +} diff --git a/tests/kb-improvement.test.ts b/tests/kb-improvement.test.ts index f381531..d26f27a 100644 --- a/tests/kb-improvement.test.ts +++ b/tests/kb-improvement.test.ts @@ -6,6 +6,7 @@ import { buildEvalKnowledgeBundle, buildKnowledgeIndex, defineReadinessSpec, + evaluateKnowledgeBaseReadiness, hashKnowledgeBase, improveKnowledgeBase, initKnowledgeBase, @@ -62,6 +63,51 @@ function refundProposal(sourceId: string, extra = ''): string { } describe('improveKnowledgeBase', () => { + it('evaluates root-level KB readiness without running an improvement loop', async () => { + await withKb(async (root) => { + const empty = await evaluateKnowledgeBaseReadiness({ + root, + goal: 'Build billing support refund-policy knowledge', + readinessSpecs: [refundSpec], + strict: true, + }) + + expect(empty.ready).toBe(false) + expect(empty.dimensions.blocking_readiness).toBe(0) + expect(empty.summary).toContain('blocking readiness requirement') + + const source = refundSource() + const improved = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'readiness-evaluator', + readinessSpecs: [refundSpec], + strict: true, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + }) + expect(improved.promoted).toBe(true) + + const ready = await evaluateKnowledgeBaseReadiness({ + root, + goal: 'Build billing support refund-policy knowledge', + readinessSpecs: [refundSpec], + strict: true, + }) + + expect(ready.ready).toBe(true) + expect(ready.summary).toBe('knowledge base passed readiness checks') + expect(ready.dimensions).toMatchObject({ + validation: 1, + kb_quality: 1, + blocking_readiness: 1, + }) + }) + }) + it('promotes a candidate KB after readiness separates weak from strong', async () => { await withKb(async (root) => { const emptyIndex = await buildKnowledgeIndex(root)