diff --git a/README.md b/README.md index a94492b..5837236 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ This package turns raw sources and generated markdown knowledge into a versionab - [Agent-Eval integration](#agent-eval-integration) — retrieval eval + readiness bundles + release reports - [Memory adapters](#memory-adapters) — generic memory contract + Neo4j Agent Memory bridge - [Research loop](#research-loop) — `runKnowledgeResearchLoop` + control-loop adapter -- [Researcher profile](#researcher-profile) — sandbox `AgentProfile` for `runLoop` +- [Runtime integration](#runtime-integration) — how agent runners plug into the pure KB loop - [Pluggable knowledge sources](#pluggable-knowledge-sources) — live authorities → eval re-runs ## Install @@ -30,8 +30,8 @@ Two ways in, depending on what you're doing: - **Author / inspect a KB by hand** → the [CLI](#cli) (`init` → `source-add` → `index` → `search` → `lint`). Fastest way to see the shape on disk. - **Drive it from an agent** → pick the primitive by intent: - *"Does the agent have enough context to run?"* → [`buildEvalKnowledgeBundle`](#agent-eval-integration) (block / ask / acquire before execution). - - *"Grow the KB as a researcher"* → [`runKnowledgeResearchLoop`](#research-loop) (deterministic mechanics; your agent owns judgment), [`runTwoAgentResearchLoop`](#two-agent-research-loop) (researcher proposes, verifier checks + fills gaps, offline), or the sandbox [researcher profile](#researcher-profile) for `runLoop`. - - *"Spawn one researcher per sub-topic and stop when the KB is ready"* → [`runResearchSupervisor`](#research-supervisor) (a supervisor brain sizes the topology over a `Scope`; LIVE, needs creds). + - *"Grow the KB as a researcher"* → [`runKnowledgeResearchLoop`](#research-loop) (deterministic mechanics; your agent owns judgment) or [`runTwoAgentResearchLoop`](#two-agent-research-loop) (researcher proposes, verifier checks + fills gaps). + - *"Spawn live agents to improve a KB"* → pass an `updateKnowledge` callback to `improveKnowledgeBase`; runtime-backed supervisors live in `@tangle-network/agent-runtime`. - *"Tune retrieval for a knowledge base"* → `runRetrievalImprovementLoop` in the [Agent-Eval integration](#agent-eval-integration) section. - *"Run an operator-grade KB improvement cycle"* → `improveKnowledgeBase` in the [Agent-Eval integration](#agent-eval-integration) section. It creates a candidate KB, lets agents or deterministic hooks improve it, runs configured evals, can be resumed, and promotes only when the live KB has not changed underneath it. @@ -537,86 +537,30 @@ await runTwoAgentResearchLoop({ }) ``` -## Research supervisor +## Runtime integration -`runResearchSupervisor()` is the LIVE counterpart: a supervisor brain creates the -topology dynamically — one researcher worker per sub-topic over a `Scope` — and -stops when the knowledge base is ready. It is a thin wrapper over `supervise()` -from `@tangle-network/agent-runtime/loops`; it builds nothing new. The worker -shape is the [researcher profile](#researcher-profile), and the completion oracle -is `knowledgeReadinessDeliverable` (re-reads the KB from disk and runs the -readiness gate, so it stops on the real grounded state, not a worker's -self-report). Needs creds: a supervisor router brain plus a worker backend. +`agent-knowledge` owns knowledge state and measurement. +It deliberately does not own an agent runner. +Live agent orchestration belongs in `@tangle-network/agent-runtime`, which can call `improveKnowledgeBase` by passing an `updateKnowledge` callback: ```ts -import { defineReadinessSpec, runResearchSupervisor } from '@tangle-network/agent-knowledge' +import { improveKnowledgeBase } from '@tangle-network/agent-knowledge' -await runResearchSupervisor({ +await improveKnowledgeBase({ root: './kb', goal: 'Build a grounded onboarding wiki for billing support', - readinessSpecs: [defineReadinessSpec({ - id: 'refund-policy', - description: 'Refund policy grounding', - query: 'refund policy customer request', - requiredFor: ['support-agent'], - })], - budget: { maxIterations: 12, maxTokens: 200_000, maxUsd: 5 }, - // WHERE researcher workers run — the real backend seam. - backend: { - backend: 'router', // or 'sandbox' / 'cli' / 'bridge' - routerBaseUrl: process.env.ROUTER_BASE_URL!, - routerKey: process.env.ROUTER_KEY!, - model: 'your-worker-model', - }, -}) -``` - -## Researcher profile - -`@tangle-network/agent-knowledge/profiles` ships a sandbox-SDK -`AgentProfile` preset for source-grounded research agents. Pairs with -`runLoop` from `@tangle-network/agent-runtime/loops` — the profile owns -the prompt + output adapter + validator; the kernel owns iteration, -concurrency, cost, and trace emission. - -```ts -import { runLoop } from '@tangle-network/agent-runtime/loops' -import { multiHarnessResearcherFanout } from '@tangle-network/agent-knowledge/profiles' - -const research = multiHarnessResearcherFanout({ - harnesses: ['opencode/zai-coding-plan/glm-5.1', 'claude-code', 'codex'], -}) - -const result = await runLoop({ - driver: research.driver, - agentRuns: research.agentRuns, - output: research.output, - validator: research.validator, - task: { - question: 'What content does cpg-founder ICP engage with on Twitter?', - knowledgeNamespace: 'cust_42', - sources: ['twitter', 'web'], - maxItems: 20, - minConfidence: 0.6, + readinessSpecs, + updateKnowledge: async ({ goal, findings }) => { + // Call your agent runner here, then apply source-backed write blocks. + return { + applied: true, + summary: `updated KB for ${goal} with ${findings.length} finding(s)`, + } }, - ctx: { sandboxClient }, }) - -if (result.winner?.verdict?.valid) { - // result.winner.output.proposedWrites: KnowledgeUpdate[] - // The profile does NOT materialize. Decide whether to apply. - for (const write of result.winner.output.proposedWrites) { - // route through applyKnowledgeWriteBlocks / a KbStore put when ready - } -} ``` -Three invariants are enforced by the validator: - -- **Namespace isolation** — every `KnowledgeItem` + `KnowledgeUpdate` - must carry `task.knowledgeNamespace`. Cross-tenant writes hard-fail. -- **Provenance** — every item carries at least one evidence entry. -- **Citation density** — quotes-with-source / items >= 0.7 by default. +This keeps the dependency graph acyclic: `agent-knowledge` depends on `agent-eval`; agent runners depend on `agent-knowledge`, not the reverse. Validator scoring (default; overridable): diff --git a/docs/recursive-research-leaf.md b/docs/recursive-research-leaf.md deleted file mode 100644 index d4de8c5..0000000 --- a/docs/recursive-research-leaf.md +++ /dev/null @@ -1,86 +0,0 @@ -# Skill: spawn a driver-loop as a leaf for a large sub-topic - -Supervisor-facing guidance. The `runResearchSupervisor` brain (see -`src/research-supervisor.ts`) decomposes a goal into sub-topics and spawns one -researcher per sub-topic over the live `Scope`. By default each spawn is a **bare -worker**: a single researcher pass that adds sources and proposes pages. - -For a **large** sub-topic, a single worker pass under-covers it. The recursive -move is to spawn, in place of that bare worker, a **driver-loop leaf** — a child -that is itself a driver running its own worker (the two-agent loop), so the -sub-topic gets verify + gap-fill + readiness-gate iteration, not one shot. - -`agent-runtime`'s `Scope.spawn(agent, task, opts)` makes this a first-class -recursion: the spawned `Agent` can itself be a driver, not just a worker. The -supervisor spawns a child `Agent` whose `act(task, childScope)` runs -`runTwoAgentResearchLoop` (or its own `runResearchSupervisor`) scoped to the -sub-topic, against the SAME knowledge base root, with a sub-topic-scoped slice of -readiness specs. The conserved budget pool reserves `opts.budget` for the child -atomically and refunds the unspent remainder on settle, so a driver-loop leaf -spends from the same pool a bare worker would have — depth costs budget, not extra -budget. - -## When it's worth it - -Spawn a driver-loop leaf (instead of a bare worker) for a sub-topic when ANY of -these hold; otherwise spawn a bare worker. - -| Signal | Bare worker | Driver-loop leaf | -| --- | --- | --- | -| **Breadth** — distinct readiness specs the sub-topic must close | 1–2 | 3+ | -| **Depth** — does a fact need a source THEN a derived claim citing it? | shallow (source ≈ answer) | layered (source → claim → cross-check) | -| **Expected rounds** — passes to reach no-blocking-gaps for the sub-topic | ~1 | 2+ (a worker's first pass predictably leaves blocking gaps) | -| **Verification risk** — is a wrong/duplicate source likely and costly here? | low | high (the leaf's driver rejects bad sources before they commit) | -| **Budget headroom** — iterations/tokens left in the pool for this branch | tight | comfortable (a leaf needs ≥ ~3× a worker's per-pass spend) | - -Rule of thumb: **breadth ≥ 3 specs OR expected rounds ≥ 2 OR high verification -risk ⇒ driver-loop leaf.** A sub-topic that is one fact from one source is a bare -worker — wrapping it in a driver only burns budget on a loop that stops after -round 1. - -## The shape - -```ts -import type { Agent, Scope } from '@tangle-network/agent-runtime/loops' -import { runTwoAgentResearchLoop, type TwoAgentResearchLoopResult } from '@tangle-network/agent-knowledge' - -// For a LARGE sub-topic, spawn a sub-driver leaf that runs the two-agent loop -// scoped to that sub-topic. The child shares the KB root and the conserved -// budget pool; it gets its own driver (verify + fill + gate) instead of a single -// worker pass. The child is an `Agent` whose `act` IS the two-agent loop. -const researchLeaf: Agent = { - name: `research-leaf:${subTopic.id}`, - async act(task, childScope: Scope) { - return runTwoAgentResearchLoop({ - root, // the SAME knowledge base - goal: task.goal, - worker, // the sub-topic researcher - driver, // verifies the worker's sources, gap-fills, gates on readiness - driverResearches: true, // the leaf's driver also researches the missed gaps - readinessSpecs: task.specs, // the sub-topic's slice of the gate - signal: childScope.signal, - }) - }, -} - -// Reserves `budget` from the conserved pool atomically; refunds on settle. -const spawned = scope.spawn(researchLeaf, subTopic, { - label: `research-leaf:${subTopic.id}`, - budget: perSubTopicBudget, // a slice of the conserved pool -}) -if (!spawned.ok) { - // fail-closed: 'budget-exhausted' | 'depth-exceeded' — spawn a bare worker or stop. -} -``` - -A bare worker, by contrast, is `scope.spawn(workerFromBackend(backend)(...), …)` -of the researcher profile — no inner driver, no per-sub-topic readiness gate, one -pass. - -## Why this is a leaf, not a fork - -The driver-loop leaf is still ONE branch of the supervisor's tree. It reads and -writes the same knowledge base, settles back through the same `Scope`, and -spends from the same budget pool. The recursion is depth (a sub-driver inside a -branch), not a second knowledge base — there is exactly one KB, and the -supervisor's readiness gate over the whole KB is still what ends the run. diff --git a/docs/results/autodata-live.md b/docs/results/autodata-live.md deleted file mode 100644 index 80df2e8..0000000 --- a/docs/results/autodata-live.md +++ /dev/null @@ -1,154 +0,0 @@ -# Autodata live result: the causal-challenger loop reliably discriminates at power — 38% accept-rate, CI [23%, 55%] (NOT a coin-flip) - -Running the agentic data-creation loop (`src/autodata/`) on real arXiv docs with real two-tier -solvers, to manufacture training examples that separate a strong solver from a weak one (the -discriminative reward of the Autodata / Agentic-Self-Instruct method). - -**Powered headline (32 independent slots, 2 docs, samples=4):** the loop **reliably manufactures -discriminating examples — accept-rate 38%, Wilson 95% CI [23%, 55%]** (12 of 32 slots cleared the -hard accept bar: weak < 0.5 ∧ strong ≥ 0.65 ∧ gap ≥ 0.2). The CI lower bound (23%) excludes ~0, so -this is a **real, repeatable rate, not the n=1–2 luck** that made it look like a coin-flip at n=3. -Acceptance is **doc-dependent** (mixtral 19%, deepseek-v3 56%) and gated by **whether the weak model -struggles** (it does on only 39% of attempts), but it is decisively above zero on both docs. This -**replaces** the earlier n=3 result, which was too noisy to tell "real rate" from "coin-flip ~0". - -## The two levers that turned the null into a positive - -The earlier null ("a small model performs as well as a frontier one") had TWO compounding causes, -both fixed here: - -1. **The question leaked the answer / asked for recall.** The challenger wrote lookup-style questions - whose answer sat in the provided context, so an 8B read it out as well as a frontier model. - Fix — the **non-extractive causal challenger**: it must author CAUSAL / COMPARATIVE / MECHANISM / - THESIS-CONSISTENCY questions whose answer is DERIVED, the context must hold premises but not state - the conclusion, the solver no longer sees the rubric (the mark scheme), and the judge now sees the - context and scores a dedicated `reasoning` dimension LOW when the answer merely restates it (the - paper's negative criterion). On reject, the fold steers per reason ("too easy" → go non-extractive - and harder; "too hard" → ease; "not discriminative" → sharpen). - -2. **The grounding doc was memorized.** The default was "Attention Is All You Need" — the most - canonical paper in ML, which an 8B has memorized, so even reasoning questions are answerable from - pretraining and capability cannot separate. Fix — **ground on docs the weak solver has not - memorized**: the Mixtral-of-Experts paper (arXiv 2401.04088, Jan 2024) and the DeepSeek-V3 paper - (arXiv 2412.19437, Dec 2024), both post-dating `llama-3.1-8b`'s knowledge cutoff, forcing it to - reason from the context. - -## Setup (all env-overridable) - -| role | model | why | -|---|---|---| -| weak solver | `groq/llama-3.1-8b-instant` | small; cutoff predates the 2024 docs → must reason, can't recall | -| strong solver | `gemini-2.5-pro` | frontier reasoner; a real wide capability gap | -| challenger + judge | `deepseek-v4-flash` | capable, fast, reliable, a DIFFERENT family from both solvers (no judge-bias) | -| grounding doc A | Mixtral-of-Experts (2401.04088) | non-memorized; MoE expert routing / gating (`focus=expert`) | -| grounding doc B | DeepSeek-V3 (2412.19437) | non-memorized; auxiliary-loss-free load balancing / expert specialization (`focus=auxiliary`) | - -Accept thresholds (the paper's): strong ≥ 0.65, weak < 0.50, gap ≥ 0.20. (`glm-5.2`, the brief's -challenger/judge, was returning upstream-capacity 503s; `deepseek-v4-flash` is the live, neutral -substitute. `routerChat` retries transient 503/429/timeout with bounded backoff.) - -The grounding chunk must be **prose, not equations**: an equation-dense chunk (e.g. DeepSeek-V3's MLA -section) breaks the challenger's strict-JSON output (LaTeX backslashes), so both `focus` terms select -the prose description of an MoE-expert mechanism. Even so, 5 of 32 slots (~16%) still hit a -LaTeX-in-JSON failure and produced no example — those count as rejects in the headline (the -conservative floor); see below. - -## The judge is reliable (checked before trusting any gap) - -A controlled probe scored one genuinely-strong vs one genuinely-weak answer to the same question, 3× -each: `deepseek-v4-flash` returned strong `[1.00, 1.00, 1.00]` (mean 1.00) vs weak `[0.23, 0.13, -0.17]` (mean 0.18) — a consistent **0.82** separation, ranking strong above weak every time. So a -measured gap reflects answer quality, not judge noise. (`gemini-2.5-flash` as judge threw parse -errors — `deepseek` is the better grader here.) - -## The powered result — a real ~38% accept-rate - -**Design (fixed-slots, not until-N-accepted):** run a fixed K = 32 independent slots (each slot = one -full challenger → refine → accept cycle), split 16 / 16 across the two docs, samples = 4 per solver -(stabilise the weak mean), maxRetries = 2 (3 challenger attempts per slot). Record each slot's -outcome (accept / reject) + best gap, so the rate is bounded-cost and unbiased. Runnable: -`src/autodata/powered.ts`; per-attempt autopsy JSONL per doc; the CIs are agent-eval's published -estimators (`wilson` for the binomial accept-rate, `pairedBootstrap` for the paired widening). - -| metric | value | read | -|---|---|---| -| **accept-rate (headline)** | **38% CI [23%, 55%]** (12 / 32) | excludes ~0 → **reliable, not a coin-flip** | -| accept-rate (producing slots) | 44% CI [28%, 63%] (12 / 27) | excludes the 5 challenger-stage (LaTeX) failures | -| — mixtral | 19% CI [7%, 43%] (3 / 16) | the harder doc; still excludes 0 | -| — deepseek-v3 | 56% CI [33%, 77%] (9 / 16) | the easier-to-discriminate doc | -| best gap / slot (n=27) | min −0.23 · median **0.42** · p90 0.80 · max 0.95 | how far each slot separated the tiers | -| plain (first-draft) gap / slot | min −0.23 · median 0.19 · p90 0.61 · max 0.95 | the un-refined baseline | -| **gap-widening Δ (plain → best-refined)** | mean **+0.103** CI [+0.029, +0.193] (paired bootstrap, n=27) | the fold's lift; **excludes 0** (median Δ 0 — it helps a minority) | -| weak score / attempt (n=33) | min 0.05 · median **0.55** · max 1.00 | the variance source — competent ~half the time | -| strong score / attempt (n=33) | min 0.21 · median **0.99** · max 1.00 | the strong solver almost always derives | - -**Accept-rule decomposition (33 quality-clean attempts):** strong ≥ 0.65 = **88%**, weak < 0.50 = -**39%** ← the binding gate, gap ≥ 0.20 = 52%, all-three (= accept) = 36%. The strong solver derives -almost everything; the bottleneck is the weak model failing — which happens on only ~39% of -attempts, so the per-slot accept-rate is set by **how often `llama-3.1-8b` actually struggles**, not -by the challenger or judge. **Total live spend: $0.57** for the 32-slot run (~$1.0 including pilots). - -## Two autopsied accepted examples (real discrimination, both answers read) - -**deepseek-v3 — gap 0.93 (weak 0.07, strong 1.00):** -> **Q:** Why does using a *sequence-wise* auxiliary loss lead to a higher validation loss than a -> *batch-wise* auxiliary loss or the auxiliary-loss-free method in MoE models? - -- **strong (`gemini-2.5-pro`): 1.00** — derives that the sequence-wise loss imposes a *stricter, - less flexible* per-sequence balance constraint that *hinders the emergence of expert - specialisation*. Correct, matches the reference. -- **weak (`llama-3.1-8b`): [0.10, 0.03, 0.10, 0.03]** — *restates the question* and never derives the - reason. A recall-shaped non-answer; the judge's `reasoning` criterion floors it. - -**mixtral — gap 0.95 (weak 0.05, strong 1.00):** -> **Q:** The text says each input is routed to 2 of 8 experts, yet the output sums `G(x)_i · E_i(x)` -> over all `n` experts. Are these consistent? If not, which should be revised? - -- **strong: 1.00** — derives YES, consistent: the gating vector `G(x)` is *sparse* (nonzero only for - the 2 selected experts), so the full-`n` sum effectively includes only those 2. Correct. -- **weak: [0.03, 0.07, 0.03, 0.07]** — concludes the statements are *inconsistent*; it never grasps - the sparse-gating equivalence. A genuine reasoning error, not a judge artifact or leakage (the - answer is derived, not in the context). - -These are real weak-fails-strong-derives examples on both docs — the loop is manufacturing genuine -discrimination, not gaming the gap. - -## The finding - -The question "does the causal-challenger loop reliably manufacture discriminating examples, or is -acceptance a coin-flip ~0?" is now **settled at power: it reliably works.** Accept-rate **38%, CI -[23%, 55%]** over 32 slots — the lower bound excludes ~0, and even the harder of the two docs -(mixtral, 19% [7%, 43%]) excludes 0. The fold also **reliably widens the gap** (mean +0.103, CI -[+0.029, +0.193]), reproducing the n=3 direction at power, though most of the discrimination comes -from the first causal draft already separating (median widening 0 — the refine helps a minority of -slots). - -Two honest caveats, both quantified, neither overturns the verdict: - -1. **Doc-dependence.** The rate ranges 19% (mixtral) → 56% (deepseek-v3). The pooled 38% is a real - average across two non-memorized MoE papers, not a single lucky doc — but expect the rate to move - with the source material's difficulty for the 8B. -2. **The binding constraint is the weak model's competence, not the method.** `llama-3.1-8b` answers - these MoE-reasoning questions competently (weak median 0.55) about as often as it flails, so - ~39% of attempts clear the "weak must struggle" gate. A weaker weak model (or harder docs) would - raise the rate; a stronger one would lower it. The loop's discriminative reward works as designed — - the rate is a property of the **tier gap**, which is exactly what it should measure. - -## Status - -Mechanism + observability + **power**: solid. The accept-rate is measured at n=32 with a Wilson CI -that excludes ~0, the gap-widening with a paired-bootstrap CI that excludes 0, every attempt dumped -to a JSONL autopsy trail, and the two headline accepted examples read end-to-end (real -discrimination). The n=3 "coin-flip ~0?" worry is **resolved: ~38% accept-rate, not zero.** - -## Reproduce - -``` -# Powered accept-rate + CIs (32 slots, 2 docs, samples=4) — the headline result: -dotenvx run -f .env -- pnpm tsx src/autodata/powered.ts -# knobs: AUTODATA_SLOTS_PER_DOC=16 AUTODATA_SAMPLES=4 AUTODATA_MAXRETRIES=2 - -# Single-doc builder + recall-vs-causal calibration (the lever's A/B): -dotenvx run -f .env -- pnpm tsx src/autodata/run.ts -dotenvx run -f .env -- pnpm tsx src/autodata/calibrate.ts -``` diff --git a/docs/supervisor-profiles.md b/docs/supervisor-profiles.md deleted file mode 100644 index 2cb7903..0000000 --- a/docs/supervisor-profiles.md +++ /dev/null @@ -1,295 +0,0 @@ -# Supervisor prompt + agent-profile ideas - -Four `AgentProfile` sketches for the research/knowledge domain — a **researcher worker**, a -**verifier driver**, a **research supervisor**, and a thin **dedup-first verifier** — plus a port -plan for the existing `~/code/supervisor-lab`. - -This is a *design doc*, not shipped code. Each sketch names the real primitive it composes (in -`agent-knowledge` or `supervisor-lab`) so building it is "assemble", not "invent". Where a primitive -already exists, the sketch says so and points at it — we extend, we do not fork. - -## What we borrowed from emilkowalski/skills - -[`emilkowalski/skills`](https://github.com/emilkowalski/skills) is a small pack of Claude skills for -design engineering (`emil-design-eng`, `review-animations`). It is not about agents, but its *skill -shape* is worth copying, and one structural choice maps directly onto our verifier: - -- **Tight frontmatter, two fields.** `name` + a one-paragraph `description` that says *when* to - invoke. Nothing else. Our profiles' `description` should read the same way: a trigger, not a - feature list. -- **Terse non-negotiable rules over prose.** `review-animations` is "The Ten Non-Negotiable - Standards" — numbered, imperative, absolute ("Animate `transform` and `opacity` only"). - A worker system prompt is more legible as a short numbered contract than as paragraphs. -- **"Default to flagging; approval is earned."** The review skill is *adversarial by posture* — it - exists to reject, and its output is a findings table + a verdict. This is exactly the right posture - for a verifier, and it informs the dedup point below. -- **`disable-model-invocation: true` on the reviewer.** The review skill is not auto-invoked by the - model; it is routed in deliberately. Our verifier is the same: a driver *calls* it at a gate, the - worker never invokes it on a whim. -- **Cheap because narrow.** Each skill does one thing. The review skill carries no design *generation* - ability — it only judges. That separation (generate vs. judge) is the whole reason the judge can be - thin. - -### The dedup insight (why the verifier is a thin profile, not a heavy one) - -`review-animations` is mostly a deduplicated rule set: the craft knowledge lives in `emil-design-eng`, -and the reviewer is a thin lens that *applies the same rules* to a diff. It does not re-derive taste; -it checks against a list. - -Our verifier is the same. In this repo the "rules" are already deterministic code: - -- `createResearcherValidator` (`src/profiles/researcher.ts:235`) — citation-density floor + namespace - check, **no LLM**. -- the `lint` / `validate --strict` CLI path — citation, link, and schema checks over markdown, **no - LLM** (`docs/architecture.md`, "The CLI ... does not call an LLM"). -- `assessAuthoredProfile` / `profileRichnessFinding` in supervisor-lab's bench — a deterministic - "is this profile a stub" gate. - -So the verifier profile should be **thin and cheap**: it mostly *calls the deterministic checker and -reports the verdict*, escalating to an LLM judge **only** for the residual a deterministic check can't -cover (does the cited source actually support the claim — a semantic check). A heavy verifier that -re-reasons every claim from scratch is wasted spend: the dedup already happened in the validator. -**Spend the model budget on the worker (generation is hard); keep the verifier a cheap lens (judging -against a list is easy).** - ---- - -## Sketch 1 — `grounded-researcher` (leaf worker) - -**Status: already exists, twice.** `agent-knowledge` ships `researcherProfile()` -(`src/profiles/researcher.ts:130`) — `tools: { web_search, fs, shell }`, propose-don't-apply, with a -matching `createResearcherValidator`. `supervisor-lab` ships -`profiles/research/grounded-researcher.ts` (web-search MCP, search→cite→synthesize). This sketch is -the **canonical shape both converge on** — build new only if you need a variant; otherwise -`mergeAgentProfiles` over the existing one. - -**When to use:** one self-contained research question, spawned one-per-sub-topic by a driver. The -leaf — it does not spawn. - -**Tools / resources:** `web_search` (the `tcloud mcp` stdio server, or the `web_search: true` tool), -`fs` for writing the cited dossier, `shell` for `agent-knowledge index/lint`. - -**System prompt (the contract):** - -``` -You are a grounded research WORKER. You answer ONE question, and every load-bearing -claim is traceable to a real source you actually retrieved. - -THE LOOP — search → cite → synthesize: -1. SEARCH issue multiple focused web_search queries; never stop at the first hit. -2. CITE attach every load-bearing claim to its source (title + url/identifier). - A claim you cannot cite is a claim you do NOT make. Never invent a source. -3. SYNTH lead with the answer; note where sources agree, conflict, or leave a gap. - -NON-NEGOTIABLE: -- Honest "the evidence is thin / conflicting" beats a confident hallucination. -- Citation density floor is enforced downstream — under-citing is a hard fail, not a warning. -- You are the LEAF. You do not spawn sub-workers. - -Return your cited synthesis as the settled output (a structured object if a schema was passed). -``` - -**Why this shape:** the validator (`createResearcherValidator`) checks citation density and namespace -*deterministically*, so the prompt's job is only to make the worker *produce* citable output — the -checking is not the worker's job. That split is what keeps the verifier cheap (Sketch 4). - ---- - -## Sketch 2 — `verifier-driver` (the gate, NOT the judge) - -**Status: composes existing primitives.** This is `agent-runtime`'s `verify({ implement, verifier })` -combinator (`src/runtime/personify/combinators.ts:333`) wired to a research worker + a thin checker. -**Do not write a new verify-loop** — the 2-node implement→gate already exists. - -**When to use:** when "did the worker actually deliver" must be proven before the result counts — -i.e. always, on a graded run. A driver that runs ONE worker behind ONE gate. - -**Tools:** the coordination verbs (`spawn_agent`, `await_event`, `steer_agent`) over a `Scope`, plus -the ability to call the deterministic checker (`agent-knowledge lint` / `createResearcherValidator`). - -**System prompt (the contract):** - -``` -You are a VERIFIER-DRIVER. You spawn ONE research worker, then you GATE its output — -you never accept the worker's own say-so. - -THE GATE — generate → check → decide: -1. SPAWN author a grounded-researcher worker for the question and spawn it. -2. AWAIT pull its settled output (await_event). Read the REAL output, not a summary. -3. CHECK run the DETERMINISTIC checker first (citation density, namespace, lint, - schema). This is cheap and catches most failures. Only if it PASSES do you - escalate to the one semantic question a checker can't answer: - "does each cited source actually support the claim it's attached to?" -4. DECIDE PASS → settle. FAIL → steer_agent with the SPECIFIC gap named - ("claim X cites source Y but Y does not say X"), or respawn a sharper worker. - -NON-NEGOTIABLE: -- selector != judge. You GATE (pass/fail against a contract); you do not re-rank or - rewrite the worker's content yourself. -- "Settled a file" is not delivery. The deterministic check passing is delivery. -- Default to FAILING. A pass is earned by surviving the checker, not by looking plausible. -``` - -**Why this shape:** the driver's intelligence is *deciding what to check and how to steer on a -fail*, not re-doing the research. The expensive judgment (does the source support the claim) is gated -behind the cheap deterministic checker, so on most runs the LLM-judge step never fires. - ---- - -## Sketch 3 — `research-supervisor` (fan-out → fuse → recurse) - -**Status: exists as `research-then-build-driver` + `parallel-fanout-supervisor` in supervisor-lab.** -This sketch is the **research-specialized merge** of those two — build new only as a thin overlay via -`mergeAgentProfiles`; the orchestration body is identical. - -**When to use:** a research question too broad for one worker — needs decomposition into disjoint -sub-topics, parallel grounded workers, and a fused cited synthesis. The top of a research tree. - -**Tools / resources:** coordination MCP (`spawn_agent` / `await_event` / `steer_agent` / -`answer_question` / `stop`) over a `Scope`; `web_search` to *ground the decomposition itself*; skills -`dynamic-workflows`, `orchestrating-workers`, `authoring-agent-profiles` (all already in -`supervisor-lab/skills/`). - -**System prompt (the contract):** - -``` -You are a research SUPERVISOR. You do NOT research the topic yourself — you ground the -decomposition, fan out grounded-researcher workers, gate them, and FUSE their cited findings. - -THE SHAPE — ground → fan-out → gate → fan-in: -1. GROUND web_search FIRST to see what actually exists. Let the real landscape, not - your priors, define the sub-topics. Cite what you find. -2. DECOMPOSE split into DISJOINT sub-questions — non-overlapping ownership. -3. FAN OUT author a grounded-researcher profile per sub-question; spawn ALL of them in - ONE wave before awaiting any. Vary their angles. Never spawn-await-spawn. -4. GATE drain await_event. For each settled worker, run the deterministic checker - (Sketch 4). REJECT uncited or unsupported claims — a finding without a - source is not a finding. Steer or respawn on a fail. -5. FAN IN author a FINAL synthesis worker that fuses ONLY the gated findings into one - cited answer. Never paste raw worker dumps. Check each result for failure first. -6. RECURSE if a sub-question is itself large, spawn a SUB-supervisor (this profile + - the coordination MCP) to own it. - -NON-NEGOTIABLE: -- Author EACH worker as a FULL profile (name, description, prompt, model, tools, skills). - A 2-sentence prompt with no tools is a stub, not a worker — the richness gate will flag it. -- Never go blind: keep pulling until every spawned worker is terminal. -- Settle only on gated, cited output. Budget is conserved and depth is bounded by the Scope. -``` - -**Why this shape:** the supervisor's only real lever is *the quality of the profiles it authors* — -that is the capability `supervisor-lab` measures (`thinProfileRatio`, -`bench/supervise-topology.ts`). The gate (step 4) reuses the cheap verifier so the supervisor doesn't -burn its budget re-judging. - ---- - -## Sketch 4 — `dedup-verifier` (thin, cheap, mostly deterministic) - -**This is the emilkowalski dedup lesson made into a profile.** It is deliberately *thin* — the -"taste" lives in the deterministic validator, the profile is just the lens that applies it and -escalates only the residual. - -**When to use:** called by a driver/supervisor at a gate (Sketch 2 step 3, Sketch 3 step 4). Never -auto-invoked — the verifier equivalent of `disable-model-invocation: true`. - -**Tools:** `shell` (to run `agent-knowledge lint` / `validate --strict`), the -`createResearcherValidator` call, and an LLM *only* for the one semantic check. **No `web_search`, no -`fs` write, no generation tools** — a verifier that can edit content is a verifier that can cheat. - -**System prompt (the whole thing — it's short on purpose):** - -``` -You are a VERIFIER. You judge ONE research output against a fixed contract. You default -to FAILING; a pass is earned. You never edit, research, or rewrite — you only judge. - -ORDER (cheapest check first, stop at the first hard fail): -1. DETERMINISTIC run the validator/lint: citation density >= floor, namespace match, - links resolve, schema valid. If any fails → FAIL with the rule name. - ~90% of failures die here, at zero LLM cost. -2. SEMANTIC ONLY if (1) passes: for each cited claim, does the cited source - actually support it? This is the one thing a checker can't do. Be terse. - -OUTPUT: a findings table (claim | source | supported? | why) + a verdict (PASS / FAIL), -grouped by severity. No prose. A violation is a finding. - -You carry NO generation ability by design. If you find yourself wanting to fix the -output, STOP — that is the worker's job, not yours. -``` - -**Why thin beats heavy here:** the dedup already happened in `createResearcherValidator` and the lint -path. A heavy verifier that re-reasons every claim with an LLM pays full model cost to re-derive a -check the validator does for free, and (worse) a verifier with generation tools can drift into doing -the work and grading itself. Thin + deterministic-first is both cheaper *and* harder to game. Measured -claim to test when built: on a real run, the LLM (step 2) should fire on **< ~10–20% of gated outputs** -because the deterministic check (step 1) absorbs the rest — if it fires on most, the validator floor -is mis-set, not the verifier. - ---- - -## supervisor-lab — status and port plan - -**It exists.** `~/code/supervisor-lab` is a real, non-trivial repo -(`github.com/tangle-network/supervisor-lab`), not a stub. `ls ~/code | grep -i supervisor` → it's -there. - -It already is what one might propose to "create": the product/experiment layer for supervisor agents -over the `agent-runtime` substrate, dependency-one-way (`supervisor-lab → agent-runtime → -agent-eval`). It ships: - -- **the two-agent loop, already ported** — `bench/supervise-topology.ts` is exactly the "two-agent - loop": a supervisor mounts the coordination MCP over a live `Scope`/`Supervisor` - (`createSupervisor`, `serveCoordinationMcp`), authors worker `AgentProfile`s as code, spawns them, - drains the bus (`await_event`), steers (`steer_agent`), and grades on a real judge. It has an - **OFFLINE $0 scripted path** and a LIVE cli-bridge path. This is the loop to plug these sketches - into — it does not need to be built. -- **profile archetypes** — `profiles/research/{grounded-researcher,research-then-build-driver}.ts`, - `profiles/engineering/{parallel-fanout-supervisor,long-horizon-plan-driver,hardware-auditor}.ts`. - Sketches 1 and 3 above are *already these files*. -- **a skill layer** — `skills/{authoring-agent-profiles,orchestrating-workers,dynamic-workflows, - spawning-research-loops}/SKILL.md`, each in the exact emilkowalski frontmatter+rules shape. -- **a catalog + ingest pipeline** — `src/catalog/`, `src/ingest/skills.ts` ingests vendored - Claude-Code-style skill packs (the same shape as `emilkowalski/skills`) by convention. -- **the richness gate** — `assessAuthoredProfile` / `profileRichnessFinding` / `thinProfileRatio`, - the deterministic "is this a stub" check that is the dedup-verifier's first line. - -### How to port the two-agent loop (it's already there — this is how to extend it) - -Because the loop already exists, "porting" means **adding these four profiles + the thin-verifier gate -into the existing harness**, not rebuilding it: - -1. **Land the profiles as catalog files.** Sketches 1 and 3 are already - `profiles/research/grounded-researcher.ts` and `research-then-build-driver.ts`. Sketch 2 - (`verifier-driver`) and Sketch 4 (`dedup-verifier`) are new files under - `profiles/research/` — author them as `defineAgentProfile` entries, resolve their skills via the - `skills(...)` helper in `profiles/_shared.ts` (fails closed on an unknown skill). - -2. **Wire the dedup-verifier into the gate.** In `bench/supervise-topology.ts`, the worker's - `runWorker` already grades on `adapter.judge`. For research tasks, replace/augment that with the - deterministic `createResearcherValidator` + `agent-knowledge lint` path FIRST, and only escalate to - the LLM semantic check on a deterministic pass. This is the dedup point made executable: the - existing `assessAuthoredProfile` richness gate is the *profile* check; the validator is the - *output* check. Both run before any LLM judge. - -3. **Add a research bench arm.** `supervise-topology.ts` auto-detects domain (repo vs. answer/text). - A research arm is an "answer/text" task whose `adapter.judge` is the cited-output validator. Add - `BENCH=research` that loads a research question + namespace and grades via - `createResearcherValidator`. The OFFLINE scripted path already proves the harness at $0; extend - `scriptedRun.workerResult` to emit a cited/uncited synthesis so the offline smoke exercises the - verifier without spend. - -4. **Run the existing experiment knob.** The whole point of the lab is the one knob: catalog/skills - ON vs. OFF (`CATALOG=1`). With these profiles + the thin verifier in the catalog, run - capability-aware vs. baseline on a hard research task and read `thinProfileRatio` + delivery rate — - does giving the supervisor the research archetypes + the cheap gate produce better-composed teams - and more *cited, gated* deliveries. - -### Cross-repo note - -`agent-knowledge` already owns the deterministic research checker (`createResearcherValidator`, the -`lint`/`validate` CLI) and the researcher profile (`researcherProfile`, -`multiHarnessResearcherFanout`). The thin verifier should **call those**, not reimplement them — -`supervisor-lab` depends on `agent-runtime`, and the research checks live in `agent-knowledge`, so the -research bench arm pulls `@tangle-network/agent-knowledge` for the verifier's deterministic line. That -keeps the dedup in one place: the validator is authored once, the verifier profile is a thin lens over -it. diff --git a/docs/two-agent-research-ab.md b/docs/two-agent-research-ab.md index ce3e29e..925e8db 100644 --- a/docs/two-agent-research-ab.md +++ b/docs/two-agent-research-ab.md @@ -101,10 +101,8 @@ proposed. It talks to the router directly through `createTangleRouterClient` — claude-code / opencode / sandbox harness, and no dynamic harness selection. The driver (`createVerifyingResearchDriver`) is one glm-5.2 chat call per source. -The repo *does* ship a real `AgentProfile` for research (`researcherProfile`), and -the **offline** control arm uses it with a stub harness — but the live arm bypasses -it for the direct pipeline. This is a deliberate shortcut (no harness to stand up, -~$0.20 to run) and also the loop's main simplification debt; see §7. +The package keeps the live research mechanics runner-agnostic. +Agent profiles and sandbox-backed runners now live in `agent-runtime`; this package owns the source, write, validation, and eval layer. ### 2.3 Equal compute @@ -378,7 +376,7 @@ measured; this is what changed. What is still **not** built remains the worker: the live worker is a ~500-line hand-wired pipeline (query-gen, search, fetch, propose) against the router directly, -where the repo's own pattern is to *author* an `AgentProfile` (`researcherProfile`) +where the runtime integration's pattern is to author an agent profile and run it on a harness with a web-search tool — reusable and harness-agnostic. The direct pipeline is cheaper to run today (no harness, no creds beyond the router) but it is the loop's main remaining piece of duplication, and the obvious next step if this diff --git a/package.json b/package.json index d6c4557..808007d 100644 --- a/package.json +++ b/package.json @@ -39,16 +39,6 @@ "import": "./dist/sources/index.js", "default": "./dist/sources/index.js" }, - "./profiles": { - "types": "./dist/profiles/index.d.ts", - "import": "./dist/profiles/index.js", - "default": "./dist/profiles/index.js" - }, - "./autodata": { - "types": "./dist/autodata/index.d.ts", - "import": "./dist/autodata/index.js", - "default": "./dist/autodata/index.js" - }, "./benchmarks": { "types": "./dist/benchmarks/index.d.ts", "import": "./dist/benchmarks/index.js", @@ -75,12 +65,10 @@ "test:watch": "vitest", "typecheck": "tsc --noEmit", "lint": "biome check src tests", - "format": "biome format --write src tests", - "autodata": "tsx src/autodata/run.ts" + "format": "biome format --write src tests" }, "dependencies": { "@tangle-network/agent-eval": "^0.107.0", - "@tangle-network/agent-runtime": "^0.89.0", "zod": "^4.3.6" }, "devDependencies": { @@ -97,7 +85,6 @@ "minimumReleaseAge": 4320, "minimumReleaseAgeExclude": [ "@tangle-network/agent-eval", - "@tangle-network/agent-runtime", "@tangle-network/sandbox" ] }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6aaf726..1b24c9d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,9 +11,6 @@ importers: '@tangle-network/agent-eval': specifier: ^0.107.0 version: 0.107.0(typescript@5.9.3) - '@tangle-network/agent-runtime': - specifier: ^0.89.0 - version: 0.89.0(@tangle-network/agent-eval@0.107.0(typescript@5.9.3))(@tangle-network/agent-interface@0.17.1)(@tangle-network/sandbox@0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3))) zod: specifier: ^4.3.6 version: 4.4.2 @@ -26,7 +23,7 @@ importers: version: 0.4.0 '@tangle-network/sandbox': specifier: ^0.9.7 - version: 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3)) + version: 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.2)) '@types/node': specifier: ^25.6.0 version: 25.6.0 @@ -634,23 +631,6 @@ packages: '@tangle-network/agent-interface@0.17.1': resolution: {integrity: sha512-B7dRJTo0HSUtgBCB1VMwkTFYkLUaRr/4BcRglrQuGhGUwOzKv1RYyMejOVh5M3a5AagY9N79f7GYbjcA3UmnIA==} - '@tangle-network/agent-runtime@0.89.0': - resolution: {integrity: sha512-tjQ/uJORuLOAE/E99w0NxWb5O0uP/nwescMRyIqxsqLXzry7WhNh9DK/BHKTEfb6XMvcIaT92J/3T1ZJTtXZ9Q==} - engines: {node: '>=20'} - hasBin: true - peerDependencies: - '@tangle-network/agent-eval': '>=0.101.0 <1.0.0' - '@tangle-network/agent-interface': '>=0.14.0 <1.0.0' - '@tangle-network/sandbox': '>=0.8.0 <1.0.0' - playwright: ^1.40.0 - peerDependenciesMeta: - '@tangle-network/agent-interface': - optional: true - '@tangle-network/sandbox': - optional: true - playwright: - optional: true - '@tangle-network/sandbox@0.9.7': resolution: {integrity: sha512-9pCwJ5MlF7RUpp0AQKQDFyR0yu+E0udEhWkqhrlb/RuoJxlt72zVPuzO4FnMb1MZTkfjStmomC3k5xQyqi1YSA==} peerDependencies: @@ -1539,19 +1519,12 @@ snapshots: dependencies: zod: 4.4.3 - '@tangle-network/agent-runtime@0.89.0(@tangle-network/agent-eval@0.107.0(typescript@5.9.3))(@tangle-network/agent-interface@0.17.1)(@tangle-network/sandbox@0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3)))': - dependencies: - '@tangle-network/agent-eval': 0.107.0(typescript@5.9.3) - optionalDependencies: - '@tangle-network/agent-interface': 0.17.1 - '@tangle-network/sandbox': 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3)) - - '@tangle-network/sandbox@0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3))': + '@tangle-network/sandbox@0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.2))': dependencies: '@tangle-network/agent-core': 0.3.8 '@tangle-network/agent-interface': 0.13.0 optionalDependencies: - viem: 2.48.8(typescript@5.9.3)(zod@4.4.2) + viem: 2.48.8(typescript@5.9.3)(zod@4.4.3) '@tangle-network/tcloud-attestation@0.1.1': {} @@ -1559,10 +1532,10 @@ snapshots: dependencies: '@scure/bip32': 2.2.0 '@scure/bip39': 2.2.0 - '@tangle-network/sandbox': 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3)) + '@tangle-network/sandbox': 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.2)) '@tangle-network/tcloud-attestation': 0.1.1 commander: 14.0.3 - viem: 2.48.8(typescript@5.9.3)(zod@4.4.2) + viem: 2.48.8(typescript@5.9.3)(zod@4.4.3) transitivePeerDependencies: - '@mastra/core' - '@modelcontextprotocol/sdk' @@ -1628,10 +1601,10 @@ snapshots: loupe: 3.2.1 tinyrainbow: 2.0.0 - abitype@1.2.3(typescript@5.9.3)(zod@4.4.2): + abitype@1.2.3(typescript@5.9.3)(zod@4.4.3): optionalDependencies: typescript: 5.9.3 - zod: 4.4.2 + zod: 4.4.3 acorn@8.16.0: {} @@ -1802,7 +1775,7 @@ snapshots: dependencies: yaml: 2.8.4 - ox@0.14.20(typescript@5.9.3)(zod@4.4.2): + ox@0.14.20(typescript@5.9.3)(zod@4.4.3): dependencies: '@adraffy/ens-normalize': 1.11.1 '@noble/ciphers': 1.3.0 @@ -1810,7 +1783,7 @@ snapshots: '@noble/hashes': 1.8.0 '@scure/bip32': 1.7.0 '@scure/bip39': 1.6.0 - abitype: 1.2.3(typescript@5.9.3)(zod@4.4.2) + abitype: 1.2.3(typescript@5.9.3)(zod@4.4.3) eventemitter3: 5.0.1 optionalDependencies: typescript: 5.9.3 @@ -1973,15 +1946,15 @@ snapshots: undici-types@7.19.2: {} - viem@2.48.8(typescript@5.9.3)(zod@4.4.2): + viem@2.48.8(typescript@5.9.3)(zod@4.4.3): dependencies: '@noble/curves': 1.9.1 '@noble/hashes': 1.8.0 '@scure/bip32': 1.7.0 '@scure/bip39': 1.6.0 - abitype: 1.2.3(typescript@5.9.3)(zod@4.4.2) + abitype: 1.2.3(typescript@5.9.3)(zod@4.4.3) isows: 1.0.7(ws@8.18.3) - ox: 0.14.20(typescript@5.9.3)(zod@4.4.2) + ox: 0.14.20(typescript@5.9.3)(zod@4.4.3) ws: 8.18.3 optionalDependencies: typescript: 5.9.3 diff --git a/src/autodata/build-dataset.ts b/src/autodata/build-dataset.ts deleted file mode 100644 index 9f6cd68..0000000 --- a/src/autodata/build-dataset.ts +++ /dev/null @@ -1,214 +0,0 @@ -/** - * Build a discriminative QA dataset from a real source document with REAL two-tier solvers. - * - * Grounds on the document (or takes an already-grounded excerpt), runs `createDataCreationLoop` with - * the live router roles, writes the accepted examples as JSONL, and returns the numbers the - * calibration depends on: the per-example strong/weak gap for the LOOP-ACCEPTED (agentic) examples - * AND for the challenger's FIRST drafts (plain), plus the cost ledger split by role. - */ - -import { appendFile, mkdir, rm, writeFile } from 'node:fs/promises' -import { dirname } from 'node:path' -import { CostLedger } from '@tangle-network/agent-eval' -import { - type AttemptRecord, - createDataCreationLoop, - discriminativeAcceptRule, - type ExampleEvaluation, -} from './data-creation-loop' -import { type GroundedDoc, groundDoc } from './grounding' -import { buildAutodataRoles, type ChallengerStyle, type RouterCallRecord } from './router-roles' - -export interface DiscriminativeThresholds { - minStrong?: number - maxWeak?: number - minGap?: number -} - -export interface AutodataDatasetConfig { - apiKey: string - baseUrl?: string - /** A grounded excerpt, or a spec to fetch + chunk one from a real URL. */ - source: GroundedDoc | { url: string; focus?: string; cacheDir?: string } - /** Where to write the JSONL dataset. */ - outPath: string - target?: number - samples?: number - maxRetries?: number - thresholds?: DiscriminativeThresholds - models?: { challenger?: string; weak?: string; strong?: string; judge?: string } - /** Challenger prompt: 'causal' (non-extractive, default) or 'recall' (the calibration baseline). */ - style?: ChallengerStyle - /** Where to write the per-attempt autopsy JSONL (every candidate, accepted or rejected). */ - attemptsPath?: string - signal?: AbortSignal -} - -/** One accepted, discriminating training example with its real strong/weak scores + provenance. */ -export interface DatasetRow { - context: string - question: string - reference: string - rubric: readonly string[] - weakScore: number - strongScore: number - gap: number - source: { url: string; headingPath: string; chunkIndex: number } -} - -export interface AutodataDatasetResult { - source: GroundedDoc - accepted: ExampleEvaluation[] - rows: DatasetRow[] - /** Mean strong−weak gap on the challenger's first-draft (plain-generation) questions. */ - plainGapMean: number | null - /** Mean strong−weak gap on the loop-accepted (agentic) questions. */ - agenticGapMean: number | null - /** Mean of the BEST gap the refinement reached per slot (accepted or not) — the informative - * comparison against `plainGapMean` even when nothing clears the accept bar. */ - refinedGapMean: number | null - plainGaps: number[] - agenticGaps: number[] - refinedGaps: number[] - /** Every evaluated candidate (accepted or rejected) with both solvers' answers — the autopsy trail. */ - attempts: AttemptRecord[] - cost: CostLedger - costPerExampleUsd: number | null - /** How many router calls were priced by the router vs rate-estimated. */ - callProvenance: { router: number; estimated: number } - outPath: string - /** Where the per-attempt autopsy JSONL was written (null if not requested). */ - attemptsPath: string | null -} - -function mean(xs: number[]): number | null { - return xs.length === 0 ? null : xs.reduce((a, b) => a + b, 0) / xs.length -} - -function isGrounded(s: AutodataDatasetConfig['source']): s is GroundedDoc { - return typeof (s as GroundedDoc).doc === 'string' -} - -/** The causal (default) user instruction — pairs with the non-extractive challenger system prompt. */ -function causalInstruction(doc: string): string { - return ( - `SOURCE DOCUMENT EXCERPT:\n\n${doc}\n\n` + - `Write ONE hard CAUSAL / COMPARATIVE / MECHANISM / THESIS-CONSISTENCY question grounded in this ` + - `excerpt — never a recall / lookup / definition. The CONTEXT must give the solver the premises ` + - `but MUST NOT state the answer; the answer has to be DERIVED. Return STRICT JSON: ` + - `{"context": string, "question": string, "reference": string, "rubric": string[] }.` - ) -} - -/** The recall (baseline) user instruction — pairs with the extractive challenger; for calibration. */ -function recallInstruction(doc: string): string { - return ( - `SOURCE DOCUMENT EXCERPT:\n\n${doc}\n\n` + - `Write ONE exam question grounded in this excerpt, with a short context excerpt the question is ` + - `answerable from, a reference answer, and a 2-3 item rubric. Return STRICT JSON: ` + - `{"context": string, "question": string, "reference": string, "rubric": string[] }.` - ) -} - -function instructionFor(style: ChallengerStyle): (doc: string) => string { - return style === 'recall' ? recallInstruction : causalInstruction -} - -/** Run the full pipeline: ground → loop → JSONL. Returns the calibration numbers + cost. */ -export async function buildAutodataDataset( - config: AutodataDatasetConfig, -): Promise { - const source = isGrounded(config.source) - ? config.source - : await groundDoc({ - url: config.source.url, - focus: config.source.focus, - cacheDir: config.source.cacheDir, - signal: config.signal, - }) - - const provenance = { router: 0, estimated: 0 } - const onCall = (rec: RouterCallRecord): void => { - if (rec.costSource === 'router') provenance.router += 1 - else provenance.estimated += 1 - } - - const ledger = new CostLedger() - const style: ChallengerStyle = config.style ?? 'causal' - - const roles = buildAutodataRoles({ - apiKey: config.apiKey, - baseUrl: config.baseUrl, - challengerModel: config.models?.challenger, - weakModel: config.models?.weak, - strongModel: config.models?.strong, - judgeModel: config.models?.judge, - challengerStyle: style, - ledger, - onCall, - }) - - // Per-attempt autopsy trail: every candidate (accepted or rejected) is appended as one JSONL row - // with both solvers' answer text + scores, so a null is diagnosable from the raw answers. - const attempts: AttemptRecord[] = [] - const attemptsPath = config.attemptsPath ?? null - if (attemptsPath) { - await mkdir(dirname(attemptsPath), { recursive: true }) - await rm(attemptsPath, { force: true }) - } - const onAttempt = async (rec: AttemptRecord): Promise => { - attempts.push(rec) - if (attemptsPath) await appendFile(attemptsPath, `${JSON.stringify({ ...rec, style })}\n`) - } - - const result = await createDataCreationLoop({ - doc: source.doc, - baseInstruction: instructionFor(style), - challenger: roles.challenger, - weakSolver: roles.weakSolver, - strongSolver: roles.strongSolver, - judge: roles.judge, - accept: (i) => discriminativeAcceptRule({ ...i, ...config.thresholds }), - target: config.target ?? 3, - samples: config.samples ?? 3, - maxRetries: config.maxRetries ?? 4, - cost: ledger, - onAttempt, - signal: config.signal, - }) - - const rows: DatasetRow[] = result.accepted.map((ex) => ({ - context: ex.example.context, - question: ex.example.question, - reference: ex.example.reference, - rubric: ex.example.rubric, - weakScore: ex.weakScore, - strongScore: ex.strongScore, - gap: ex.gap, - source: { url: source.url, headingPath: source.headingPath, chunkIndex: source.chunkIndex }, - })) - - await mkdir(dirname(config.outPath), { recursive: true }) - await writeFile( - config.outPath, - rows.map((r) => JSON.stringify(r)).join('\n') + (rows.length ? '\n' : ''), - ) - - return { - source, - accepted: result.accepted, - rows, - plainGapMean: mean(result.plainGaps), - agenticGapMean: mean(result.agenticGaps), - refinedGapMean: mean(result.refinedGaps), - plainGaps: result.plainGaps, - agenticGaps: result.agenticGaps, - refinedGaps: result.refinedGaps, - attempts, - cost: result.cost, - costPerExampleUsd: result.cost.costPerCompletedTask(), - callProvenance: provenance, - outPath: config.outPath, - attemptsPath, - } -} diff --git a/src/autodata/calibrate.ts b/src/autodata/calibrate.ts deleted file mode 100644 index b024aa7..0000000 --- a/src/autodata/calibrate.ts +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Autodata calibration: does the non-extractive CAUSAL challenger widen the strong/weak gap vs the - * old RECALL challenger, on the SAME grounded document? This is the lever's proof — the prior null - * was "recall question → answer leaks into context → an 8B reads it out → gap ~0". If the fix works, - * the causal arm's mean gap (and accepted count) clears the recall arm's by a clear margin. - * - * Run (key never printed): - * dotenvx run -f /home/drew/company/devops/secrets/agent-state.env -- \ - * pnpm tsx src/autodata/calibrate.ts - * - * Env knobs: AUTODATA_URL, AUTODATA_FOCUS, AUTODATA_TARGET, AUTODATA_SAMPLES, AUTODATA_MAXRETRIES, - * AUTODATA_{WEAK,STRONG,CHALLENGER,JUDGE}_MODEL, TANGLE_API_KEY. - */ - -import { type AutodataDatasetResult, buildAutodataDataset } from './build-dataset' -import type { AttemptRecord } from './data-creation-loop' -import { DEFAULT_SOURCE_URL, groundDoc } from './grounding' -import { - CHALLENGER_MODEL, - type ChallengerStyle, - JUDGE_MODEL, - STRONG_SOLVER_MODEL, - smokeTestModels, - WEAK_SOLVER_MODEL, -} from './router-roles' - -function envInt(name: string, fallback: number): number { - const raw = process.env[name] - if (!raw) return fallback - const n = Number.parseInt(raw, 10) - if (!Number.isFinite(n) || n <= 0) throw new Error(`${name}='${raw}' is not a positive integer`) - return n -} - -function mean(xs: number[]): number | null { - return xs.length === 0 ? null : xs.reduce((a, b) => a + b, 0) / xs.length -} - -function fmt(x: number | null, d = 3): string { - return x === null ? 'n/a' : x.toFixed(d) -} - -/** Mean strong/weak gap over the quality-clean attempts of one arm — the discriminating power. */ -function armGap(attempts: AttemptRecord[]): number | null { - return mean(attempts.filter((a) => a.qualityOk).map((a) => a.gap)) -} - -async function runArm(args: { - apiKey: string - source: Awaited> - style: ChallengerStyle - target: number - samples: number - maxRetries: number -}): Promise { - return buildAutodataDataset({ - apiKey: args.apiKey, - source: args.source, - outPath: `data/autodata-calib-${args.style}.jsonl`, - attemptsPath: `data/autodata-calib-${args.style}-attempts.jsonl`, - style: args.style, - target: args.target, - samples: args.samples, - maxRetries: args.maxRetries, - }) -} - -async function main(): Promise { - const apiKey = process.env.TANGLE_API_KEY ?? process.env.TANGLE_ROUTER_KEY - if (!apiKey) throw new Error('no TANGLE_API_KEY in env — run under dotenvx so the key is set') - - const url = process.env.AUTODATA_URL ?? DEFAULT_SOURCE_URL - const focus = process.env.AUTODATA_FOCUS ?? 'attention' - const target = envInt('AUTODATA_TARGET', 2) - const samples = envInt('AUTODATA_SAMPLES', 2) - const maxRetries = envInt('AUTODATA_MAXRETRIES', 2) - - console.log('Autodata calibration · recall vs causal challenger (same doc)\n') - console.log( - ` challenger/judge=${CHALLENGER_MODEL}/${JUDGE_MODEL} weak=${WEAK_SOLVER_MODEL} strong=${STRONG_SOLVER_MODEL}`, - ) - - const smoke = await smokeTestModels({ - apiKey, - models: [CHALLENGER_MODEL, WEAK_SOLVER_MODEL, STRONG_SOLVER_MODEL], - }) - const dead = smoke.filter((s) => !s.ok) - if (dead.length > 0) - throw new Error(`cost gate failed — empty from: ${dead.map((d) => d.model).join(', ')}`) - console.log(' cost gate ok (all models returned content)\n') - - const source = await groundDoc({ url, focus }) - console.log(`Grounded on ${source.url} section='${source.headingPath}'\n`) - - // Same doc, same budget, only the challenger prompt changes — a clean A/B on the lever. - const recall = await runArm({ apiKey, source, style: 'recall', target, samples, maxRetries }) - const causal = await runArm({ apiKey, source, style: 'causal', target, samples, maxRetries }) - - const recallGap = armGap(recall.attempts) - const causalGap = armGap(causal.attempts) - - console.log('— Calibration result —') - console.log( - ` RECALL attempts=${recall.attempts.length} mean gap=${fmt(recallGap)} accepted=${recall.accepted.length}`, - ) - console.log( - ` CAUSAL attempts=${causal.attempts.length} mean gap=${fmt(causalGap)} accepted=${causal.accepted.length}`, - ) - if (recallGap !== null && causalGap !== null) { - const delta = causalGap - recallGap - console.log( - ` Δ (causal − recall) = ${delta >= 0 ? '+' : ''}${delta.toFixed(3)} ` + - (delta >= 0.1 - ? '→ the causal challenger WIDENS the gap (the lever works)' - : '→ no meaningful widening from the causal challenger (honest null)'), - ) - } - - const totalUsd = recall.cost.summary().totalCostUsd + causal.cost.summary().totalCostUsd - console.log(`\n spend: $${totalUsd.toFixed(4)} (both arms)`) - console.log(` trails: ${recall.attemptsPath} ${causal.attemptsPath}`) -} - -main().catch((err) => { - console.error(err) - process.exit(1) -}) diff --git a/src/autodata/data-creation-loop.test.ts b/src/autodata/data-creation-loop.test.ts deleted file mode 100644 index ec42325..0000000 --- a/src/autodata/data-creation-loop.test.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - type AttemptRecord, - createDataCreationLoop, - discriminativeAcceptRule, - qualityCheck, -} from './data-creation-loop' -import { - baseInstruction, - buildRubricJudge, - challengerClient, - groundingDoc, - solverClient, -} from './offline-fixtures' -import { parseDataExample } from './router-roles' - -describe('discriminativeAcceptRule (the new piece)', () => { - it('accepts an example that separates strong from weak', () => { - const d = discriminativeAcceptRule({ strongScore: 0.77, weakScore: 0.46 }) - expect(d.accept).toBe(true) - expect(d.reason).toContain('discriminates') - }) - - it('rejects "too easy" when the weak solver passes', () => { - const d = discriminativeAcceptRule({ strongScore: 0.86, weakScore: 0.84 }) - expect(d.accept).toBe(false) - expect(d.reason).toContain('too easy') - }) - - it('rejects "too hard" when even the strong solver misses', () => { - const d = discriminativeAcceptRule({ strongScore: 0.55, weakScore: 0.3 }) - expect(d.accept).toBe(false) - expect(d.reason).toContain('too hard') - }) - - it('rejects when the gap is below minGap even if both thresholds hold', () => { - const d = discriminativeAcceptRule({ strongScore: 0.66, weakScore: 0.48 }) - expect(d.accept).toBe(false) - expect(d.reason).toContain('not discriminative') - }) - - it('honors custom thresholds', () => { - const strict = discriminativeAcceptRule({ strongScore: 0.77, weakScore: 0.46, minGap: 0.4 }) - expect(strict.accept).toBe(false) - }) -}) - -describe('qualityCheck', () => { - it('rejects a reference that leaks verbatim into the context', () => { - const q = qualityCheck({ - context: 'The answer is 42 and nothing else matters.', - question: 'What is the answer?', - reference: 'The answer is 42', - rubric: ['a', 'b'], - }) - expect(q.ok).toBe(false) - expect(q.reason).toContain('leaked') - }) - - it('rejects a thin rubric', () => { - const q = qualityCheck({ context: 'c', question: 'q', reference: 'r', rubric: ['only one'] }) - expect(q.ok).toBe(false) - expect(q.reason).toContain('thin rubric') - }) - - it('passes a clean example', () => { - const q = qualityCheck({ - context: 'Some grounding context that does not contain the answer phrasing.', - question: 'Why does it matter?', - reference: 'Because of a distinct reasoning chain.', - rubric: ['states X', 'explains Y'], - }) - expect(q.ok).toBe(true) - }) -}) - -describe('parseDataExample (challenger JSON parsing)', () => { - it('parses a bare JSON object', () => { - const ex = parseDataExample('{"context":"c","question":"q","reference":"r","rubric":["a","b"]}') - expect(ex.question).toBe('q') - expect(ex.rubric).toHaveLength(2) - }) - - it('parses JSON wrapped in a ```json fence with surrounding prose', () => { - const ex = parseDataExample( - 'Sure, here is the example:\n```json\n{"context":"c","question":"q","reference":"r","rubric":["a","b"]}\n```\nDone.', - ) - expect(ex.reference).toBe('r') - }) - - it('throws loud when no JSON object is present', () => { - expect(() => parseDataExample('no json here')).toThrow() - }) - - it('throws loud when a required field is missing', () => { - expect(() => parseDataExample('{"context":"c","question":"q"}')).toThrow() - }) -}) - -describe('createDataCreationLoop (offline)', () => { - it('manufactures discriminating examples and separates plain from agentic gaps', async () => { - const result = await createDataCreationLoop({ - doc: groundingDoc, - baseInstruction, - challenger: challengerClient(), - weakSolver: solverClient('weak'), - strongSolver: solverClient('strong'), - judge: buildRubricJudge(), - target: 2, - samples: 3, - maxRetries: 4, - }) - - expect(result.accepted).toHaveLength(2) - for (const ex of result.accepted) { - expect(ex.decision.accept).toBe(true) - expect(ex.gap).toBeGreaterThanOrEqual(0.2) - } - - const mean = (xs: number[]) => xs.reduce((a, b) => a + b, 0) / xs.length - const plain = mean(result.plainGaps) - const agentic = mean(result.agenticGaps) - expect(plain).toBeLessThan(0.1) - expect(agentic).toBeGreaterThan(0.25) - expect(agentic - plain).toBeGreaterThanOrEqual(0.15) - - const stored = await result.corpus.query({ area: 'training-data' }) - expect(stored).toHaveLength(2) - expect(result.cost.summary().totalCostUsd).toBeGreaterThan(0) - }) - - it('emits a per-attempt record (with both solvers’ answers) for every candidate, accepted or rejected', async () => { - const attempts: AttemptRecord[] = [] - const result = await createDataCreationLoop({ - doc: groundingDoc, - baseInstruction, - challenger: challengerClient(), - weakSolver: solverClient('weak'), - strongSolver: solverClient('strong'), - judge: buildRubricJudge(), - target: 1, - samples: 3, - maxRetries: 4, - onAttempt: (rec) => { - attempts.push(rec) - }, - }) - - // The first slot's first draft is the EASY example (rejected "too easy"), then the fold steers - // to a HARD example (accepted) — so we observe at least one reject AND one accept. - expect(attempts.length).toBeGreaterThanOrEqual(2) - const rejected = attempts.filter((a) => !a.decision.accept) - const accepted = attempts.filter((a) => a.decision.accept) - expect(rejected.length).toBeGreaterThanOrEqual(1) - expect(accepted.length).toBeGreaterThanOrEqual(1) - expect(result.accepted).toHaveLength(1) - - // Every emitted attempt carries the raw answers + scores the autopsy needs. - for (const a of attempts) { - expect(a.weak.samples).toHaveLength(3) - expect(a.strong.samples).toHaveLength(3) - expect(typeof a.weak.samples[0]?.answer).toBe('string') - expect(a.gap).toBeCloseTo(a.strong.mean - a.weak.mean, 6) - } - - // The first attempt (iteration 0) is the plain draft and should NOT discriminate; a later - // attempt should — the fold widened the gap. - const plain = attempts.find((a) => a.iteration === 0) - expect(plain?.decision.accept).toBe(false) - expect(Math.max(...attempts.map((a) => a.gap))).toBeGreaterThan(plain?.gap ?? 0) - }) -}) diff --git a/src/autodata/data-creation-loop.ts b/src/autodata/data-creation-loop.ts deleted file mode 100644 index 4d026a7..0000000 --- a/src/autodata/data-creation-loop.ts +++ /dev/null @@ -1,595 +0,0 @@ -/** - * The Autodata / Agentic Self-Instruct INNER loop: an agent MANUFACTURES hard training examples - * from a grounding doc and keeps only the ones that DISCRIMINATE a strong solver from a weak one. - * - * PROVENANCE — this loop is vendored verbatim from agent-runtime - * `examples/agentic-data-creation/agentic-data-creation.ts` (branch - * `examples/agentic-data-creation`). It is an EXAMPLE in agent-runtime, not a published runtime - * export — examples are not shipped in the npm dist — so agent-knowledge cannot import it and - * vendors it here (the "copy with a note" path). Every primitive it COMPOSES is reused from the - * published packages, nothing is re-implemented: the judge is `llmJudge` (agent-eval), the loop - * kernel is `runLoop` (agent-runtime/loops), the store is `InMemoryCorpus` (agent-runtime/loops), - * the cost accounting is `CostLedger` (agent-eval). The REAL grounding (arXiv ingestion) + the REAL - * two-tier router solvers live in the sibling files; this file stays domain- and transport-agnostic. - * - * The whole method is four roles + one accept rule: - * 1. CHALLENGER writes a candidate {context, question, reference, rubric} from the doc. - * 2. WEAK solver and STRONG solver each attempt it, sampled N× to average out variance. - * 3. JUDGE scores every attempt against the rubric (one `llmJudge` call per attempt). - * 4. ACCEPT keeps the example ONLY IF it discriminates: strong >= hi, weak < lo, gap >= g — - * plus a quality check (no context leakage, a real rubric). - * On reject, the CHALLENGER driver FOLDS the reject reason into its next prompt and retries. - * Accepted examples accrete into a `Corpus`. - * - * The ONE genuinely new piece is `discriminativeAcceptRule` — the paper's reward, written as a - * small Validator-shaped accept/reject. It is a lift candidate for agent-eval (next to - * `blendHeldout` / `HeldOutGate`) if it proves out across real domains; it lives here until then. - */ - -import { CostLedger } from '@tangle-network/agent-eval' -import type { JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign' -import { - type AgentRunSpec, - type Corpus, - type CorpusRecord, - type Driver, - InMemoryCorpus, - type OutputAdapter, - runLoop, - type SandboxClient, - type Validator, -} from '@tangle-network/agent-runtime/loops' -import type { AgentProfile, SandboxEvent } from '@tangle-network/sandbox' - -// ── The four-role data shapes ───────────────────────────────────────────────────────────── - -/** One manufactured training example, grounded in `context` excerpted from the doc. */ -export interface DataExample { - /** The grounding excerpt the question is answerable from. */ - readonly context: string - readonly question: string - /** The reference answer the rubric is graded against. */ - readonly reference: string - /** Scoring criteria the judge applies (>= 2 for a usable example). */ - readonly rubric: readonly string[] -} - -/** What the judge scores: a solver's `answer` to one `example`. */ -export interface SolverArtifact { - readonly example: DataExample - readonly answer: string -} - -/** The accept rule's verdict — keep this example, and why (or why not). */ -export interface AcceptDecision { - readonly accept: boolean - readonly reason: string -} - -// ═══════════════════════════════════════════════════════════════════════════════════════════ -// THE ONE NEW PIECE — the paper's discriminative reward, as a small Validator-shaped rule. -// ═══════════════════════════════════════════════════════════════════════════════════════════ -// -// Autodata keeps an example ONLY IF it separates a strong solver from a weak one: the strong -// solver should mostly get it (>= minStrong), the weak solver should mostly miss it (< maxWeak), -// and the margin between them (the "gap") must clear minGap. That is the whole objective, so the -// rule is the LITERAL accept criterion, never softened. The three reject reasons map one-to-one -// onto the challenger's next-prompt fold. -export function discriminativeAcceptRule(input: { - /** Strong solver's mean rubric score, [0,1]. */ - strongScore: number - /** Weak solver's mean rubric score, [0,1]. */ - weakScore: number - /** Strong must reach at least this (else the example is unfair / too hard). Default 0.65. */ - minStrong?: number - /** Weak must stay strictly below this (else the example is too easy). Default 0.5. */ - maxWeak?: number - /** strong − weak must be at least this (else it does not discriminate). Default 0.2. */ - minGap?: number -}): AcceptDecision { - const { strongScore, weakScore } = input - const minStrong = input.minStrong ?? 0.65 - const maxWeak = input.maxWeak ?? 0.5 - const minGap = input.minGap ?? 0.2 - const gap = strongScore - weakScore - - if (strongScore < minStrong) { - return { - accept: false, - reason: `too hard: strong solver reached only ${pct(strongScore)} (< ${pct(minStrong)})`, - } - } - if (weakScore >= maxWeak) { - return { - accept: false, - reason: `too easy: weak solver reached ${pct(weakScore)} (>= ${pct(maxWeak)})`, - } - } - if (gap < minGap) { - return { accept: false, reason: `not discriminative: gap ${pct(gap)} (< ${pct(minGap)})` } - } - return { - accept: true, - reason: `discriminates: strong ${pct(strongScore)} >= ${pct(minStrong)}, weak ${pct(weakScore)} < ${pct(maxWeak)}, gap ${pct(gap)} >= ${pct(minGap)}`, - } -} - -/** - * The quality gate the paper pairs with the gap: reject examples that LEAK the answer into the - * context (a copy-paste solver would pass), or that ship a thin rubric. Deterministic, no LLM. - */ -export function qualityCheck(ex: DataExample): { ok: boolean; reason: string } { - const ref = ex.reference.trim() - if (ref.length > 0 && ex.context.includes(ref)) { - return { ok: false, reason: 'leaked: the reference answer appears verbatim in the context' } - } - if (ex.rubric.length < 2) { - return { ok: false, reason: 'thin rubric: an example needs >= 2 scoring criteria' } - } - return { ok: true, reason: 'clean' } -} - -// ── Tasks + output adapters (the worker seam) ────────────────────────────────────────────── - -/** The challenger's task: ground on `doc`, run the `prompt` the driver authored this round. */ -interface ChallengerTask { - readonly doc: string - /** The instruction for THIS round — the refine driver rewrites it from the last reject. */ - readonly prompt: string -} - -/** One solver attempt over an `example`; `sampleIndex` distinguishes the N parallel samples. */ -interface SolverTask { - readonly example: DataExample - readonly sampleIndex: number -} - -const challengerOutput: OutputAdapter = { - parse(events) { - const ex = resultPayload(events) - if (isDataExample(ex)) return ex - // Fail loud: a challenger that produced no parseable example is a real defect, not an empty pass. - throw new Error('challenger produced no parseable DataExample') - }, -} - -const solverOutput: OutputAdapter<{ answer: string }> = { - parse(events) { - const r = resultPayload(events) - if ( - r && - typeof r === 'object' && - 'answer' in r && - typeof (r as { answer: unknown }).answer === 'string' - ) { - return { answer: (r as { answer: string }).answer } - } - throw new Error('solver produced no answer') - }, -} - -/** One solver attempt's recorded answer + judge score — the unit of the per-example autopsy dump. */ -export interface SolverSample { - readonly answer: string - readonly score: number - readonly notes?: string -} - -/** A solver's N× sampled result: the variance-reduced mean the accept rule compares + the raw samples. */ -export interface SolverEval { - readonly mean: number - readonly samples: readonly SolverSample[] -} - -// ── N× solver sampling = an inline FANOUT driver over runLoop ──────────────────────────────── -// -// A "round" returns N independent solver tasks (no fold between them) → the kernel runs all N, -// the `llmJudge`-as-validator scores each against the rubric, and we AVERAGE the N scores (the -// variance-reduced estimate the accept rule compares — not argmax). runLoop already aggregated -// the N calls' cost, so we roll its total into the ledger under this solver's channel. Each sample's -// ANSWER TEXT and score are captured (not just the mean) so a null is autopsy-able per example. -async function sampleSolverScore(args: { - solver: SandboxClient - solverSpec: AgentRunSpec - example: DataExample - judge: JudgeConfig - samples: number - channel: string - ledger: CostLedger - signal?: AbortSignal -}): Promise { - const { solver, solverSpec, example, judge, samples, channel, ledger } = args - - const collected: SolverSample[] = [] - const validator: Validator<{ answer: string }> = { - async validate(out, ctx) { - const score = await judge.score({ - artifact: { example, answer: out.answer }, - scenario: solveScenario, - signal: ctx.signal, - }) - collected.push({ answer: out.answer, score: score.composite, notes: score.notes }) - return { valid: !score.failed, score: score.composite, notes: score.notes } - }, - } - - const fanout: Driver = { - name: `${channel}/sample-x${samples}`, - plan: async (task, history) => - history.length === 0 - ? Array.from({ length: samples }, (_, i) => ({ ...task, sampleIndex: i })) - : [], - decide: () => 'done', - } - - const result = await runLoop({ - driver: fanout, - agentRun: solverSpec, - output: solverOutput, - validator, - task: { example, sampleIndex: 0 }, - ctx: { sandboxClient: solver, signal: args.signal }, - maxIterations: samples, - maxConcurrency: samples, - }) - - ledger.record({ - model: solverSpec.profile.name ?? channel, - channel, - usage: { inputTokens: result.tokenUsage.input, outputTokens: result.tokenUsage.output }, - actualCostUsd: result.costUsd, - tags: { role: channel }, - }) - - if (collected.length === 0) - throw new Error(`${channel}: every solver sample errored — no score to average`) - const mean = collected.reduce((a, b) => a + b.score, 0) / collected.length - return { mean, samples: collected } -} - -// ── The challenger refine driver — the FOLD ────────────────────────────────────────────────── - -type ChallengerDecision = 'refine' | 'accept' | 'reject' - -/** - * The steer the fold appends per reject reason. The accept rule and this map are a matched pair: - * "too easy" almost always means the answer leaked into the context (recall), so the steer is to go - * non-extractive; "too hard" eases the derivation; "not discriminative" sharpens the contrast. - */ -function foldGuidance(why: string): string { - if (/leaked/i.test(why)) { - return ( - 'The reference answer leaked into the context. Rewrite so the CONTEXT holds ONLY the premises ' + - 'and the answer must be DERIVED, never quoted.' - ) - } - if (/too easy/i.test(why)) { - return ( - 'The weak solver scored too high — the answer is extractable from the context (recall / ' + - 'leakage). Ask a CAUSAL or COMPARATIVE question whose answer is NOT stated in the context and ' + - 'must be derived, and delete any sentence from the context that states the conclusion.' - ) - } - if (/too hard/i.test(why)) { - return ( - 'Even the strong solver missed it — ease it. Keep it causal, but shorten the required ' + - 'reasoning chain and make sure EVERY premise needed to derive the answer is present in the ' + - 'context (without stating the conclusion).' - ) - } - if (/not discriminative/i.test(why)) { - return ( - 'Both solvers scored similarly — sharpen the contrast: the question must need a multi-step ' + - 'derivation a small model gets wrong, while keeping every premise a strong model needs in the ' + - 'context.' - ) - } - return 'Write a new example that fixes exactly the stated problem.' -} - -function challengerDriver( - maxRetries: number, - baseInstruction: (doc: string) => string, -): Driver { - return { - name: 'challenger-refine', - async plan(task, history) { - if (history.length === 0) return [task] // shot 0: a first draft straight from the doc - const last = history[history.length - 1] - if (last?.verdict?.valid) return [] // accepted → stop - if (history.length >= maxRetries) return [] // out of budget → stop - // THE FOLD: read WHY the last example was rejected and rewrite the instruction to target it. - // Each accept-rule reason maps to a specific steer toward "just right". - const why = last?.verdict?.notes ?? 'rejected' - const prompt = `${baseInstruction(task.doc)}\n\nYour previous example was REJECTED: ${why}.\n${foldGuidance(why)}` - return [{ ...task, prompt }] - }, - decide(history) { - if (history.some((it) => it.verdict?.valid)) return 'accept' - return history.length < maxRetries ? 'refine' : 'reject' - }, - } -} - -// ── One example's full evaluation (used for both the accept loop and calibration) ───────────── - -export interface ExampleEvaluation { - readonly example: DataExample - readonly weakScore: number - readonly strongScore: number - readonly gap: number - readonly decision: AcceptDecision -} - -/** - * One fully-evaluated challenger attempt — emitted to `onAttempt` for EVERY candidate, accepted or - * rejected. The whole point is diagnosability: a null is only a finding if you can read the actual - * answers and see WHY the gap didn't open (weak read it out of the context, judge couldn't separate, - * strong erred, …). This carries the answer text both solvers produced, not just the scalar scores. - */ -export interface AttemptRecord { - /** Which target slot (outer loop index). */ - readonly slotIndex: number - /** Which refine iteration within the slot (0 = first draft / plain). */ - readonly iteration: number - readonly example: DataExample - readonly weak: SolverEval - readonly strong: SolverEval - readonly gap: number - readonly decision: AcceptDecision - /** Whether the deterministic quality gate (no leak, real rubric) passed before solving. */ - readonly qualityOk: boolean -} - -// ── The loop ──────────────────────────────────────────────────────────────────────────────── - -export interface DataCreationConfig { - /** The grounding document the challenger writes examples from. */ - readonly doc: string - /** The challenger worker (prompt → DataExample). The driver authors each round's prompt. */ - readonly challenger: SandboxClient - /** The weak + strong solver workers (rendered example → answer). */ - readonly weakSolver: SandboxClient - readonly strongSolver: SandboxClient - /** The rubric judge — an `llmJudge` `JudgeConfig`. */ - readonly judge: JudgeConfig - /** The challenger's base instruction over the doc (the un-folded prompt). */ - readonly baseInstruction: (doc: string) => string - /** How a solver sees one example. Default: context + question + numbered rubric + sample tag. */ - readonly renderSolverPrompt?: (example: DataExample, sampleIndex: number) => string - /** Profiles materialized for each worker (names surface in traces + the cost ledger). */ - readonly challengerProfile?: AgentProfile - readonly weakSolverProfile?: AgentProfile - readonly strongSolverProfile?: AgentProfile - /** The accept rule. Defaults to `discriminativeAcceptRule` at its paper thresholds. */ - readonly accept?: (input: { strongScore: number; weakScore: number }) => AcceptDecision - /** How many accepted examples to manufacture. Default 3. */ - readonly target?: number - /** Solver samples per example (variance reduction). Default 3. */ - readonly samples?: number - /** Refine budget per example. Default 4. */ - readonly maxRetries?: number - /** Where accepted examples accrete. Default a fresh `InMemoryCorpus`. */ - readonly corpus?: Corpus - /** Cost ledger to record into. Default a fresh `CostLedger`. */ - readonly cost?: CostLedger - /** - * Observability hook: fired once per evaluated candidate (accepted OR rejected) with both solvers' - * answer text + scores. Wire a JSONL writer here so every null is autopsy-able. Errors are not - * swallowed — a throwing observer fails the run loud. - */ - readonly onAttempt?: (rec: AttemptRecord) => void | Promise - readonly signal?: AbortSignal -} - -/** - * Default solver prompt: the premises + the question — never the rubric. The rubric is the grading - * key; showing it to the solver hands the weak model the mark scheme and closes the very gap the - * discriminative reward is trying to open. The answer is not stated in the context (the challenger - * withholds the conclusion), so the prompt tells the solver to DERIVE it. - */ -function defaultRenderSolverPrompt(example: DataExample, sampleIndex: number): string { - return ( - `Answer the QUESTION by reasoning from the CONTEXT. The answer is NOT stated verbatim in the ` + - `context — you must derive it.\n\n` + - `CONTEXT:\n${example.context}\n\n` + - `QUESTION:\n${example.question}\n` + - `[sample ${sampleIndex}]` - ) -} - -export interface DataCreationResult { - /** The accepted, discriminating examples (the manufactured training set). */ - readonly accepted: ExampleEvaluation[] - /** The `gap` of each accepted example — large by construction (the agentic arm). */ - readonly agenticGaps: number[] - /** The `gap` of each FIRST (un-refined) draft — the plain-generation baseline for calibration. */ - readonly plainGaps: number[] - /** Per slot, the BEST gap the refinement reached (max over the budget), accepted or not. Lets the - * plain-vs-refined calibration stay informative even when no example clears the accept bar. */ - readonly refinedGaps: number[] - readonly corpus: Corpus - readonly cost: CostLedger -} - -/** - * Run the Autodata inner loop: manufacture `target` discriminating examples from `doc`, refining - * each via the challenger fold until it is accepted (or its retry budget runs out). Returns the - * accepted set, the per-example gap for the accepted (agentic) AND the first-draft (plain) examples - * for calibration, the corpus they accreted into, and the cost ledger. - */ -export async function createDataCreationLoop( - config: DataCreationConfig, -): Promise { - const corpus = config.corpus ?? new InMemoryCorpus() - const cost = config.cost ?? new CostLedger() - const accept = config.accept ?? ((i) => discriminativeAcceptRule(i)) - const target = config.target ?? 3 - const samples = config.samples ?? 3 - const maxRetries = config.maxRetries ?? 4 - const renderSolverPrompt = config.renderSolverPrompt ?? defaultRenderSolverPrompt - - // Build the three worker specs once (task → prompt + the profile the substrate materializes). - const challengerSpec: AgentRunSpec = { - profile: config.challengerProfile ?? ({ name: 'challenger' } as AgentProfile), - taskToPrompt: (t) => t.prompt, - } - const weakSolverSpec: AgentRunSpec = { - profile: config.weakSolverProfile ?? ({ name: 'weak-solver' } as AgentProfile), - taskToPrompt: (t) => renderSolverPrompt(t.example, t.sampleIndex), - } - const strongSolverSpec: AgentRunSpec = { - profile: config.strongSolverProfile ?? ({ name: 'strong-solver' } as AgentProfile), - taskToPrompt: (t) => renderSolverPrompt(t.example, t.sampleIndex), - } - - const accepted: ExampleEvaluation[] = [] - const agenticGaps: number[] = [] - const plainGaps: number[] = [] - const refinedGaps: number[] = [] - - for (let i = 0; i < target; i++) { - // The challenger validator evaluates a candidate example: sample both solvers, judge each, then - // apply the accept rule. It stashes each iteration's evaluation so the loop can read back the - // ACCEPTED one (the agentic arm) and the FIRST draft (the plain calibration baseline). - const evaluations = new Map() - const emptyEval: SolverEval = { mean: 0, samples: [] } - const validator: Validator = { - async validate(example, ctx) { - const quality = qualityCheck(example) - const weak = quality.ok - ? await sampleSolverScore({ - solver: config.weakSolver, - solverSpec: weakSolverSpec, - example, - judge: config.judge, - samples, - channel: 'weak-solver', - ledger: cost, - signal: ctx.signal, - }) - : emptyEval - const strong = quality.ok - ? await sampleSolverScore({ - solver: config.strongSolver, - solverSpec: strongSolverSpec, - example, - judge: config.judge, - samples, - channel: 'strong-solver', - ledger: cost, - signal: ctx.signal, - }) - : emptyEval - const weakScore = weak.mean - const strongScore = strong.mean - const decision = quality.ok - ? accept({ strongScore, weakScore }) - : { accept: false, reason: quality.reason } - const gap = strongScore - weakScore - evaluations.set(ctx.iteration, { example, weakScore, strongScore, gap, decision }) - // Emit the full attempt (accept OR reject) so the null is diagnosable from the raw answers. - if (config.onAttempt) { - await config.onAttempt({ - slotIndex: i, - iteration: ctx.iteration, - example, - weak, - strong, - gap, - decision, - qualityOk: quality.ok, - }) - } - return { valid: decision.accept, score: gap, notes: decision.reason } - }, - } - - const result = await runLoop({ - driver: challengerDriver(maxRetries, config.baseInstruction), - agentRun: challengerSpec, - output: challengerOutput, - validator, - task: { doc: config.doc, prompt: config.baseInstruction(config.doc) }, - ctx: { sandboxClient: config.challenger, signal: config.signal }, - maxIterations: maxRetries + 1, - }) - - cost.record({ - model: challengerSpec.profile.name ?? 'challenger', - channel: 'challenger', - usage: { inputTokens: result.tokenUsage.input, outputTokens: result.tokenUsage.output }, - actualCostUsd: result.costUsd, - tags: { role: 'challenger' }, - }) - - // The "plain" (un-refined) baseline is the FIRST candidate that was actually evaluated — the - // earliest recorded iteration. Reading a hardcoded index 0 silently drops the baseline whenever - // the first challenger draft errored (e.g. unparseable JSON), which is exactly when the slot's - // first SUCCESSFUL draft sits at a later index. Take the min recorded iteration instead. - const firstIteration = [...evaluations.keys()].sort((a, b) => a - b)[0] - const plain = firstIteration === undefined ? undefined : evaluations.get(firstIteration) - if (plain) plainGaps.push(plain.gap) - - const slotGaps = [...evaluations.values()].map((e) => e.gap) - if (slotGaps.length > 0) refinedGaps.push(Math.max(...slotGaps)) - - // ONLY a genuinely-accepted winner counts. `defaultSelectWinner` falls back to the best-scoring - // iteration when none is valid, so `result.winner` is set even when the accept rule rejected - // every candidate — with real solvers that frequently happens (no question separated the tiers - // inside the budget). Gate on `verdict.valid` so the manufactured set never includes a rejected - // example; a target slot that never produced a discriminating example is simply left unfilled. - if (result.winner?.verdict?.valid) { - const winnerEval = evaluations.get(result.winner.iterationIndex) - if (!winnerEval) throw new Error('internal: accepted iteration has no recorded evaluation') - const append = await corpus.append(toCorpusRecord(winnerEval, i)) - if (!append.succeeded) throw new Error(`corpus append failed: ${append.error}`) - accepted.push(winnerEval) - agenticGaps.push(winnerEval.gap) - cost.markCompleted() - } - } - - return { accepted, agenticGaps, plainGaps, refinedGaps, corpus, cost } -} - -// ── Helpers ─────────────────────────────────────────────────────────────────────────────── - -const solveScenario: Scenario = { id: 'agentic-data-creation', kind: 'solve' } - -function toCorpusRecord(evalRec: ExampleEvaluation, index: number): CorpusRecord { - return { - schemaVersion: '1.0.0', - id: `example-${index}`, - runId: 'agentic-data-creation', - producedAt: new Date().toISOString(), - area: 'training-data', - claim: JSON.stringify(evalRec.example), - rationale: evalRec.decision.reason, - tags: ['discriminative', `gap:${evalRec.gap.toFixed(2)}`], - // The gap is the producing run's confidence this example is hard — clamped into [0,1]. - confidence: Math.min(1, Math.max(0, evalRec.gap)), - } -} - -function resultPayload(events: SandboxEvent[]): unknown { - for (const ev of events) { - if (ev.type === 'result') return (ev as { data?: { result?: unknown } }).data?.result - } - return undefined -} - -function isDataExample(value: unknown): value is DataExample { - if (typeof value !== 'object' || value === null) return false - const v = value as Record - return ( - typeof v.context === 'string' && - typeof v.question === 'string' && - typeof v.reference === 'string' && - Array.isArray(v.rubric) - ) -} - -function pct(x: number): string { - return x.toFixed(2) -} diff --git a/src/autodata/grounding.ts b/src/autodata/grounding.ts deleted file mode 100644 index 29a7e3d..0000000 --- a/src/autodata/grounding.ts +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Ground the Autodata loop on a REAL source document, reusing agent-knowledge's ingestion utils - * (`politeFetch` → `htmlToText` → `chunkMarkdown`). Fetches the page, strips it to text, chunks it, - * and selects ONE content-rich chunk as the grounding excerpt the challenger writes questions from. - * - * The default source is the Mixtral-of-Experts paper (arXiv 2401.04088) via ar5iv. The doc CHOICE is - * load-bearing: a hard question only separates a small solver from a frontier one if the small solver - * cannot just RECALL the answer from pretraining. The canonical "Attention Is All You Need" paper is - * the worst case — an 8B has memorized it, so even reasoning questions are answerable from memory and - * the strong/weak gap collapses (an empirically-verified null). Mixtral (Jan 2024) post-dates the 8B - * weak solver's knowledge cutoff, so it must reason from the provided context — which is where a - * non-extractive causal question opens a real gap. Any arXiv / ar5iv URL works; pass `focus` to bias - * chunk selection toward a section. - */ - -import { chunkMarkdown } from '../chunking' -import { htmlToText } from '../sources/html' -import { politeFetch } from '../sources/http' - -/** A stable real arXiv paper (Mixtral of Experts) rendered to HTML by ar5iv — see the note above on - * why a NON-memorized doc is required for the strong/weak gap to open. */ -export const DEFAULT_SOURCE_URL = 'https://ar5iv.labs.arxiv.org/html/2401.04088' - -export interface GroundDocOptions { - url: string - cacheDir?: string - /** Bias chunk selection toward chunks mentioning this term (case-insensitive). */ - focus?: string - /** Chunk size ceiling. Default 1800 chars — a paragraph or two of grounding context. */ - maxChars?: number - /** Minimum letters a chunk must have to be eligible (skips nav / citation scraps). Default 400. */ - minLetters?: number - signal?: AbortSignal -} - -export interface GroundedDoc { - url: string - sourceUpdatedAt: string - /** The selected grounding excerpt — the `doc` passed to the loop. */ - doc: string - chunkIndex: number - headingPath: string - totalChunks: number -} - -function letterCount(s: string): number { - return (s.match(/[a-zA-Z]/g) ?? []).length -} - -/** Reference/bibliography chunks are citation soup — never good question material. */ -function looksLikeReferences(headingPath: string, text: string): boolean { - if (/references|bibliography|acknowledg/i.test(headingPath)) return true - // A chunk that is mostly "[n]" / "et al." / years is a reference list. - const refMarkers = (text.match(/\[\d+\]|et al\.|arXiv:|doi:/gi) ?? []).length - return refMarkers >= 5 -} - -/** - * Fetch + chunk + select a grounding excerpt from a real document. Fails loud if the fetch is - * unverifiable or yields no usable prose chunk. - */ -export async function groundDoc(opts: GroundDocOptions): Promise { - const res = await politeFetch(opts.url, { cacheDir: opts.cacheDir, signal: opts.signal }) - if (!res.verifiable) { - throw new Error(`source not verifiable (${opts.url}): ${res.unverifiableReason ?? 'unknown'}`) - } - const text = htmlToText(res.body) - const maxChars = opts.maxChars ?? 1800 - const chunks = chunkMarkdown(text, { maxChars, targetChars: Math.round(maxChars * 0.8) }) - const minLetters = opts.minLetters ?? 400 - - const eligible = chunks.filter( - (c) => - !c.oversized && - letterCount(c.text) >= minLetters && - !looksLikeReferences(c.headingPath, c.text), - ) - if (eligible.length === 0) { - throw new Error( - `no usable prose chunk from ${opts.url} (${chunks.length} chunks, none eligible)`, - ) - } - - const focus = opts.focus?.toLowerCase() - const score = (text: string): number => { - const letters = letterCount(text) - if (!focus) return letters - const hits = ( - text.toLowerCase().match(new RegExp(focus.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g')) ?? [] - ).length - return hits * 2000 + letters - } - const selected = eligible.reduce((best, c) => (score(c.text) > score(best.text) ? c : best)) - - return { - url: opts.url, - sourceUpdatedAt: res.sourceUpdatedAt, - doc: selected.text, - chunkIndex: selected.index, - headingPath: selected.headingPath, - totalChunks: chunks.length, - } -} diff --git a/src/autodata/index.ts b/src/autodata/index.ts deleted file mode 100644 index deb45ca..0000000 --- a/src/autodata/index.ts +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Autodata — the LIVE dataset builder: ground on a real source document, run the agentic - * data-creation loop with REAL two-tier solver models (the paper's Qwen tiers via the Tangle - * router), and emit a discriminative QA dataset + the empirical strong/weak gap. - * - * The inner loop (`createDataCreationLoop` / `discriminativeAcceptRule`) is vendored from the - * agent-runtime example and composes only published substrate primitives. This package adds the - * real grounding (`grounding`), the real router roles (`router-roles`), and the pipeline + - * JSONL writer (`build-dataset`). - */ - -export { - type AutodataDatasetConfig, - type AutodataDatasetResult, - buildAutodataDataset, - type DatasetRow, - type DiscriminativeThresholds, -} from './build-dataset' -export { - type AcceptDecision, - type AttemptRecord, - createDataCreationLoop, - type DataCreationConfig, - type DataCreationResult, - type DataExample, - discriminativeAcceptRule, - type ExampleEvaluation, - qualityCheck, - type SolverArtifact, - type SolverEval, - type SolverSample, -} from './data-creation-loop' -export { - DEFAULT_SOURCE_URL, - type GroundDocOptions, - type GroundedDoc, - groundDoc, -} from './grounding' -export { analyzeTrails, type DocTrail, type PoweredStats } from './powered' -export { - type AutodataRoles, - buildAutodataRoles, - CHALLENGER_MODEL, - type ChallengerStyle, - DEFAULT_BASE_URL, - JUDGE_MODEL, - parseDataExample, - type RouterCallRecord, - type RouterChatInput, - type RouterChatResult, - type RouterRolesConfig, - routerChat, - type SmokeResult, - STRONG_SOLVER_MODEL, - smokeTestModels, - WEAK_SOLVER_MODEL, -} from './router-roles' diff --git a/src/autodata/offline-fixtures.ts b/src/autodata/offline-fixtures.ts deleted file mode 100644 index 77e0c9c..0000000 --- a/src/autodata/offline-fixtures.ts +++ /dev/null @@ -1,200 +0,0 @@ -/** - * Credentialless offline stand-ins so the Autodata loop runs in CI with ZERO creds and reproducible - * numbers: scripted challenger/solvers + a mock-transport judge. None of this is the lesson — it is - * the minimum that lets the wiring be tested offline. The LIVE roles (real router models) live in - * `router-roles.ts`; the scores here are tuned to reproduce the paper's Table 1 separation (an EASY - * first-draft example barely separates the two solvers, a HARD loop-accepted one separates widely). - * - * Ported from agent-runtime `examples/agentic-data-creation/offline-fixtures.ts`. - */ - -import { createChatClient, llmJudge } from '@tangle-network/agent-eval' -import type { JudgeConfig } from '@tangle-network/agent-eval/campaign' -import { inProcessSandboxClient, type SandboxClient } from '@tangle-network/agent-runtime/loops' -import type { SandboxEvent } from '@tangle-network/sandbox' -import type { DataExample, SolverArtifact } from './data-creation-loop' - -export const groundingDoc = `Idempotency in the Payments API - -Every write to POST /charges may carry an Idempotency-Key header. The server stores the first -response under that key for 24 hours. A retry with the SAME key and the SAME request body replays -the stored response instead of charging again. A retry with the SAME key but a DIFFERENT body is a -conflict: the server rejects it with 422 Unprocessable Entity and creates no second charge. -Idempotency keys are scoped per merchant account.` - -export const baseInstruction = (doc: string): string => - `You are writing ONE training example from the document below. Produce a context excerpt, a ` + - `question answerable from it, a reference answer, and a 2-3 item rubric.\n\nDOCUMENT:\n${doc}` - -const easyExample: DataExample = { - context: 'Every write to POST /charges may carry an Idempotency-Key header.', - question: 'Which HTTP header carries the idempotency key on a POST /charges write?', - reference: 'The request uses an Idempotency-Key header.', - rubric: ['Names the Idempotency-Key header', 'Ties it to a POST /charges write'], -} - -const hardExamples: DataExample[] = [ - { - context: - 'A retry with the same key but a different body is a conflict: the server rejects it with 422 ' + - 'Unprocessable Entity and creates no second charge.', - question: - 'Why must the server reject a same-key, different-body retry with 422 instead of replaying the ' + - 'stored response, and what failure does that prevent?', - reference: - 'Replaying the stored response would apply it to a different request; rejecting with 422 surfaces ' + - 'the mismatch and prevents a double or incorrect charge.', - rubric: [ - 'States the server rejects the retry with 422', - 'Explains replaying the stored response would be wrong for a different body', - 'Identifies the prevented failure: a double or incorrect charge', - ], - }, - { - context: - 'The server stores the first response under the idempotency key for 24 hours and replays it on a ' + - 'retry with the same key and the same body.', - question: - 'Why does replaying the stored response on a same-key, same-body retry matter, and what failure ' + - 'does it prevent when a client retries after a dropped connection?', - reference: - 'The original request may already have charged; replaying returns that one result so a network ' + - 'retry does not create a second charge.', - rubric: [ - 'Explains the first request may have already succeeded', - 'States the stored response is replayed instead of re-charging', - 'Identifies the prevented failure: a duplicate charge on retry', - ], - }, - { - context: 'Idempotency keys are scoped per merchant account.', - question: - 'Why are idempotency keys scoped per merchant account, and what would break if they were global ' + - 'across all merchants?', - reference: - 'Per-merchant scoping isolates key spaces; a global scope would let one merchant key collide ' + - 'with another and replay the wrong merchant charge.', - rubric: [ - 'States keys are isolated per merchant account', - 'Explains a global scope risks cross-merchant key collisions', - 'Identifies the failure: replaying the wrong merchant charge', - ], - }, -] - -const hardQuestionPattern = /\b(why|explain|under what|what happens if|reason)\b/i - -/** - * Scripted challenger: first draft (no "REJECTED" in the prompt) → the EASY example; once the refine - * driver folds a "too easy" reject into the prompt, it ships the next HARD example — proving the - * loop's behavior changed because of the fold. Stateful so successive targets get DISTINCT examples. - */ -export function challengerClient(): SandboxClient { - let hardServed = 0 - return inProcessSandboxClient({ - onPrompt: (prompt): SandboxEvent[] => { - const wantsHarder = /rejected|too easy/i.test(prompt) - const example = wantsHarder - ? (hardExamples[hardServed++ % hardExamples.length] ?? easyExample) - : easyExample - return [ - { - type: 'llm_call', - data: { model: 'offline-challenger', tokensIn: 320, tokensOut: 90, costUsd: 0.0006 }, - }, - { type: 'result', data: { result: example } }, - ] - }, - }) -} - -/** - * Scripted solver: answers the rendered example and tags the answer with a grade marker the offline - * judge reads. The weak solver produces a thin answer; the strong solver a complete one. - */ -export function solverClient(strength: 'weak' | 'strong'): SandboxClient { - return inProcessSandboxClient({ - onPrompt: (prompt): SandboxEvent[] => { - const hard = hardQuestionPattern.test(prompt) - const sample = Number(/\[sample (\d+)\]/.exec(prompt)?.[1] ?? '0') - const body = - strength === 'strong' - ? 'A complete, rubric-covering answer grounded in the context.' - : 'A short, partial answer.' - const answer = `${body} <>` - return [ - { - type: 'llm_call', - data: { - model: `offline-${strength}-solver`, - tokensIn: 140, - tokensOut: 30, - costUsd: 0.0003, - }, - }, - { type: 'result', data: { result: { answer } } }, - ] - }, - }) -} - -/** A REAL `llmJudge` over a MOCK transport: returns a scripted [0,1] score from the grade marker. */ -export function buildRubricJudge(): JudgeConfig { - const chat = createChatClient({ - transport: 'mock', - defaultModel: 'offline-judge', - handler: async (req) => { - const text = req.messages - .map((m) => (typeof m.content === 'string' ? m.content : '')) - .join('\n') - const m = /<>/.exec(text) - const strength = m?.[1] - const difficulty = m?.[2] - const sampleIndex = m?.[3] - if (!strength || !difficulty || sampleIndex === undefined) { - throw new Error('offline judge: answer carried no grade marker') - } - const base = - difficulty === 'hard' - ? strength === 'strong' - ? 0.77 - : 0.46 - : strength === 'strong' - ? 0.86 - : 0.84 - // Per-sample jitter over samples 0,1,2 → −0.02, 0, +0.02, so the N× mean lands back on `base`. - const jitter = (Number(sampleIndex) - 1) * 0.02 - const score = Math.min(1, Math.max(0, base + jitter)) - return { - content: JSON.stringify({ - dimensions: { rubric_coverage: score, correctness: score }, - notes: `offline: ${strength} solver on ${difficulty} example (sample ${sampleIndex})`, - }), - usage: { promptTokens: 130, completionTokens: 25, totalTokens: 155 }, - costUsd: 0.0001, - model: 'offline-judge', - durationMs: 1, - raw: {}, - } - }, - }) - - return llmJudge( - 'rubric-judge', - 'Score the candidate ANSWER against the example RUBRIC. Return JSON ' + - '{"dimensions":{"rubric_coverage":N,"correctness":N},"notes":"..."} with each score in [0,1].', - { - chat, - dimensions: [ - { - key: 'rubric_coverage', - description: 'fraction of the rubric criteria the answer satisfies', - }, - { key: 'correctness', description: 'agreement with the reference answer' }, - ], - scale: 'unit', - renderUser: ({ artifact }) => - `RUBRIC:\n${artifact.example.rubric.map((r, i) => `${i + 1}. ${r}`).join('\n')}\n\nANSWER:\n${artifact.answer}`, - }, - ) -} diff --git a/src/autodata/powered.test.ts b/src/autodata/powered.test.ts deleted file mode 100644 index 861e127..0000000 --- a/src/autodata/powered.test.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { describe, expect, it } from 'vitest' -import type { AttemptRecord, SolverEval } from './data-creation-loop' -import { analyzeTrails, type DocTrail } from './powered' - -// A minimal attempt-row factory: only the fields `analyzeTrails` reads matter. -function attempt(args: { - slot: number - iteration: number - weak: number - strong: number - accept: boolean - qualityOk?: boolean -}): AttemptRecord { - const weak: SolverEval = { mean: args.weak, samples: [] } - const strong: SolverEval = { mean: args.strong, samples: [] } - const gap = args.strong - args.weak - return { - slotIndex: args.slot, - iteration: args.iteration, - example: { context: 'c', question: 'q', reference: 'r', rubric: ['a', 'b'] }, - weak, - strong, - gap, - decision: { accept: args.accept, reason: args.accept ? 'discriminates' : 'rejected' }, - qualityOk: args.qualityOk ?? true, - } -} - -describe('analyzeTrails (powered aggregation)', () => { - it('counts accept-rate per slot over the requested target, not the trailed slots', () => { - // 3 slots requested, only 2 left a trail; slot 0 accepted, slot 1 rejected, slot 2 errored (no rows). - const trail: DocTrail = { - tag: 'd', - url: 'u', - target: 3, - rows: [ - attempt({ slot: 0, iteration: 0, weak: 0.7, strong: 0.8, accept: false }), - attempt({ slot: 0, iteration: 1, weak: 0.3, strong: 0.9, accept: true }), - attempt({ slot: 1, iteration: 0, weak: 0.6, strong: 0.7, accept: false }), - ], - } - const s = analyzeTrails([trail], { bootstrapSeed: 1 }) - expect(s.totalSlots).toBe(3) // denominator = requested target, not 2 trailed slots - expect(s.acceptedSlots).toBe(1) - expect(s.acceptRate.estimate).toBeCloseTo(1 / 3, 6) - // Wilson lower bound is strictly above 0 and the point estimate sits inside the interval. - expect(s.acceptRate.lower).toBeGreaterThan(0) - expect(s.acceptRate.lower).toBeLessThan(s.acceptRate.estimate) - expect(s.acceptRate.upper).toBeGreaterThan(s.acceptRate.estimate) - // Slot 2 left no trail → counted as a challenger-stage failure, still in the denominator. - expect(s.slotsWithAttempts).toBe(2) - expect(s.challengerFailedSlots).toBe(1) - // Among only producing slots the denominator is 2, so the rate is higher. - expect(s.acceptRateAmongProducing.estimate).toBeCloseTo(0.5, 6) - }) - - it('takes the slot best-gap from quality-clean attempts and pairs plain→refined', () => { - const trail: DocTrail = { - tag: 'd', - url: 'u', - target: 1, - rows: [ - attempt({ slot: 0, iteration: 0, weak: 0.7, strong: 0.8, accept: false }), // plain gap 0.10 - attempt({ slot: 0, iteration: 1, weak: 0.3, strong: 0.9, accept: true }), // refined gap 0.60 - ], - } - const s = analyzeTrails([trail], { bootstrapSeed: 1 }) - expect(s.bestGapPerSlot.max).toBeCloseTo(0.6, 6) - expect(s.plainGap.median).toBeCloseTo(0.1, 6) - // The fold widened the gap from 0.10 to 0.60 → a positive paired delta. - expect(s.widening.meanDelta).toBeCloseTo(0.5, 6) - }) - - it('a leaky (quality-failed) draft contributes a 0 best-gap but is excluded from the condition decomposition', () => { - const trail: DocTrail = { - tag: 'd', - url: 'u', - target: 1, - rows: [ - attempt({ slot: 0, iteration: 0, weak: 0, strong: 0, accept: false, qualityOk: false }), - ], - } - const s = analyzeTrails([trail], { bootstrapSeed: 1 }) - expect(s.bestGapPerSlot.max).toBe(0) - expect(s.conditions.nAttempts).toBe(0) // quality-failed attempt excluded - expect(s.acceptedSlots).toBe(0) - }) - - it('decomposes the accept rule: the binding gate is weak < 0.5', () => { - // All attempts have strong high + gap wide, but weak only struggles half the time. - const trail: DocTrail = { - tag: 'd', - url: 'u', - target: 4, - rows: [ - attempt({ slot: 0, iteration: 0, weak: 0.2, strong: 0.9, accept: true }), - attempt({ slot: 1, iteration: 0, weak: 0.7, strong: 0.95, accept: false }), // weak too competent - attempt({ slot: 2, iteration: 0, weak: 0.1, strong: 0.85, accept: true }), - attempt({ slot: 3, iteration: 0, weak: 0.75, strong: 0.98, accept: false }), // weak too competent - ], - } - const s = analyzeTrails([trail], { bootstrapSeed: 1 }) - expect(s.conditions.strongHi).toBeCloseTo(1, 6) // strong always high - expect(s.conditions.gapWide).toBeCloseTo(1, 6) // gap always wide - expect(s.conditions.weakLo).toBeCloseTo(0.5, 6) // weak struggles only half → the binding gate - expect(s.conditions.all).toBeCloseTo(0.5, 6) - expect(s.acceptRate.estimate).toBeCloseTo(0.5, 6) - }) - - it('aggregates across multiple docs (denominators add)', () => { - const a: DocTrail = { - tag: 'a', - url: 'u1', - target: 2, - rows: [attempt({ slot: 0, iteration: 0, weak: 0.2, strong: 0.9, accept: true })], - } - const b: DocTrail = { - tag: 'b', - url: 'u2', - target: 2, - rows: [attempt({ slot: 0, iteration: 0, weak: 0.6, strong: 0.7, accept: false })], - } - const s = analyzeTrails([a, b], { bootstrapSeed: 1 }) - expect(s.totalSlots).toBe(4) - expect(s.acceptedSlots).toBe(1) - expect(s.perDoc).toHaveLength(2) - expect(s.perDoc[0]?.accepted).toBe(1) - expect(s.perDoc[1]?.accepted).toBe(0) - }) -}) diff --git a/src/autodata/powered.ts b/src/autodata/powered.ts deleted file mode 100644 index 37f9d2f..0000000 --- a/src/autodata/powered.ts +++ /dev/null @@ -1,450 +0,0 @@ -/** - * Autodata — POWERED accept-rate measurement. - * - * Settles the question n=3 was too noisy to answer: does the causal-challenger loop RELIABLY - * manufacture discriminating examples, or is acceptance a coin-flip? It runs a FIXED number of - * independent slots (each slot = one full challenger→refine→accept cycle) split across >= 2 - * non-memorized grounding docs, and reports the ACCEPTED-RATE with a Wilson 95% CI, the per-slot - * best-gap distribution, and the plain-vs-refined gap-widening with a paired-bootstrap CI. - * - * This is a FIXED-SLOTS harness, NOT until-N-accepted: it runs K slots and records each slot's - * outcome (accept/reject) + best gap, so the rate is bounded-cost and unbiased. It reuses - * `buildAutodataDataset` — which already runs exactly `target` independent slots — so nothing in the - * loop is rebuilt; only the cross-slot aggregation + the two confidence intervals are added here. - * The CIs are agent-eval's published estimators (`wilson` for the binomial accept-rate, `pairedBootstrap` - * for the paired plain-vs-refined widening), never hand-rolled. - * - * The source of truth is the per-attempt autopsy JSONL each doc writes incrementally: the final - * statistics are recomputed by re-reading those trails from disk (`analyzeTrails`), so an interrupted - * run loses no data — re-run the analysis over the JSONL. - * - * Run (key never printed): - * dotenvx run -f /home/drew/company/devops/secrets/agent-state.env -- \ - * pnpm tsx src/autodata/powered.ts - * - * Env knobs: AUTODATA_SLOTS_PER_DOC (default 14), AUTODATA_SAMPLES (default 4), - * AUTODATA_MAXRETRIES (default 3), AUTODATA_DOCS ("url|focus|tag,url|focus|tag" override), - * AUTODATA_{WEAK,STRONG,CHALLENGER,JUDGE}_MODEL, TANGLE_API_KEY (or TANGLE_ROUTER_KEY). - */ - -import { readFile } from 'node:fs/promises' -import { pairedBootstrap, wilson } from '@tangle-network/agent-eval' -import { buildAutodataDataset } from './build-dataset' -import type { AttemptRecord } from './data-creation-loop' -import { groundDoc } from './grounding' -import { - CHALLENGER_MODEL, - JUDGE_MODEL, - STRONG_SOLVER_MODEL, - smokeTestModels, - WEAK_SOLVER_MODEL, -} from './router-roles' - -/** One grounding document to split slots across. */ -interface DocSpec { - url: string - focus: string - tag: string -} - -/** - * Two non-memorized, reasoning-rich MoE papers (both post-date `llama-3.1-8b`'s knowledge cutoff, so - * the weak solver must REASON from the context, not recall) — the precondition for any gap to open. - * Each `focus` selects a PROSE mechanism chunk in the same "MoE-expert reasoning" band: a chunk dense - * with LaTeX equations (e.g. DeepSeek-V3's MLA section) breaks the challenger's strict-JSON output, so - * both focuses target the prose description of an expert-routing mechanism, not the equations. - * • Mixtral-of-Experts (2401.04088, Jan 2024) — sparse MoE expert routing / gating. - * • DeepSeek-V3 (2412.19437, Dec 2024) — auxiliary-loss-free load balancing / expert specialization. - */ -const defaultDocs: DocSpec[] = [ - { url: 'https://ar5iv.labs.arxiv.org/html/2401.04088', focus: 'expert', tag: 'mixtral' }, - { url: 'https://ar5iv.labs.arxiv.org/html/2412.19437', focus: 'auxiliary', tag: 'deepseek-v3' }, -] - -function envInt(name: string, fallback: number): number { - const raw = process.env[name] - if (!raw) return fallback - const n = Number.parseInt(raw, 10) - if (!Number.isFinite(n) || n <= 0) throw new Error(`${name}='${raw}' is not a positive integer`) - return n -} - -function parseDocsEnv(): DocSpec[] { - const raw = process.env.AUTODATA_DOCS - if (!raw) return defaultDocs - return raw.split(',').map((entry) => { - const [url, focus, tag] = entry.split('|').map((s) => s.trim()) - if (!url || !focus || !tag) - throw new Error(`AUTODATA_DOCS entry '${entry}' is not url|focus|tag`) - return { url, focus, tag } - }) -} - -// ── Descriptive distribution helpers (NOT inferential — those reuse agent-eval) ──────────────── - -/** Linear-interpolated quantile of a sorted-or-unsorted numeric sample. */ -function quantile(xs: number[], q: number): number { - if (xs.length === 0) return Number.NaN - const s = [...xs].sort((a, b) => a - b) - if (s.length === 1) return s[0] as number - const pos = (s.length - 1) * q - const lo = Math.floor(pos) - const hi = Math.ceil(pos) - const frac = pos - lo - return (s[lo] as number) * (1 - frac) + (s[hi] as number) * frac -} - -function mean(xs: number[]): number { - return xs.length === 0 ? Number.NaN : xs.reduce((a, b) => a + b, 0) / xs.length -} - -interface Distribution { - n: number - min: number - median: number - p90: number - max: number - mean: number -} - -function describe(xs: number[]): Distribution { - return { - n: xs.length, - min: xs.length ? Math.min(...xs) : Number.NaN, - median: quantile(xs, 0.5), - p90: quantile(xs, 0.9), - max: xs.length ? Math.max(...xs) : Number.NaN, - mean: mean(xs), - } -} - -// ── The aggregation: per-attempt JSONL trail → per-slot outcomes → CIs ───────────────────────── - -/** One JSONL row from a per-attempt trail: an `AttemptRecord` plus the challenger style tag. */ -type TrailRow = AttemptRecord & { style?: string } - -/** One doc's trail + the number of slots it was asked to run (the accept-rate denominator). */ -export interface DocTrail { - tag: string - url: string - /** Slots requested for this doc — the denominator (a slot whose drafts all errored counts as a - * reject, so the denominator is the requested count, not the number of slots that left a trail). */ - target: number - rows: TrailRow[] -} - -export interface PoweredStats { - totalSlots: number - acceptedSlots: number - /** Slots that produced >= 1 attempt (the challenger authored a parseable example at least once). */ - slotsWithAttempts: number - /** Slots where the challenger threw on every refine (0 attempts) — an infra/parse failure, NOT a - * discrimination reject. Surfaced so it can be flagged as a threat to validity. */ - challengerFailedSlots: number - /** Wilson 95% CI on the accept-rate (binomial: a slot accepts or it does not). */ - acceptRate: { estimate: number; lower: number; upper: number } - /** Wilson 95% CI on the accept-rate among only the slots that produced >= 1 attempt (excludes the - * challenger-stage failures) — the rate if every slot had at least authored an example. */ - acceptRateAmongProducing: { estimate: number; lower: number; upper: number } - /** Per-slot BEST gap (max gap over the slot's quality-clean attempts) — the discriminating power - * each slot reached, accepted or not. */ - bestGapPerSlot: Distribution - /** Plain (first-draft) gap per slot. */ - plainGap: Distribution - /** Paired plain→best-refined gap-widening with a bootstrap CI on the mean delta. */ - widening: { n: number; meanDelta: number; medianDelta: number; lower: number; upper: number } - /** Weak solver mean-score distribution over quality-clean attempts (the coin-flip's source). */ - weakScore: Distribution - /** Strong solver mean-score distribution over quality-clean attempts. */ - strongScore: Distribution - /** Per-attempt pass fractions for each sub-condition of the accept rule (decomposes the rate). */ - conditions: { - nAttempts: number - strongHi: number // fraction with strong >= 0.65 - weakLo: number // fraction with weak < 0.50 (the "weak must struggle" gate) - gapWide: number // fraction with gap >= 0.20 - all: number // fraction passing all three (== accept) - } - perDoc: { tag: string; target: number; accepted: number; meanBestGap: number }[] -} - -const minStrong = 0.65 -const maxWeak = 0.5 -const minGap = 0.2 - -/** - * Compute the powered statistics from the per-doc attempt trails. Pure + deterministic (seeded - * bootstrap), so it can be re-run standalone over the on-disk JSONL after an interrupted run. - */ -export function analyzeTrails(trails: DocTrail[], opts?: { bootstrapSeed?: number }): PoweredStats { - const seed = opts?.bootstrapSeed ?? 0xc0ffee - - let totalSlots = 0 - let acceptedSlots = 0 - let slotsWithAttempts = 0 - const bestGapPerSlotAll: number[] = [] - const plainGapsPaired: number[] = [] - const refinedGapsPaired: number[] = [] - const weakScores: number[] = [] - const strongScores: number[] = [] - let nAttempts = 0 - let strongHi = 0 - let weakLo = 0 - let gapWide = 0 - let acceptAll = 0 - const perDoc: PoweredStats['perDoc'] = [] - - for (const trail of trails) { - totalSlots += trail.target - - // Group this doc's attempts by slot index. - const bySlot = new Map() - for (const row of trail.rows) { - const arr = bySlot.get(row.slotIndex) ?? [] - arr.push(row) - bySlot.set(row.slotIndex, arr) - } - - slotsWithAttempts += bySlot.size - let docAccepted = 0 - const docBestGaps: number[] = [] - for (const [, rows] of bySlot) { - // A slot is ACCEPTED iff any of its attempts cleared the accept rule. - const accepted = rows.some((r) => r.decision.accept) - if (accepted) { - acceptedSlots += 1 - docAccepted += 1 - } - // Best gap the slot reached over its quality-clean attempts (a leaky/thin draft has gap 0). - const cleanGaps = rows.filter((r) => r.qualityOk).map((r) => r.gap) - const bestGap = cleanGaps.length ? Math.max(...cleanGaps) : 0 - bestGapPerSlotAll.push(bestGap) - docBestGaps.push(bestGap) - - // Paired plain→refined: plain = the earliest iteration's gap, refined = the slot's best gap. - const earliest = [...rows].sort((a, b) => a.iteration - b.iteration)[0] - if (earliest) { - plainGapsPaired.push(earliest.gap) - refinedGapsPaired.push(bestGap) - } - - // Per-attempt condition decomposition (quality-clean attempts only — a leaky draft never - // reached the solvers, so its 0/0 scores would falsely depress every sub-condition). - for (const r of rows) { - if (!r.qualityOk) continue - nAttempts += 1 - weakScores.push(r.weak.mean) - strongScores.push(r.strong.mean) - if (r.strong.mean >= minStrong) strongHi += 1 - if (r.weak.mean < maxWeak) weakLo += 1 - if (r.gap >= minGap) gapWide += 1 - if (r.strong.mean >= minStrong && r.weak.mean < maxWeak && r.gap >= minGap) acceptAll += 1 - } - } - perDoc.push({ - tag: trail.tag, - target: trail.target, - accepted: docAccepted, - meanBestGap: mean(docBestGaps), - }) - } - - const accept = wilson(acceptedSlots, totalSlots, 0.95) - const acceptProducing = wilson(acceptedSlots, slotsWithAttempts, 0.95) - const boot = pairedBootstrap(plainGapsPaired, refinedGapsPaired, { - confidence: 0.95, - statistic: 'mean', - seed, - resamples: 5000, - }) - - return { - totalSlots, - acceptedSlots, - slotsWithAttempts, - challengerFailedSlots: totalSlots - slotsWithAttempts, - acceptRate: { estimate: accept.estimate, lower: accept.lower, upper: accept.upper }, - acceptRateAmongProducing: { - estimate: acceptProducing.estimate, - lower: acceptProducing.lower, - upper: acceptProducing.upper, - }, - bestGapPerSlot: describe(bestGapPerSlotAll), - plainGap: describe(plainGapsPaired), - widening: { - n: boot.n, - meanDelta: boot.mean, - medianDelta: boot.median, - lower: boot.low, - upper: boot.high, - }, - weakScore: describe(weakScores), - strongScore: describe(strongScores), - conditions: { - nAttempts, - strongHi: nAttempts ? strongHi / nAttempts : Number.NaN, - weakLo: nAttempts ? weakLo / nAttempts : Number.NaN, - gapWide: nAttempts ? gapWide / nAttempts : Number.NaN, - all: nAttempts ? acceptAll / nAttempts : Number.NaN, - }, - perDoc, - } -} - -async function readTrail(path: string): Promise { - // A slot whose every challenger draft errored produces no attempt → no trail file is written. - // That is a legitimate all-reject outcome (the loop failed to manufacture an example), not a - // crash: return an empty trail so those slots still count against the accept-rate denominator. - let text: string - try { - text = await readFile(path, 'utf8') - } catch (err) { - if ((err as NodeJS.ErrnoException).code === 'ENOENT') return [] - throw err - } - return text - .split('\n') - .filter((l) => l.trim().length > 0) - .map((l) => JSON.parse(l) as TrailRow) -} - -function pctCI(x: { estimate: number; lower: number; upper: number }): string { - return `${(x.estimate * 100).toFixed(0)}% CI[${(x.lower * 100).toFixed(0)}%, ${(x.upper * 100).toFixed(0)}%]` -} - -function dist(d: Distribution): string { - return `n=${d.n} min=${d.min.toFixed(2)} median=${d.median.toFixed(2)} p90=${d.p90.toFixed(2)} max=${d.max.toFixed(2)} (mean=${d.mean.toFixed(2)})` -} - -function printReport(stats: PoweredStats, spendUsd: number): void { - console.log('\n══════════════════════════════════════════════════════════════════════════') - console.log(' POWERED ACCEPT-RATE — does the causal-challenger loop reliably discriminate?') - console.log('══════════════════════════════════════════════════════════════════════════\n') - console.log(` slots run : ${stats.totalSlots} (${stats.acceptedSlots} accepted)`) - console.log(` ACCEPTED-RATE : ${pctCI(stats.acceptRate)} ← the headline (Wilson 95%)`) - if (stats.challengerFailedSlots > 0) { - console.log( - ` challenger-failed : ${stats.challengerFailedSlots} slot(s) produced 0 attempts (infra/parse, not a discrimination reject)`, - ) - console.log( - ` accept-rate (producing): ${pctCI(stats.acceptRateAmongProducing)} (excl. challenger-failed slots)`, - ) - } - console.log('') - console.log(` best gap / slot : ${dist(stats.bestGapPerSlot)}`) - console.log(` plain gap / slot : ${dist(stats.plainGap)}`) - console.log( - ` gap-widening Δ : mean ${stats.widening.meanDelta >= 0 ? '+' : ''}${stats.widening.meanDelta.toFixed( - 3, - )} median ${stats.widening.medianDelta >= 0 ? '+' : ''}${stats.widening.medianDelta.toFixed(3)} ` + - `CI[${stats.widening.lower.toFixed(3)}, ${stats.widening.upper.toFixed(3)}] (paired bootstrap, n=${stats.widening.n})`, - ) - console.log('') - console.log(` weak score / attempt : ${dist(stats.weakScore)}`) - console.log(` strong score / attempt: ${dist(stats.strongScore)}`) - console.log('') - const c = stats.conditions - console.log(` accept-rule decomposition over ${c.nAttempts} quality-clean attempts:`) - console.log(` strong >= 0.65 : ${(c.strongHi * 100).toFixed(0)}%`) - console.log( - ` weak < 0.50 : ${(c.weakLo * 100).toFixed(0)}% ← the binding gate (weak must struggle)`, - ) - console.log(` gap >= 0.20 : ${(c.gapWide * 100).toFixed(0)}%`) - console.log(` all three (accept): ${(c.all * 100).toFixed(0)}%`) - console.log('') - console.log(' per-doc:') - for (const d of stats.perDoc) { - console.log( - ` ${d.tag.padEnd(14)} accepted ${d.accepted}/${d.target} mean best-gap ${d.meanBestGap.toFixed(3)}`, - ) - } - console.log(`\n total live spend : $${spendUsd.toFixed(4)}`) - const lo = stats.acceptRate.lower - const verdict = - lo > 0.15 - ? `WORKS AT POWER — accept-rate CI lower bound ${(lo * 100).toFixed(0)}% excludes ~0` - : stats.acceptRate.upper < 0.2 - ? 'COIN-FLIP / DOES NOT RELIABLY WORK — accept-rate CI sits near 0' - : 'UNDER-POWERED / MARGINAL — accept-rate CI still includes near-0; not settled' - console.log(`\n VERDICT: ${verdict}\n`) -} - -async function main(): Promise { - const apiKey = process.env.TANGLE_API_KEY ?? process.env.TANGLE_ROUTER_KEY - if (!apiKey) throw new Error('no TANGLE_API_KEY in env — run under dotenvx so the key is set') - - const docs = parseDocsEnv() - const slotsPerDoc = envInt('AUTODATA_SLOTS_PER_DOC', 14) - const samples = envInt('AUTODATA_SAMPLES', 4) - const maxRetries = envInt('AUTODATA_MAXRETRIES', 3) - - console.log('Autodata · POWERED accept-rate measurement') - console.log( - ` challenger/judge=${CHALLENGER_MODEL}/${JUDGE_MODEL} weak=${WEAK_SOLVER_MODEL} strong=${STRONG_SOLVER_MODEL}`, - ) - console.log( - ` ${docs.length} docs × ${slotsPerDoc} slots = ${docs.length * slotsPerDoc} slots · samples=${samples} maxRetries=${maxRetries}\n`, - ) - - // ── COST GATE: one cheap call per model, all must return non-empty content before the burn ── - const smoke = await smokeTestModels({ - apiKey, - models: [CHALLENGER_MODEL, WEAK_SOLVER_MODEL, STRONG_SOLVER_MODEL], - }) - for (const s of smoke) { - console.log( - ` ${s.ok ? 'ok ' : 'DEAD'} ${s.model.padEnd(28)} chars=${String(s.contentChars).padStart(4)} ` + - `finish=${s.finishReason ?? '?'} cost=$${s.costUsd.toFixed(5)} (${s.costSource})`, - ) - } - const dead = smoke.filter((s) => !s.ok) - if (dead.length > 0) { - throw new Error(`cost gate failed — empty content from: ${dead.map((d) => d.model).join(', ')}`) - } - - // ── Run K slots per doc (each doc writes its own incremental autopsy trail). ── - const trails: DocTrail[] = [] - let spendUsd = 0 - for (const doc of docs) { - const grounded = await groundDoc({ url: doc.url, focus: doc.focus }) - console.log( - `\n[${doc.tag}] grounded ${grounded.url} chunk=${grounded.chunkIndex}/${grounded.totalChunks} (${grounded.doc.length} chars)`, - ) - const attemptsPath = `data/powered-${doc.tag}-attempts.jsonl` - const result = await buildAutodataDataset({ - apiKey, - source: grounded, - outPath: `data/powered-${doc.tag}.jsonl`, - attemptsPath, - target: slotsPerDoc, - samples, - maxRetries, - }) - const docSpend = result.cost.summary().totalCostUsd - spendUsd += docSpend - console.log( - `[${doc.tag}] done · ${result.accepted.length}/${slotsPerDoc} accepted · ${result.attempts.length} attempts · $${docSpend.toFixed(4)}`, - ) - trails.push({ - tag: doc.tag, - url: doc.url, - target: slotsPerDoc, - rows: await readTrail(attemptsPath), - }) - } - - // ── Recompute everything from the on-disk trails (durable source of truth). ── - const stats = analyzeTrails(trails) - printReport(stats, spendUsd) - - // Emit the machine-readable result alongside the prose, for the doc + any re-analysis. - console.log(`RESULT_JSON ${JSON.stringify({ ...stats, spendUsd })}`) -} - -// Only auto-run when invoked directly (keeps `analyzeTrails` importable + unit-testable). -if (process.argv[1]?.endsWith('powered.ts')) { - main().catch((err) => { - console.error(err) - process.exit(1) - }) -} diff --git a/src/autodata/router-roles.ts b/src/autodata/router-roles.ts deleted file mode 100644 index 30b5769..0000000 --- a/src/autodata/router-roles.ts +++ /dev/null @@ -1,583 +0,0 @@ -/** - * The REAL two-tier roles for the Autodata loop, over the Tangle router. - * - * One transport seam — `routerChat` — POSTs `/chat/completions` and returns content + exact token - * usage + a per-call USD cost (the router's own cost when it returns one, else a documented - * rate-table estimate over the exact token counts; the source is flagged, never silently faked). It - * retries only TRANSIENT failures (the router's "upstream capacity, retry shortly" 503s, 429/502/504, - * network blips, per-request timeouts) with bounded backoff; a non-transient non-2xx fails loud. - * The four roles are materialized on top of it (all models env-overridable — see the constants below): - * • challenger (`deepseek-v4-flash`) → an `inProcessSandboxClient` that authors ONE NON-EXTRACTIVE - * causal/comparative/mechanism/thesis-consistency JSON example and parses it. - * • weak solver (`groq/llama-3.1-8b-instant`) / strong solver (`gemini-2.5-pro`) → answer workers. - * • judge (`deepseek-v4-flash`) → an `llmJudge` `JudgeConfig` whose transport is a `sandbox-sdk` - * ChatClient wrapping `routerChat`; the judge's own spend is recorded into the same `CostLedger` - * (the loop only aggregates challenger + solver spend, so the judge channel is recorded here). - * - * A reasoning model spends its budget on hidden reasoning and returns EMPTY visible content when - * `max_tokens` is too low (a glm/gemini footgun), so every call is floored and solvers fail loud on - * empty content rather than scoring a non-answer as 0 (which would corrupt the gap). - */ - -import { - type ChatCallOpts, - type ChatRequest, - type ChatResponse, - type CostLedger, - createChatClient, - llmJudge, -} from '@tangle-network/agent-eval' -import type { JudgeConfig } from '@tangle-network/agent-eval/campaign' -import { inProcessSandboxClient, type SandboxClient } from '@tangle-network/agent-runtime/loops' -import type { SandboxEvent } from '@tangle-network/sandbox' -import type { DataExample, SolverArtifact } from './data-creation-loop' - -export const DEFAULT_BASE_URL = 'https://router.tangle.tools/v1' - -// The proven-working tier on the live Tangle router, every id env-overridable: -// • weak solver `groq/llama-3.1-8b-instant` — an 8B whose knowledge cutoff predates the default -// grounding doc, so on non-memorized content it must REASON from the context (it can't recall), -// which is what lets a hard causal question separate it from a frontier solver. -// • strong solver `gemini-2.5-pro` — a frontier reasoner (a real wide capability gap vs the 8B). -// • challenger + judge `deepseek-v4-flash` — a capable, fast, RELIABLE author/grader that is a -// DIFFERENT family from both solvers (so the judge does not favour either solver's style). The -// brief's `glm-5.2` works too when the router has GLM capacity; swap it back via env when it is up. -// The solver tier is the experiment's load-bearing knob — a real strong>weak capability gap is -// required for any example to clear the discriminative bar. -export const WEAK_SOLVER_MODEL = process.env.AUTODATA_WEAK_MODEL ?? 'groq/llama-3.1-8b-instant' -export const STRONG_SOLVER_MODEL = process.env.AUTODATA_STRONG_MODEL ?? 'gemini-2.5-pro' -export const CHALLENGER_MODEL = process.env.AUTODATA_CHALLENGER_MODEL ?? 'deepseek-v4-flash' -export const JUDGE_MODEL = process.env.AUTODATA_JUDGE_MODEL ?? 'deepseek-v4-flash' - -interface ModelPrice { - /** USD per 1M input tokens. */ - inputPerM: number - /** USD per 1M output tokens. */ - outputPerM: number -} - -/** - * Rate table for the $ estimate. The TOKEN COUNTS are exact (read from the router's `usage`); these - * rates are the documented basis for converting them to dollars WHEN the router returns no per-call - * cost. They are estimates, not invoices — `routerChat` flags every call's `costSource` so a report - * can say how many calls were router-priced vs rate-estimated. - */ -const PRICE_TABLE: Record = { - 'glm-4.5-air': { inputPerM: 0.2, outputPerM: 0.6 }, - 'glm-4.6': { inputPerM: 0.6, outputPerM: 2.2 }, - 'glm-5.2': { inputPerM: 0.95, outputPerM: 3.0 }, - 'deepseek-v4-flash': { inputPerM: 0.27, outputPerM: 0.41 }, - // Wide-tier solver pair (a genuine small-vs-frontier capability gap). Approximate router rates. - 'groq/llama-3.1-8b-instant': { inputPerM: 0.05, outputPerM: 0.08 }, - 'gemini-2.5-pro': { inputPerM: 1.25, outputPerM: 10.0 }, - 'gemini-2.5-flash': { inputPerM: 0.3, outputPerM: 2.5 }, -} - -/** Per-call usage record surfaced to an optional sink for cost-provenance reporting. */ -export interface RouterCallRecord { - model: string - promptTokens: number - completionTokens: number - costUsd: number - costSource: 'router' | 'estimated' - finishReason: string | null -} - -export interface RouterChatInput { - apiKey: string - baseUrl?: string - model: string - messages: { role: 'system' | 'user' | 'assistant'; content: string }[] - maxTokens: number - temperature?: number - jsonMode?: boolean - signal?: AbortSignal - onCall?: (rec: RouterCallRecord) => void - /** Per-request deadline so a stalled upstream can't hang the loop. Default 60s. */ - timeoutMs?: number - /** Bounded retries on TRANSIENT failures (503/429/502/504, network, timeout). Default 4. */ - maxRetries?: number -} - -export interface RouterChatResult { - content: string - promptTokens: number - completionTokens: number - costUsd: number - costSource: 'router' | 'estimated' - finishReason: string | null - raw: Record -} - -/** glm spends its budget on hidden reasoning and returns empty content unless max_tokens is high. */ -function maxTokensFloor(model: string): number { - return /glm/i.test(model) ? 1500 : 512 -} - -/** Read a per-call cost the router may return, across the field names proxies use. */ -function routerReportedCost(body: Record): number | null { - const usage = (body.usage ?? {}) as Record - const candidates = [body._response_cost, body.cost, usage.cost, usage.total_cost] - for (const c of candidates) { - if (typeof c === 'number' && Number.isFinite(c) && c > 0) return c - } - return null -} - -function estimateCostUsd(model: string, promptTokens: number, completionTokens: number): number { - const price = PRICE_TABLE[model] - if (!price) { - // Fail loud: a model we route to but cannot price would emit a 0 that masquerades as "free". - throw new Error(`no price-table entry for model '${model}' — add it before routing live spend`) - } - return (promptTokens * price.inputPerM + completionTokens * price.outputPerM) / 1_000_000 -} - -/** - * One Tangle-router chat call. Fails loud on a non-2xx status. Returns the visible content, the - * exact prompt/completion token counts, and a USD cost (router-reported when present, else - * rate-estimated over the real token counts) with its source flagged. - */ -/** Transient upstream statuses the router itself tells us to "retry shortly" — safe to re-issue. */ -const transientStatuses = new Set([429, 502, 503, 504]) - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)) -} - -/** Exponential backoff with jitter: ~1s, 2s, 4s, 8s (capped 10s) — bounded by maxRetries. */ -function backoffMs(attempt: number): number { - return Math.min(10_000, 2 ** attempt * 1000) + Math.floor(Math.random() * 250) -} - -export async function routerChat(input: RouterChatInput): Promise { - const baseUrl = (input.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, '') - const max_tokens = Math.max(input.maxTokens, maxTokensFloor(input.model)) - const timeoutMs = input.timeoutMs ?? 60_000 - const maxRetries = input.maxRetries ?? 4 - const payload = JSON.stringify({ - model: input.model, - messages: input.messages, - max_tokens, - temperature: input.temperature ?? 0.2, - stream: false, - ...(input.jsonMode ? { response_format: { type: 'json_object' } } : {}), - }) - - // One non-streaming chat call, retried only on TRANSIENT failures (the router's own - // "upstream capacity, retry shortly" 503s, plus 429/502/504, network errors, and per-request - // timeouts). A non-transient non-2xx (401/400/404) fails loud immediately — never silently. - let body: Record | undefined - let lastTransient = '' - for (let attempt = 0; attempt <= maxRetries; attempt++) { - // Combine the caller's abort with a per-request deadline so a stalled upstream can't hang us. - const deadline = AbortSignal.timeout(timeoutMs) - const signal = input.signal ? AbortSignal.any([input.signal, deadline]) : deadline - let res: Response - try { - res = await fetch(`${baseUrl}/chat/completions`, { - method: 'POST', - headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${input.apiKey}` }, - signal, - body: payload, - }) - } catch (err) { - // The caller's own abort is final; a timeout/network blip is transient and retryable. - if (input.signal?.aborted) throw err - lastTransient = `network/timeout: ${String(err).slice(0, 120)}` - if (attempt < maxRetries) { - await sleep(backoffMs(attempt)) - continue - } - throw new Error( - `router call for ${input.model} failed after ${attempt + 1} tries — ${lastTransient}`, - ) - } - if (!res.ok) { - const detail = await res.text().catch(() => res.statusText) - if (transientStatuses.has(res.status) && attempt < maxRetries) { - lastTransient = `${res.status}: ${detail.slice(0, 120)}` - await sleep(backoffMs(attempt)) - continue - } - throw new Error(`router ${res.status} for ${input.model}: ${detail.slice(0, 400)}`) - } - body = (await res.json()) as Record - break - } - if (!body) { - throw new Error( - `router call for ${input.model} exhausted ${maxRetries + 1} tries — ${lastTransient}`, - ) - } - const choice = (body.choices as { message?: { content?: string }; finish_reason?: string }[])?.[0] - const usage = (body.usage ?? {}) as { prompt_tokens?: number; completion_tokens?: number } - const promptTokens = usage.prompt_tokens ?? 0 - const completionTokens = usage.completion_tokens ?? 0 - const reported = routerReportedCost(body) - const costUsd = reported ?? estimateCostUsd(input.model, promptTokens, completionTokens) - const costSource: 'router' | 'estimated' = reported !== null ? 'router' : 'estimated' - const finishReason = choice?.finish_reason ?? null - input.onCall?.({ - model: input.model, - promptTokens, - completionTokens, - costUsd, - costSource, - finishReason, - }) - return { - content: choice?.message?.content ?? '', - promptTokens, - completionTokens, - costUsd, - costSource, - finishReason, - raw: body, - } -} - -// ── Parsing the challenger's JSON example ───────────────────────────────────────────────────── - -/** Extract the first balanced top-level JSON object from a model response (handles ```json fences). */ -function extractJsonObject(text: string): string | null { - const fenced = /```(?:json)?\s*([\s\S]*?)```/i.exec(text) - const body = fenced ? (fenced[1] ?? '') : text - const start = body.indexOf('{') - if (start < 0) return null - let depth = 0 - let inString = false - let escaped = false - for (let i = start; i < body.length; i++) { - const ch = body[i] - if (escaped) { - escaped = false - continue - } - if (ch === '\\') { - escaped = true - continue - } - if (ch === '"') inString = !inString - if (inString) continue - if (ch === '{') depth++ - else if (ch === '}') { - depth-- - if (depth === 0) return body.slice(start, i + 1) - } - } - return null -} - -/** Parse a challenger response into a `DataExample`, or throw loud (the loop refines on the error). */ -export function parseDataExample(text: string): DataExample { - const json = extractJsonObject(text) - if (!json) throw new Error('challenger response contained no JSON object') - const parsed = JSON.parse(json) as Record - const rubric = parsed.rubric - if ( - typeof parsed.context !== 'string' || - typeof parsed.question !== 'string' || - typeof parsed.reference !== 'string' || - !Array.isArray(rubric) - ) { - throw new Error('challenger JSON missing a required field (context/question/reference/rubric)') - } - return { - context: parsed.context, - question: parsed.question, - reference: parsed.reference, - rubric: rubric.map((r) => String(r)), - } -} - -// ── The roles ───────────────────────────────────────────────────────────────────────────────── - -// The non-extractive challenger. The prior loop nulled because the context LEAKED the answer: -// the question was recall/lookup, so an 8B read it out as well as a frontier model and the gap -// collapsed to ~0. The fix is the paper's: ask CAUSAL / COMPARATIVE / MECHANISM / THESIS-CONSISTENCY -// questions whose answer is an INFERENCE, and withhold the conclusion from the context the solver -// sees ("problems only, no solution") so the answer must be DERIVED, never quoted. -const challengerSystem = - 'You are an exam author. From a source excerpt, write ONE hard question that tests REASONING, ' + - 'not recall.\n\n' + - 'The question MUST be exactly one of these kinds:\n' + - ' • CAUSAL — "why does X fail / what breaks if Y is omitted".\n' + - ' • COMPARATIVE — "how does the tradeoff of X differ from Y, and why".\n' + - ' • MECHANISM — "walk through how X produces Y; what fails if a step is skipped".\n' + - ' • THESIS-CONSISTENCY — "which of two explanations the text offers is more consistent with its ' + - 'overall conclusion, and how would the other undermine it".\n' + - 'It must NEVER be recall / lookup / definition / enumeration ("what is X", "which header", ' + - '"list the steps", "name the ...").\n\n' + - 'ANTI-LEAKAGE (mandatory):\n' + - ' • The CONTEXT must contain ONLY the premises/evidence the solver needs. It MUST NOT contain a ' + - 'sentence that states the answer or the conclusion — the answer has to be DERIVED from the ' + - 'premises, not quotable from the context.\n' + - ' • The REFERENCE is the correct DERIVED conclusion plus its reasoning chain. It must NOT appear ' + - 'verbatim in the context.\n\n' + - 'The RUBRIC is 2-3 criteria that reward the REASONING STEPS (e.g. "identifies that X depends on ' + - 'Y", "explains why removing Y causes the failure", "ties it to the stated conclusion") — never ' + - '"mentions the keyword".\n\n' + - 'Return STRICT JSON and nothing else: ' + - '{"context": string, "question": string, "reference": string, "rubric": string[] }.' - -// The recall / extractive challenger — the prior (nulling) behavior, kept ONLY as the calibration -// baseline. It writes a normal question answerable straight from the excerpt, so the answer is in -// the context and a small model reads it out as well as a frontier one. The causal vs recall gap is -// the calibration that proves the lever. -const recallChallengerSystem = - 'You write ONE exam question from a source excerpt. Provide a short CONTEXT excerpt the question ' + - 'is answerable from, the QUESTION, the REFERENCE answer, and a 2-3 item RUBRIC. ' + - 'Return STRICT JSON and nothing else: ' + - '{"context": string, "question": string, "reference": string, "rubric": string[] }.' - -/** Which challenger prompt to materialize: the non-extractive causal author, or the recall baseline. */ -export type ChallengerStyle = 'causal' | 'recall' - -function challengerSystemFor(style: ChallengerStyle): string { - return style === 'recall' ? recallChallengerSystem : challengerSystem -} - -// The reasoning judge. It sees the CONTEXT the solver saw, so it can tell a derived answer from one -// that merely restates the context. The `reasoning` dimension IS the negative criterion: an answer -// that paraphrases the context without deriving the conclusion scores near 0 there — which is what -// pulls a recall-style weak answer below the strong model's derivation and opens the gap. -const judgeSystem = - 'You grade a candidate ANSWER to a REASONING question. You are given the CONTEXT the solver was ' + - 'shown, the REFERENCE answer, and the RUBRIC.\n' + - 'Return JSON {"dimensions":{"rubric_coverage":N,"correctness":N,"reasoning":N},"notes":"..."} ' + - 'with each score in [0,1].\n' + - ' • rubric_coverage = fraction of the rubric criteria the answer genuinely satisfies.\n' + - ' • correctness = how well the DERIVED conclusion agrees with the reference.\n' + - ' • reasoning = quality of the DERIVATION. Score HIGH only if the answer works through WHY/HOW ' + - 'from the premises. Score near 0 if it merely RESTATES or QUOTES the context, asserts the ' + - 'conclusion without justifying it, or is vague.\n' + - 'NEGATIVE CRITERION: an answer that just paraphrases the context without deriving the conclusion ' + - 'is a recall answer — it must score LOW on reasoning AND correctness, no matter how many keywords ' + - 'it echoes. Be strict.' - -export interface RouterRolesConfig { - apiKey: string - baseUrl?: string - challengerModel?: string - weakModel?: string - strongModel?: string - judgeModel?: string - /** Challenger prompt: 'causal' (non-extractive, default) or 'recall' (the calibration baseline). */ - challengerStyle?: ChallengerStyle - /** Judge spend is recorded here directly (the loop captures only challenger + solver spend). */ - ledger: CostLedger - /** Optional sink for every router call's cost provenance. */ - onCall?: (rec: RouterCallRecord) => void -} - -export interface AutodataRoles { - challenger: SandboxClient - weakSolver: SandboxClient - strongSolver: SandboxClient - judge: JudgeConfig -} - -function solverClient(cfg: RouterRolesConfig, model: string): SandboxClient { - return inProcessSandboxClient({ - onPrompt: async (prompt, ctx): Promise => { - const r = await routerChat({ - apiKey: cfg.apiKey, - baseUrl: cfg.baseUrl, - model, - messages: [{ role: 'user', content: prompt }], - // Reasoning models (gemini-2.5-pro, glm-5.2, …) spend their budget on hidden reasoning and - // emit EMPTY visible content when it is too low — at 1024 a "strong" solver returned nothing - // and was scored 0, manufacturing a false negative strong−weak gap. Give every solver room - // for reasoning + a full answer. - maxTokens: 8000, - signal: ctx.signal, - onCall: cfg.onCall, - }) - // Fail loud: an empty answer is a measurement failure, not a score of 0. Letting empty → 0 - // silently corrupts the strong/weak gap (the whole signal), so refuse to score it. - if (r.content.trim() === '') { - throw new Error( - `solver '${model}' returned empty visible content (likely all tokens spent on hidden ` + - `reasoning) — raise maxTokens or pick a non-reasoning solver; refusing to score it as 0`, - ) - } - return [ - { - type: 'llm_call', - data: { - model, - tokensIn: r.promptTokens, - tokensOut: r.completionTokens, - costUsd: r.costUsd, - }, - }, - { type: 'result', data: { result: { answer: r.content } } }, - ] - }, - }) -} - -function challengerClient(cfg: RouterRolesConfig): SandboxClient { - const model = cfg.challengerModel ?? CHALLENGER_MODEL - const system = challengerSystemFor(cfg.challengerStyle ?? 'causal') - return inProcessSandboxClient({ - onPrompt: async (prompt, ctx): Promise => { - const r = await routerChat({ - apiKey: cfg.apiKey, - baseUrl: cfg.baseUrl, - model, - messages: [ - { role: 'system', content: system }, - { role: 'user', content: prompt }, - ], - maxTokens: 1500, - jsonMode: true, - signal: ctx.signal, - onCall: cfg.onCall, - }) - const example = parseDataExample(r.content) - return [ - { - type: 'llm_call', - data: { - model, - tokensIn: r.promptTokens, - tokensOut: r.completionTokens, - costUsd: r.costUsd, - }, - }, - { type: 'result', data: { result: example } }, - ] - }, - }) -} - -function rubricJudge(cfg: RouterRolesConfig): JudgeConfig { - const judgeModel = cfg.judgeModel ?? JUDGE_MODEL - const chat = createChatClient({ - transport: 'sandbox-sdk', - defaultModel: judgeModel, - chat: async (req: ChatRequest, opts?: ChatCallOpts): Promise => { - const model = req.model ?? judgeModel - const messages = req.messages.map((m) => ({ - role: m.role, - content: - typeof m.content === 'string' - ? m.content - : m.content.map((p) => (p.type === 'text' ? p.text : '')).join('\n'), - })) - const r = await routerChat({ - apiKey: cfg.apiKey, - baseUrl: cfg.baseUrl, - model, - messages, - maxTokens: req.maxTokens ?? 1500, - temperature: req.temperature, - jsonMode: req.jsonMode, - signal: opts?.signal, - onCall: cfg.onCall, - }) - cfg.ledger.record({ - model, - channel: 'judge', - usage: { inputTokens: r.promptTokens, outputTokens: r.completionTokens }, - actualCostUsd: r.costUsd, - tags: { role: 'judge' }, - }) - return { - content: r.content, - usage: { - promptTokens: r.promptTokens, - completionTokens: r.completionTokens, - totalTokens: r.promptTokens + r.completionTokens, - }, - costUsd: r.costUsd, - model, - durationMs: 0, - finishReason: r.finishReason, - contentEmpty: r.content.trim() === '', - raw: r.raw, - } - }, - }) - - return llmJudge('autodata-rubric-judge', judgeSystem, { - chat, - maxTokens: 1500, - dimensions: [ - { - key: 'rubric_coverage', - description: 'fraction of the rubric criteria the answer satisfies', - }, - { key: 'correctness', description: 'agreement with the reference answer' }, - { - key: 'reasoning', - description: - 'quality of the derivation; near 0 if the answer merely restates/quotes the context', - }, - ], - scale: 'unit', - // The judge sees the CONTEXT so it can distinguish a derived answer from a restated one (the - // negative criterion). Without it, a paraphrase of the context is indistinguishable from real - // reasoning and the gap stays closed. - renderUser: ({ artifact }) => - `CONTEXT THE SOLVER WAS GIVEN:\n${artifact.example.context}\n\n` + - `REFERENCE ANSWER:\n${artifact.example.reference}\n\n` + - `RUBRIC:\n${artifact.example.rubric.map((r, i) => `${i + 1}. ${r}`).join('\n')}\n\n` + - `CANDIDATE ANSWER:\n${artifact.answer}`, - }) -} - -/** Materialize all four live roles over the Tangle router. */ -export function buildAutodataRoles(cfg: RouterRolesConfig): AutodataRoles { - return { - challenger: challengerClient(cfg), - // weak/strong solvers + judge are style-independent; only the challenger prompt changes. - weakSolver: solverClient(cfg, cfg.weakModel ?? WEAK_SOLVER_MODEL), - strongSolver: solverClient(cfg, cfg.strongModel ?? STRONG_SOLVER_MODEL), - judge: rubricJudge(cfg), - } -} - -export interface SmokeResult { - model: string - ok: boolean - contentChars: number - finishReason: string | null - costUsd: number - costSource: 'router' | 'estimated' -} - -/** - * The cost gate: one cheap call per model, asserting non-empty content, BEFORE the loop burn. - * Returns a row per model so the caller can fail loud if any tier is dead. - */ -export async function smokeTestModels(cfg: { - apiKey: string - baseUrl?: string - models?: string[] - signal?: AbortSignal -}): Promise { - const models = cfg.models ?? [CHALLENGER_MODEL, WEAK_SOLVER_MODEL, STRONG_SOLVER_MODEL] - const rows: SmokeResult[] = [] - for (const model of models) { - const r = await routerChat({ - apiKey: cfg.apiKey, - baseUrl: cfg.baseUrl, - model, - messages: [{ role: 'user', content: 'Reply with the single word: ready.' }], - maxTokens: 32, - signal: cfg.signal, - }) - rows.push({ - model, - ok: r.content.trim().length > 0, - contentChars: r.content.trim().length, - finishReason: r.finishReason, - costUsd: r.costUsd, - costSource: r.costSource, - }) - } - return rows -} diff --git a/src/autodata/run.ts b/src/autodata/run.ts deleted file mode 100644 index 928b69c..0000000 --- a/src/autodata/run.ts +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Autodata — the LIVE runnable: cost-gate the three models, ground on a REAL arXiv document, run the - * agentic data-creation loop with the real two-tier solvers, and report the empirical strong/weak - * gap (plain first-draft vs loop-accepted), the cost split by role, and the JSONL dataset path. - * - * Run (key never printed): - * dotenvx run -f /home/drew/company/devops/secrets/agent-state.env -- \ - * pnpm tsx src/autodata/run.ts - * - * Env knobs: AUTODATA_URL, AUTODATA_FOCUS, AUTODATA_TARGET, AUTODATA_SAMPLES, AUTODATA_MAXRETRIES, - * AUTODATA_OUT, AUTODATA_ATTEMPTS (per-attempt autopsy JSONL), - * AUTODATA_{WEAK,STRONG,CHALLENGER,JUDGE}_MODEL, TANGLE_API_KEY (or TANGLE_ROUTER_KEY). - */ - -import { buildAutodataDataset } from './build-dataset' -import { DEFAULT_SOURCE_URL, groundDoc } from './grounding' -import { - CHALLENGER_MODEL, - STRONG_SOLVER_MODEL, - smokeTestModels, - WEAK_SOLVER_MODEL, -} from './router-roles' - -function envInt(name: string, fallback: number): number { - const raw = process.env[name] - if (!raw) return fallback - const n = Number.parseInt(raw, 10) - if (!Number.isFinite(n) || n <= 0) throw new Error(`${name}='${raw}' is not a positive integer`) - return n -} - -function fmt(x: number | null, digits = 3): string { - return x === null ? 'n/a' : x.toFixed(digits) -} - -async function main(): Promise { - const apiKey = process.env.TANGLE_API_KEY ?? process.env.TANGLE_ROUTER_KEY - if (!apiKey) throw new Error('no TANGLE_API_KEY in env — run under dotenvx so the key is set') - - const url = process.env.AUTODATA_URL ?? DEFAULT_SOURCE_URL - const focus = process.env.AUTODATA_FOCUS ?? 'attention' - const target = envInt('AUTODATA_TARGET', 3) - const samples = envInt('AUTODATA_SAMPLES', 3) - const maxRetries = envInt('AUTODATA_MAXRETRIES', 4) - const outPath = process.env.AUTODATA_OUT ?? 'data/autodata-dataset.jsonl' - const attemptsPath = process.env.AUTODATA_ATTEMPTS ?? 'data/autodata-attempts.jsonl' - - // ── 1. COST GATE: one cheap call per model, all must return non-empty content before the burn ── - console.log('Autodata · cost gate (one call per model)\n') - const smoke = await smokeTestModels({ - apiKey, - models: [CHALLENGER_MODEL, WEAK_SOLVER_MODEL, STRONG_SOLVER_MODEL], - }) - for (const s of smoke) { - console.log( - ` ${s.ok ? 'ok ' : 'DEAD'} ${s.model.padEnd(28)} chars=${String(s.contentChars).padStart(4)} ` + - `finish=${s.finishReason ?? '?'} cost=$${s.costUsd.toFixed(5)} (${s.costSource})`, - ) - } - const dead = smoke.filter((s) => !s.ok) - if (dead.length > 0) { - throw new Error(`cost gate failed — empty content from: ${dead.map((d) => d.model).join(', ')}`) - } - - // ── 2. Ground on a REAL document ── - const grounded = await groundDoc({ url, focus }) - console.log( - `\nGrounded on ${grounded.url}\n section='${grounded.headingPath}' chunk=${grounded.chunkIndex}/${grounded.totalChunks} ` + - `(${grounded.doc.length} chars, updated ${grounded.sourceUpdatedAt})`, - ) - console.log(` excerpt: ${grounded.doc.slice(0, 200).replace(/\s+/g, ' ')}...`) - - // ── 3. Run the loop with real two-tier solvers ── - console.log( - `\nManufacturing up to ${target} discriminating example(s) · samples=${samples} maxRetries=${maxRetries}\n` + - ` challenger/judge=${CHALLENGER_MODEL} weak=${WEAK_SOLVER_MODEL} strong=${STRONG_SOLVER_MODEL}`, - ) - const result = await buildAutodataDataset({ - apiKey, - source: grounded, - outPath, - attemptsPath, - target, - samples, - maxRetries, - }) - - // ── 4. The accepted set ── - console.log(`\n— Accepted examples (${result.accepted.length}/${target}) —`) - for (const [i, ex] of result.accepted.entries()) { - console.log(`\n [${i}] Q: ${ex.example.question}`) - console.log( - ` weak=${ex.weakScore.toFixed(2)} strong=${ex.strongScore.toFixed(2)} gap=${ex.gap.toFixed(2)}`, - ) - console.log(` ${ex.decision.reason}`) - } - - // ── 4b. Autopsy: the single widest-gap attempt, with BOTH solvers' actual answers ── - // A gap number is only a finding if you can read why it opened. Show the strongest discrimination - // we saw (highest gap, accepted or not) so a human can confirm it is real reasoning, not an - // artifact: the weak model should genuinely fail the reasoning and the strong model get it. - const best = result.attempts.filter((a) => a.qualityOk).sort((a, b) => b.gap - a.gap)[0] - if (best) { - const oneLine = (s: string): string => s.replace(/\s+/g, ' ').trim() - console.log('\n— Autopsy: widest-gap attempt (read the answers, confirm real discrimination) —') - console.log(` Q: ${oneLine(best.example.question)}`) - console.log(` reference: ${oneLine(best.example.reference).slice(0, 240)}`) - console.log( - ` gap=${best.gap.toFixed(2)} (${best.decision.accept ? 'ACCEPTED' : 'rejected'}: ${best.decision.reason})`, - ) - console.log(` WEAK mean=${best.weak.mean.toFixed(2)}`) - for (const [i, s] of best.weak.samples.entries()) - console.log(` [w${i} score=${s.score.toFixed(2)}] ${oneLine(s.answer).slice(0, 220)}`) - console.log(` STRONG mean=${best.strong.mean.toFixed(2)}`) - for (const [i, s] of best.strong.samples.entries()) - console.log(` [s${i} score=${s.score.toFixed(2)}] ${oneLine(s.answer).slice(0, 220)}`) - } - - // ── 5. The empirical calibration (paper Table 1) ── - console.log('\n— Calibration: plain first-draft gap vs agentic loop-accepted gap —') - console.log( - ` plain (first-draft questions, n=${result.plainGaps.length}) mean gap = ${fmt(result.plainGapMean)}`, - ) - console.log( - ` agentic (loop-accepted questions, n=${result.agenticGaps.length}) mean gap = ${fmt(result.agenticGapMean)}`, - ) - console.log( - ` refined (best gap reached per slot, n=${result.refinedGaps.length}) mean gap = ${fmt(result.refinedGapMean)}`, - ) - // The honest comparison: plain first-draft gap vs the best the refinement reached. Acceptance is - // strict (gap >= 0.20); refined-vs-plain shows whether the fold widened the gap at all. - if (result.plainGapMean !== null && result.refinedGapMean !== null) { - const delta = result.refinedGapMean - result.plainGapMean - console.log( - ` Δ (refined − plain) = ${delta >= 0 ? '+' : ''}${delta.toFixed(3)} ` + - (delta >= 0.1 - ? '→ the loop WIDENS the strong/weak gap (empirical Table-1 direction)' - : '→ NO meaningful widening on these real models (honest null)'), - ) - } else { - console.log(' (insufficient data to compare — see accepted count)') - } - if (result.accepted.length === 0) { - console.log( - ` NOTE: 0 examples cleared the discriminative accept bar — ${WEAK_SOLVER_MODEL} and ` + - `${STRONG_SOLVER_MODEL} did not separate on these questions (see the autopsy trail).`, - ) - } - - // ── 6. Cost split by role ── - const summary = result.cost.summary() - console.log('\n— Cost (CostLedger, by role) —') - console.log( - ` total: $${summary.totalCostUsd.toFixed(4)} over ${summary.totalCalls} recorded loops/calls` + - (summary.fullyPriced - ? ' (fully priced)' - : ` (unpriced models: ${summary.unpricedModels.join(', ')})`), - ) - for (const ch of summary.byChannel) { - console.log(` ${ch.channel.padEnd(14)} $${ch.costUsd.toFixed(4)} (${ch.calls} loops/calls)`) - } - if (result.costPerExampleUsd !== null) { - console.log(` cost per accepted example: $${result.costPerExampleUsd.toFixed(4)}`) - } - console.log( - ` call provenance: ${result.callProvenance.router} router-priced, ${result.callProvenance.estimated} rate-estimated`, - ) - - console.log(`\n— Dataset — ${result.rows.length} row(s) written to ${result.outPath}`) - if (result.attemptsPath) { - console.log( - `— Autopsy trail — ${result.attempts.length} attempt(s) (accepted + rejected) at ${result.attemptsPath}`, - ) - } -} - -main().catch((err) => { - console.error(err) - process.exit(1) -}) diff --git a/src/index.ts b/src/index.ts index 23b0e3c..0618d7b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,7 +28,6 @@ export * from './rag-improvement-loop' export * from './release' export * from './research-driving-driver' export * from './research-loop' -export * from './research-supervisor' export * from './retrieval-eval' export * from './schemas' export * from './search' diff --git a/src/kb-improvement.ts b/src/kb-improvement.ts index 5bc6336..a531a18 100644 --- a/src/kb-improvement.ts +++ b/src/kb-improvement.ts @@ -17,6 +17,8 @@ import { type RagKnowledgeImprovementPhase, type RagKnowledgeImprovementPhaseResult, type RagKnowledgeResearchOptions, + type RagKnowledgeUpdateInput, + type RagKnowledgeUpdateResult, type RagPromotionResult, type RunRagKnowledgeImprovementLoopOptions, type RunRagKnowledgeImprovementLoopResult, @@ -24,7 +26,6 @@ import { } from './rag-improvement-loop' import { readinessFor } from './readiness-helpers' import type { RunKnowledgeResearchLoopOptions } from './research-loop' -import { type ResearchSupervisorOptions, runResearchSupervisor } from './research-supervisor' import type { RetrievalConfig, RunRetrievalImprovementLoopOptions } from './retrieval-eval' import { initKnowledgeBase, layoutFor } from './store' import type { KnowledgeIndex } from './types' @@ -128,6 +129,20 @@ export interface KnowledgeImprovementRetrievalOptions runDir?: RunRetrievalImprovementLoopOptions['runDir'] } +export interface KnowledgeImprovementUpdateInput extends RagKnowledgeUpdateInput { + runId: string + iteration: number + candidateId: string + root: string + baselineRoot: string + candidateRoot: string + baseHash: string +} + +export type KnowledgeImprovementUpdate = ( + input: KnowledgeImprovementUpdateInput, +) => Promise | RagKnowledgeUpdateResult + export interface KnowledgeImprovementOptions { root: string goal: string @@ -146,14 +161,10 @@ export interface KnowledgeImprovementOptions { kbQuality?: KnowledgeBaseQualityOptions step?: RunKnowledgeResearchLoopOptions['step'] knowledgeResearch?: Omit - supervisor?: Omit< - ResearchSupervisorOptions, - 'root' | 'goal' | 'readinessSpecs' | 'readinessTaskId' | 'readiness' - > retrieval?: KnowledgeImprovementRetrievalOptions diagnose?: NonNullable acquireKnowledge?: NonNullable - updateKnowledge?: NonNullable + updateKnowledge?: KnowledgeImprovementUpdate evaluateAnswers?: NonNullable decidePromotion?: NonNullable enabledPhases?: readonly RagKnowledgeImprovementPhase[] @@ -289,7 +300,7 @@ export async function improveKnowledgeBase( }) } - lifecycle = await runCandidateLifecycle(runDir, candidate, options, now) + lifecycle = await runCandidateLifecycle(runDir, runId, candidate, options, now) candidate.status = 'candidate-ready' candidate.updatedAt = now().toISOString() state.status = 'candidate-ready' @@ -375,21 +386,18 @@ function assertKnowledgeImprovementOptions(options: KnowledgeImprovementOptions) } const updateDrivers = [ Boolean(options.step ?? options.knowledgeResearch), - Boolean(options.supervisor), Boolean(options.updateKnowledge), ].filter(Boolean).length if (updateDrivers > 1) { throw new Error( - 'improveKnowledgeBase accepts only one knowledge-update driver: knowledgeResearch, supervisor, or updateKnowledge', + 'improveKnowledgeBase accepts only one knowledge-update driver: knowledgeResearch or updateKnowledge', ) } - if (options.supervisor && (!options.readinessSpecs || options.readinessSpecs.length === 0)) { - throw new Error('improveKnowledgeBase supervisor mode requires readinessSpecs') - } } async function runCandidateLifecycle( runDir: string, + runId: string, candidate: KnowledgeImprovementCandidateRecord, options: KnowledgeImprovementOptions, now: () => Date, @@ -401,7 +409,7 @@ async function runCandidateLifecycle( goal: options.goal, acquireKnowledge: options.acquireKnowledge, knowledgeResearch: candidateKnowledgeResearchOptions(candidate.candidateRoot, options), - updateKnowledge: candidateUpdateHook(candidate.candidateRoot, options), + updateKnowledge: candidateUpdateHook(runId, candidate, options), enabledPhases: selectedStagePhases(options, UPDATE_PHASES), requiredPhases: selectedStageRequiredPhases(options, UPDATE_PHASES), signal: options.signal, @@ -458,26 +466,22 @@ function candidateKnowledgeResearchOptions( } function candidateUpdateHook( - candidateRoot: string, + runId: string, + candidate: KnowledgeImprovementCandidateRecord, options: KnowledgeImprovementOptions, ): RunRagKnowledgeImprovementLoopOptions['updateKnowledge'] { - if (options.updateKnowledge) return options.updateKnowledge - if (!options.supervisor) return undefined - return async () => { - await runResearchSupervisor({ - ...options.supervisor!, - root: candidateRoot, - goal: options.goal, - readinessSpecs: options.readinessSpecs ?? [], - readinessTaskId: options.readinessTaskId, - readiness: options.readiness, + if (!options.updateKnowledge) return undefined + return (input) => + options.updateKnowledge!({ + ...input, + runId, + iteration: candidate.iteration, + candidateId: candidate.candidateId, + root: candidate.candidateRoot, + baselineRoot: options.root, + candidateRoot: candidate.candidateRoot, + baseHash: candidate.baseHash, }) - return { - applied: true, - summary: 'research supervisor completed', - metadata: { supervised: true }, - } - } } function shouldRunUpdateStage(options: KnowledgeImprovementOptions): boolean { @@ -487,7 +491,6 @@ function shouldRunUpdateStage(options: KnowledgeImprovementOptions): boolean { options.acquireKnowledge || options.step || options.knowledgeResearch || - options.supervisor || options.updateKnowledge || selectedStageRequiredPhases(options, UPDATE_PHASES).length > 0, ) diff --git a/src/profiles/index.ts b/src/profiles/index.ts deleted file mode 100644 index c38285e..0000000 --- a/src/profiles/index.ts +++ /dev/null @@ -1,24 +0,0 @@ -/** - * @experimental - * - * Pre-built `AgentRunSpec` + output adapter + validator bundles for - * knowledge-focused agent roles. Each preset bundles a sandbox-SDK - * `AgentProfile`, a task-to-prompt formatter, an output adapter, and a - * per-task validator constructor — all of the pieces `runLoop` needs to - * drive a topology around `@tangle-network/agent-runtime/loops`. - */ - -export type { - KnowledgeItem, - KnowledgeUpdate, - MultiHarnessResearcherFanoutOptions, - ResearcherProfileOptions, - ResearchOutput, - ResearchSource, - ResearchTask, -} from './researcher' -export { - createResearcherValidator, - multiHarnessResearcherFanout, - researcherProfile, -} from './researcher' diff --git a/src/profiles/researcher.ts b/src/profiles/researcher.ts deleted file mode 100644 index a16868e..0000000 --- a/src/profiles/researcher.ts +++ /dev/null @@ -1,614 +0,0 @@ -/** - * @experimental - * - * `researcherProfile` — opinionated preset for source-grounded research - * tasks. The agent is told to: - * - bound its work to a single `knowledgeNamespace` - * - emit `items[]` carrying provenance + confidence - * - emit `citations[]` linking quotes back to source urls - * - emit `proposedWrites[]` — never call materialize itself - * - describe `gaps` it could not answer - * - * The profile is stateless and agent-agnostic. `harness` selects the - * sandbox-SDK backend. For heterogeneous fanout, use - * `multiHarnessResearcherFanout`. - * - * Propose-don't-apply: the profile NEVER writes to the knowledge base. - * It produces `proposedWrites: KnowledgeUpdate[]` in the output. The - * caller (gtm-agent, journey-eval, user) decides whether to feed those - * updates through `applyKnowledgeWriteBlocks` / a KbStore put. - * - * Namespace isolation: every `KnowledgeItem` + `KnowledgeUpdate` in the - * output carries `namespace`. The validator hard-fails when any item - * touches a namespace other than `task.knowledgeNamespace`. - */ - -import type { - AgentProfile, - AgentRunSpec, - DefaultVerdict, - Driver, - OutputAdapter, - SandboxEvent, - Validator, -} from '@tangle-network/agent-runtime/loops' - -/** @experimental */ -export type ResearchSource = 'web' | 'corpus' | 'twitter' | 'github' | 'docs' - -/** @experimental */ -export interface ResearchTask { - /** The research question to answer. */ - question: string - /** Bound: e.g. "audience for cpg-founder ICP". */ - scope?: string - /** Multi-tenant scope (customer-id, workspace-id). Validator enforces. */ - knowledgeNamespace: string - sources?: ResearchSource[] - recencyWindow?: { since?: Date; until?: Date } - maxItems?: number - /** Per-item minimum confidence in [0, 1]. Validator scores recall vs this. */ - minConfidence?: number -} - -/** - * Knowledge item emitted by the researcher. - * - * Profile-local type. When agent-knowledge promotes `KnowledgeClaim` → - * top-level `KnowledgeItem` substrate-wide, these fields collapse 1:1. - * - * @experimental - */ -export interface KnowledgeItem { - id: string - /** Multi-tenant scope. MUST equal `task.knowledgeNamespace`. */ - namespace: string - /** The factual claim, in the researcher's words. */ - claim: string - /** Provenance — at least one entry required. */ - evidence: Array<{ source: string; quote?: string; url?: string; capturedAt: number }> - /** Researcher's self-reported confidence in [0, 1]. */ - confidence: number - /** Prior item ids this supersedes (chain). */ - supersedes?: string[] - /** Set if the agent is retracting an earlier item. Unix ms. */ - retractedAt?: number - authoredBy: { kind: 'human' | 'agent'; id: string } -} - -/** - * A proposed write to the knowledge base. The profile does NOT apply - * these — the caller decides. - * - * @experimental - */ -export type KnowledgeUpdate = - | { kind: 'insert'; namespace: string; item: KnowledgeItem } - | { kind: 'supersede'; namespace: string; previousId: string; item: KnowledgeItem } - | { kind: 'retract'; namespace: string; itemId: string; reason: string } - -/** - * Researcher output. Required fields are typed; optional fields preserve - * the agent's free-form intelligence (`notes`, `raw`). The validator - * enforces the typed minimum. - * - * @experimental - */ -export interface ResearchOutput { - items: KnowledgeItem[] - citations: Array<{ url: string; quote: string; confidence: number }> - proposedWrites: KnowledgeUpdate[] - gaps?: string[] - notes?: string - /** Anything the agent emitted beyond the typed fields. */ - raw?: unknown -} - -/** @experimental */ -export interface ResearcherProfileOptions { - /** Sandbox-SDK backend.type. Default `'opencode/zai-coding-plan/glm-5.1'`. */ - harness?: string - /** Default model id passed in `AgentProfile.model.default`. */ - model?: string - /** Custom system prompt replacement. Default = built-in researcher preset. */ - systemPrompt?: string - /** Stable name for `AgentRunSpec.name`. Default = `researcher-${harness}`. */ - name?: string - /** - * Default 0.7. Minimum (citations with quote) / items ratio for `valid=true`. - * Below this floor, citation_density scores < 1 and the item set is gated. - */ - citationDensityMin?: number -} - -const DEFAULT_HARNESS = 'opencode/zai-coding-plan/glm-5.1' -const DEFAULT_CITATION_DENSITY_MIN = 0.7 - -/** @experimental */ -export function researcherProfile( - options: ResearcherProfileOptions & { task?: ResearchTask } = {}, -): { - profile: AgentProfile - taskToPrompt: (task: ResearchTask) => string - output: OutputAdapter - validator: Validator - agentRunSpec: AgentRunSpec -} { - const harness = options.harness ?? DEFAULT_HARNESS - const name = options.name ?? `researcher-${harness}` - const systemPrompt = options.systemPrompt ?? DEFAULT_RESEARCHER_SYSTEM_PROMPT - const citationDensityMin = options.citationDensityMin ?? DEFAULT_CITATION_DENSITY_MIN - const profile: AgentProfile = { - name, - description: "Source-grounded research agent. Propose-don't-apply.", - prompt: { systemPrompt }, - model: options.model ? { default: options.model } : undefined, - tools: { web_search: true, fs: true, shell: true }, - metadata: { backendType: harness, role: 'researcher' }, - } - const output: OutputAdapter = { parse: parseResearcherEvents } - const validator: Validator = options.task - ? createResearcherValidator(options.task, { citationDensityMin }) - : createResearcherValidator( - { question: '', knowledgeNamespace: '' }, - { citationDensityMin, namespaceCheck: false }, - ) - const agentRunSpec: AgentRunSpec = { - name, - profile, - taskToPrompt: formatResearcherPrompt, - } - return { profile, taskToPrompt: formatResearcherPrompt, output, validator, agentRunSpec } -} - -/** @experimental */ -export interface MultiHarnessResearcherFanoutOptions { - /** Backend.type identifiers, one per parallel agent. */ - harnesses?: string[] - /** Optional per-harness model override. Indexed parallel to `harnesses`. */ - models?: (string | undefined)[] - /** Default citation density floor for the shared validator. */ - citationDensityMin?: number - /** Optional task — narrows the validator's namespace check. */ - task?: ResearchTask -} - -/** - * Build a fanout topology over multiple harnesses. The kernel round-robins - * `agentRuns` across the N parallel iterations and the `FanoutVote` driver - * picks the highest-scoring valid output. - * - * @experimental - */ -export function multiHarnessResearcherFanout(options: MultiHarnessResearcherFanoutOptions = {}): { - agentRuns: AgentRunSpec[] - output: OutputAdapter - validator: Validator - driver: Driver -} { - const harnesses = - options.harnesses && options.harnesses.length > 0 - ? options.harnesses - : ['opencode/zai-coding-plan/glm-5.1', 'claude-code', 'codex'] - const models = options.models ?? [] - const agentRuns = harnesses.map((harness, i) => { - const { agentRunSpec } = researcherProfile({ harness, model: models[i] }) - return agentRunSpec - }) - const { output, validator } = researcherProfile({ - citationDensityMin: options.citationDensityMin, - task: options.task, - }) - // Single fanout round across the N harnesses, then stop: the kernel - // round-robins `agentRuns` over the N branches and selects the winner - // (best valid score) across all iterations via `defaultSelectWinner`. - const driver: Driver = { - name: 'researcher-fanout', - async plan(task, history) { - return history.length === 0 ? Array.from({ length: harnesses.length }, () => task) : [] - }, - // 'done' is a terminal decision in the kernel; the loop finalizes after - // the single fanout round and selects the winner via `defaultSelectWinner`. - decide() { - return 'done' - }, - describePlan() { - return { kind: 'fanout' } - }, - } - return { agentRuns, output, validator, driver } -} - -/** - * Build a validator that closes over a specific `ResearchTask`'s constraints. - * - * Checks in order: - * 1. Items must be non-empty. - * 2. Every item carries `evidence.length >= 1`. - * 3. Every item + proposedWrite is scoped to `task.knowledgeNamespace` - * (hard-fail on any namespace mismatch — defence in depth for the - * multi-tenant invariant). - * 4. Citation density (citations with quote / items) >= floor. - * - * Aggregate score: - * 0.4 · citation_density - * + 0.2 · source_diversity (distinct sources / max(items, 1)) - * + 0.2 · recency_match (mean fraction within `recencyWindow`) - * + 0.2 · (1 − gaps/maxGaps), maxGaps = max(items, 1) - * - * @experimental - */ -export function createResearcherValidator( - task: ResearchTask, - config: { citationDensityMin?: number; namespaceCheck?: boolean } = {}, -): Validator { - const citationDensityMin = config.citationDensityMin ?? DEFAULT_CITATION_DENSITY_MIN - const namespaceCheck = config.namespaceCheck ?? true - return { - async validate(output) { - const notes: string[] = [] - const scores: Record = {} - let pass = true - - if (!Array.isArray(output.items) || output.items.length === 0) { - pass = false - notes.push('no items') - scores.items = 0 - } else { - scores.items = 1 - } - - const missingEvidence = (output.items ?? []).filter( - (item) => !Array.isArray(item.evidence) || item.evidence.length === 0, - ) - if (missingEvidence.length > 0) { - pass = false - notes.push(`${missingEvidence.length} item(s) without evidence`) - scores.provenance = 0 - } else { - scores.provenance = 1 - } - - if (namespaceCheck) { - const foreignItems = (output.items ?? []).filter( - (item) => item.namespace !== task.knowledgeNamespace, - ) - const foreignWrites = (output.proposedWrites ?? []).filter( - (write) => write.namespace !== task.knowledgeNamespace, - ) - if (foreignItems.length > 0 || foreignWrites.length > 0) { - pass = false - notes.push( - `namespace violation: ${foreignItems.length} item(s) + ${foreignWrites.length} write(s) ` + - `outside ${task.knowledgeNamespace}`, - ) - scores.namespace = 0 - } else { - scores.namespace = 1 - } - } - - const itemCount = Math.max(output.items?.length ?? 0, 1) - const citationsWithQuote = (output.citations ?? []).filter( - (citation) => typeof citation.quote === 'string' && citation.quote.length > 0, - ).length - const citationDensity = Math.min(1, citationsWithQuote / itemCount) - if (citationDensity < citationDensityMin) { - pass = false - notes.push( - `citation density ${citationDensity.toFixed(2)} below floor ${citationDensityMin.toFixed(2)}`, - ) - } - scores.citation_density = citationDensity - - const sourceSet = new Set() - for (const item of output.items ?? []) { - for (const evidence of item.evidence ?? []) { - if (evidence.source) sourceSet.add(evidence.source) - } - } - scores.source_diversity = Math.min(1, sourceSet.size / itemCount) - - scores.recency_match = recencyMatchScore(output.items ?? [], task.recencyWindow) - - const maxGaps = itemCount - const gapCount = output.gaps?.length ?? 0 - scores.gap_coverage = Math.max(0, 1 - gapCount / maxGaps) - - const score = - 0.4 * scores.citation_density + - 0.2 * scores.source_diversity + - 0.2 * scores.recency_match + - 0.2 * scores.gap_coverage - - const verdict: DefaultVerdict = { - valid: pass, - score: Number.isFinite(score) ? score : 0, - scores, - } - if (notes.length > 0) verdict.notes = notes.join('; ') - return verdict - }, - } -} - -function recencyMatchScore(items: KnowledgeItem[], window: ResearchTask['recencyWindow']): number { - if (!window || (window.since === undefined && window.until === undefined)) return 1 - if (items.length === 0) return 0 - const sinceMs = window.since?.getTime() ?? Number.NEGATIVE_INFINITY - const untilMs = window.until?.getTime() ?? Number.POSITIVE_INFINITY - let hits = 0 - let total = 0 - for (const item of items) { - for (const evidence of item.evidence ?? []) { - if (typeof evidence.capturedAt !== 'number') continue - total += 1 - if (evidence.capturedAt >= sinceMs && evidence.capturedAt <= untilMs) hits += 1 - } - } - return total === 0 ? 0 : hits / total -} - -const DEFAULT_RESEARCHER_SYSTEM_PROMPT = [ - 'You are a research agent. Your job is to answer a research question with', - 'source-grounded knowledge items that the caller will choose whether to', - 'persist to a multi-tenant knowledge base.', - '', - 'Hard rules:', - " 1. Every item you emit MUST carry the task's knowledgeNamespace exactly.", - ' Never write to a different namespace.', - ' 2. Every item MUST carry at least one evidence entry with a source.', - ' A quote + url is strongly preferred; capturedAt is unix ms.', - ' 3. You propose writes — you do NOT apply them. The caller decides.', - ' 4. Self-report confidence honestly in [0, 1]. Do not inflate.', - ' 5. List what you could not answer in `gaps`. Better to admit a gap', - ' than fabricate.', - '', - 'When you finish, emit a single final structured message of the shape:', - ' ```json', - ' { "items": [{ "id": "...", "namespace": "...", "claim": "...",', - ' "evidence": [{ "source": "...", "quote": "...",', - ' "url": "...", "capturedAt": 0 }],', - ' "confidence": 0.0,', - ' "authoredBy": { "kind": "agent", "id": "..." } }],', - ' "citations": [{ "url": "...", "quote": "...", "confidence": 0.0 }],', - ' "proposedWrites": [{ "kind": "insert", "namespace": "...",', - ' "item": { /* same shape as items[] */ } }],', - ' "gaps": ["..."],', - ' "notes": "free-form commentary" }', - ' ```', -].join('\n') - -function formatResearcherPrompt(task: ResearchTask): string { - const sources = task.sources?.length ? task.sources.join(', ') : '(no preference)' - const window = formatRecencyWindow(task.recencyWindow) - return [ - `Question: ${task.question}`, - `Knowledge namespace (DO NOT cross): ${task.knowledgeNamespace}`, - `Scope: ${task.scope ?? '(unspecified)'}`, - `Preferred sources: ${sources}`, - `Recency window: ${window}`, - `Max items: ${task.maxItems ?? '(no cap)'}`, - `Per-item minimum confidence: ${task.minConfidence ?? '(no floor)'}`, - '', - 'Produce knowledge items with provenance + citations + proposed writes.', - 'List gaps for anything you could not answer. Emit the final JSON', - 'result block exactly as instructed.', - ].join('\n') -} - -function formatRecencyWindow(window: ResearchTask['recencyWindow']): string { - if (!window) return '(none)' - const since = window.since ? window.since.toISOString() : '-∞' - const until = window.until ? window.until.toISOString() : 'now' - return `${since} .. ${until}` -} - -/** - * Walk the event stream and return the last structured `research.result` - * payload. Falls back to scanning text deltas for a fenced JSON block. - */ -function parseResearcherEvents(events: SandboxEvent[]): ResearchOutput { - for (let i = events.length - 1; i >= 0; i -= 1) { - const event = events[i] - if (!event) continue - const type = String(event.type ?? '') - const data = isRecord(event.data) ? event.data : {} - if (type === 'result' || type === 'final' || type === 'research.result') { - const direct = coerceResearchOutput(data.result ?? data.output ?? data) - // opencode reports the agent's terminal answer in `finalText` (not result/output). - // Parse it too, and prefer it when `direct` is an all-empty shape — otherwise an - // event carrying both `{items:[],citations:[],proposedWrites:[]}` and a rich - // finalText would silently drop the real answer. - const finalText = pickString(data.finalText) - const fenced = finalText ? extractFencedJson(finalText) : undefined - const fromFinalText = fenced ? coerceResearchOutput(fenced) : undefined - if (direct && !isEmptyOutput(direct)) return direct - if (fromFinalText) return fromFinalText - if (direct) return direct - } - } - for (let i = events.length - 1; i >= 0; i -= 1) { - const event = events[i] - if (!event) continue - const data = isRecord(event.data) ? event.data : {} - const text = pickString(data.text) ?? pickString(data.delta) ?? pickString(data.finalText) - if (!text) continue - const fenced = extractFencedJson(text) - if (!fenced) continue - const coerced = coerceResearchOutput(fenced) - if (coerced) return coerced - } - return { items: [], citations: [], proposedWrites: [] } -} - -function isRecord(value: unknown): value is Record { - return value !== null && typeof value === 'object' && !Array.isArray(value) -} - -/** A coerced output that carries no items, citations, or proposed writes. */ -function isEmptyOutput(o: ResearchOutput): boolean { - return o.items.length === 0 && o.citations.length === 0 && o.proposedWrites.length === 0 -} - -function pickString(value: unknown): string | undefined { - return typeof value === 'string' && value.length > 0 ? value : undefined -} - -function extractFencedJson(text: string): unknown | undefined { - const match = text.match(/```(?:json)?\s*([\s\S]*?)```/i) - if (!match) return undefined - const body = (match[1] ?? '').trim() - if (!body) return undefined - try { - return JSON.parse(body) - } catch { - return undefined - } -} - -function coerceResearchOutput(value: unknown): ResearchOutput | undefined { - if (!isRecord(value)) return undefined - const items = coerceItems(value.items) - const citations = coerceCitations(value.citations) - const proposedWrites = coerceProposedWrites(value.proposedWrites) - // Reject completely empty payloads — those signal "no result block", - // not "a valid empty result". - if (items === undefined && citations === undefined && proposedWrites === undefined) { - return undefined - } - const output: ResearchOutput = { - items: items ?? [], - citations: citations ?? [], - proposedWrites: proposedWrites ?? [], - } - if (Array.isArray(value.gaps)) { - output.gaps = value.gaps.filter((entry): entry is string => typeof entry === 'string') - } - const notes = pickString(value.notes) - if (notes) output.notes = notes - // Preserve any extra fields the agent emitted that don't fit the typed - // surface — callers can inspect `raw` without forcing them through the - // typed coercion. We only include `raw` when the agent emitted fields - // beyond the known set. - const known = new Set(['items', 'citations', 'proposedWrites', 'gaps', 'notes']) - const extras: Record = {} - let extrasCount = 0 - for (const [key, val] of Object.entries(value)) { - if (known.has(key)) continue - extras[key] = val - extrasCount += 1 - } - if (extrasCount > 0) output.raw = extras - return output -} - -function coerceItems(value: unknown): KnowledgeItem[] | undefined { - if (!Array.isArray(value)) return undefined - const out: KnowledgeItem[] = [] - for (const entry of value) { - if (!isRecord(entry)) continue - const id = pickString(entry.id) - const namespace = pickString(entry.namespace) - const claim = pickString(entry.claim) - const evidence = coerceEvidence(entry.evidence) - const confidence = toFiniteNumber(entry.confidence) - const authoredBy = coerceAuthoredBy(entry.authoredBy) - if (!id || !namespace || !claim || !authoredBy) continue - const item: KnowledgeItem = { - id, - namespace, - claim, - evidence, - confidence: clamp01(confidence), - authoredBy, - } - if (Array.isArray(entry.supersedes)) { - item.supersedes = entry.supersedes.filter((s): s is string => typeof s === 'string') - } - const retractedAt = toFiniteNumber(entry.retractedAt) - if (retractedAt > 0) item.retractedAt = retractedAt - out.push(item) - } - return out -} - -function coerceEvidence(value: unknown): KnowledgeItem['evidence'] { - if (!Array.isArray(value)) return [] - const out: KnowledgeItem['evidence'] = [] - for (const entry of value) { - if (!isRecord(entry)) continue - const source = pickString(entry.source) - if (!source) continue - const item: KnowledgeItem['evidence'][number] = { - source, - capturedAt: toFiniteNumber(entry.capturedAt), - } - const quote = pickString(entry.quote) - if (quote) item.quote = quote - const url = pickString(entry.url) - if (url) item.url = url - out.push(item) - } - return out -} - -function coerceAuthoredBy(value: unknown): KnowledgeItem['authoredBy'] | undefined { - if (!isRecord(value)) return undefined - const kind = value.kind === 'human' || value.kind === 'agent' ? value.kind : undefined - const id = pickString(value.id) - if (!kind || !id) return undefined - return { kind, id } -} - -function coerceCitations(value: unknown): ResearchOutput['citations'] | undefined { - if (!Array.isArray(value)) return undefined - const out: ResearchOutput['citations'] = [] - for (const entry of value) { - if (!isRecord(entry)) continue - const url = pickString(entry.url) - const quote = pickString(entry.quote) - if (!url || !quote) continue - out.push({ url, quote, confidence: clamp01(toFiniteNumber(entry.confidence)) }) - } - return out -} - -function coerceProposedWrites(value: unknown): KnowledgeUpdate[] | undefined { - if (!Array.isArray(value)) return undefined - const out: KnowledgeUpdate[] = [] - for (const entry of value) { - if (!isRecord(entry)) continue - const namespace = pickString(entry.namespace) - if (!namespace) continue - if (entry.kind === 'insert') { - const items = coerceItems([entry.item]) - const item = items?.[0] - if (!item) continue - out.push({ kind: 'insert', namespace, item }) - } else if (entry.kind === 'supersede') { - const previousId = pickString(entry.previousId) - const items = coerceItems([entry.item]) - const item = items?.[0] - if (!previousId || !item) continue - out.push({ kind: 'supersede', namespace, previousId, item }) - } else if (entry.kind === 'retract') { - const itemId = pickString(entry.itemId) - const reason = pickString(entry.reason) - if (!itemId || !reason) continue - out.push({ kind: 'retract', namespace, itemId, reason }) - } - } - return out -} - -function toFiniteNumber(value: unknown): number { - return typeof value === 'number' && Number.isFinite(value) ? value : 0 -} - -function clamp01(value: number): number { - if (!Number.isFinite(value)) return 0 - if (value < 0) return 0 - if (value > 1) return 1 - return value -} diff --git a/src/research-supervisor.ts b/src/research-supervisor.ts deleted file mode 100644 index c0a421d..0000000 --- a/src/research-supervisor.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { - type Budget, - type DeliverableSpec, - type ExecutorConfig, - type SupervisedResult, - type SuperviseOptions, - type SupervisorProfile, - supervise, -} from '@tangle-network/agent-runtime/loops' -import type { BuildEvalKnowledgeBundleOptions, KnowledgeReadinessSpec } from './eval-readiness' -import { buildKnowledgeIndex } from './indexer' -import { researcherProfile } from './profiles/researcher' -import { readinessFor } from './readiness-helpers' -import { initKnowledgeBase } from './store' - -/** - * Default standing instructions for a research supervisor. It does NOT solve the - * research itself — it DECOMPOSES the goal into sub-topics and spawns one - * researcher per sub-topic over the live `Scope`, widening/narrowing until the - * readiness gate (the deliverable check) reports the knowledge base is ready. - */ -export const RESEARCH_SUPERVISOR_SYSTEM_PROMPT = [ - 'You are a research SUPERVISOR. You do not answer the research question yourself —', - 'you create and manage a team of researcher workers that grow ONE shared', - 'knowledge base until it is ready.', - '', - 'Each round:', - ' 1. Read the goal and the gaps the readiness gate still reports.', - ' 2. DECOMPOSE the open work into independent sub-topics. Spawn ONE researcher', - ' per sub-topic (spawn_agent) — split by sub-topic, never duplicate work.', - ' Spawn more researchers for a broad goal, fewer for a narrow one.', - ' 3. Wait for the researchers to settle (await_event). Steer or re-spawn for', - ' the sub-topics that came back thin.', - ' 4. STOP as soon as the deliverable check reports the knowledge base is ready.', - '', - 'Conserve your budget: every spawn spends from one shared pool. Prefer a small', - 'number of well-scoped researchers over a large fanout that re-covers the same', - 'ground.', -].join('\n') - -export interface ResearchSupervisorOptions { - /** Where the knowledge base lives. The deliverable check reads readiness here. */ - root: string - /** The research goal handed to the supervisor as its task. */ - goal: string - /** - * Readiness specs define DONE: the supervisor's deliverable check passes once - * `scoreKnowledgeReadiness` reports no blocking gaps over the live KB. Required - * — without a gate the supervisor would never know when to stop. - */ - readinessSpecs: KnowledgeReadinessSpec[] - readinessTaskId?: string - readiness?: Omit - /** The conserved compute pool for the whole supervised run. */ - budget: Budget - /** - * WHERE researcher workers run — the caller's real backend seam (sandbox with - * a `sandboxClient`, or router/cli/bridge). The supervisor spawns researchers - * on this backend; assembling the seam (creds, sandbox SDK) is the caller's, - * so there is no fabricated default. - */ - backend: ExecutorConfig - /** Harness for the researcher worker profile (default `researcherProfile`'s). */ - harness?: string - /** The supervisor brain's router model. */ - supervisorModel?: string - /** Custom supervisor instructions. Defaults to {@link RESEARCH_SUPERVISOR_SYSTEM_PROMPT}. */ - supervisorSystemPrompt?: string - /** Extra `supervise()` knobs (perWorker, maxLiveWorkers, router, allowedModels, …). */ - superviseOptions?: Partial> -} - -/** - * Build the deliverable (completion oracle) for a research supervisor: "settled - * ⟺ the knowledge base is ready". It re-reads the KB from disk and runs the - * readiness gate, so the supervisor stops on the REAL grounded state — not on a - * worker's self-report. - */ -export function knowledgeReadinessDeliverable( - options: Pick< - ResearchSupervisorOptions, - 'root' | 'goal' | 'readinessSpecs' | 'readinessTaskId' | 'readiness' - >, -): DeliverableSpec { - return { - describe: `knowledge base at ${options.root} is ready for: ${options.goal}`, - async check() { - const index = await buildKnowledgeIndex(options.root) - const bundle = readinessFor(options, index) - // `readinessSpecs` is required on the supervisor, so the bundle is always - // built; an empty/undefined gate means nothing blocks readiness. - return (bundle?.report.blockingMissingRequirements.length ?? 0) === 0 - }, - } -} - -/** - * A research supervisor: a SUPERVISOR brain creates the topology dynamically — - * how many researchers, split by sub-topic — over the `Scope`, with the - * knowledge-readiness gate as its stop condition. - * - * This is a thin wrapper over `supervise(profile, goal, opts)`. It composes the - * existing pieces and builds nothing new: the researcher `AgentProfile` (from - * `researcherProfile`) is the worker shape the supervisor spawns, and - * `knowledgeReadinessDeliverable` (over `scoreKnowledgeReadiness`) is the - * completion oracle that decides when the KB is ready. - * - * Needs creds (a real supervisor router brain + a worker backend), so it is the - * LIVE path. The offline two-agent loop is `runVerifiedResearchLoop`. - */ -export async function runResearchSupervisor( - options: ResearchSupervisorOptions, -): Promise> { - await initKnowledgeBase(options.root) - - // The researcher profile is the worker CONTRACT the supervisor spawns against. - // Its system prompt is appended so the supervisor authors researcher workers - // that emit source-grounded, propose-don't-apply contributions. - const { profile: workerProfile } = researcherProfile({ harness: options.harness }) - const baseInstructions = options.supervisorSystemPrompt ?? RESEARCH_SUPERVISOR_SYSTEM_PROMPT - const workerContract = workerProfile.prompt?.systemPrompt - const systemPrompt = workerContract - ? `${baseInstructions}\n\nEach researcher worker you spawn follows this contract:\n${workerContract}` - : baseInstructions - - const profile: SupervisorProfile = { - name: 'research-supervisor', - model: options.supervisorModel, - systemPrompt, - } - - return supervise(profile, options.goal, { - ...options.superviseOptions, - budget: options.budget, - backend: options.backend, - deliverable: knowledgeReadinessDeliverable(options), - }) -} diff --git a/tests/kb-improvement.test.ts b/tests/kb-improvement.test.ts index 47b1236..f381531 100644 --- a/tests/kb-improvement.test.ts +++ b/tests/kb-improvement.test.ts @@ -198,6 +198,68 @@ describe('improveKnowledgeBase', () => { }) }) + it('passes the candidate KB root into updateKnowledge callbacks', async () => { + await withKb(async (root) => { + const seen: Array<{ + root: string + baselineRoot: string + candidateRoot: string + runId: string + iteration: number + }> = [] + + const result = await improveKnowledgeBase({ + root, + goal: 'Let a runtime supervisor update the candidate KB', + runId: 'candidate-update-root', + promote: false, + async updateKnowledge(input) { + seen.push({ + root: input.root, + baselineRoot: input.baselineRoot, + candidateRoot: input.candidateRoot, + runId: input.runId, + iteration: input.iteration, + }) + const page = join(input.candidateRoot, 'knowledge', 'runtime-supervisor.md') + await mkdir(dirname(page), { recursive: true }) + await writeFile( + page, + [ + '---', + 'id: runtime-supervisor', + 'title: Runtime Supervisor', + '---', + '# Runtime Supervisor', + 'The runtime supervisor writes to the candidate workspace only.', + ].join('\n'), + ) + return { applied: true, summary: 'candidate workspace updated' } + }, + evaluate: () => ({ score: 1, passed: true }), + }) + + expect(result.state.status).toBe('candidate-ready') + expect(seen).toHaveLength(1) + expect(seen[0]).toMatchObject({ + root: result.candidate?.candidateRoot, + baselineRoot: root, + candidateRoot: result.candidate?.candidateRoot, + runId: 'candidate-update-root', + iteration: 1, + }) + await expect( + readFile(join(root, 'knowledge', 'runtime-supervisor.md'), 'utf8'), + ).rejects.toThrow() + await expect( + readFile( + join(result.candidate!.candidateRoot, 'knowledge', 'runtime-supervisor.md'), + 'utf8', + ), + ).resolves.toContain('candidate workspace only') + }) + }) + it('blocks promotion when the live KB changed after candidate creation', async () => { await withKb(async (root) => { const source = refundSource() diff --git a/tests/loops/research-loop-equal-compute.test.ts b/tests/loops/research-loop-equal-compute.test.ts deleted file mode 100644 index 397dbaf..0000000 --- a/tests/loops/research-loop-equal-compute.test.ts +++ /dev/null @@ -1,747 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { pairedBootstrap } from '@tangle-network/agent-eval' -import { - type Driver, - inProcessSandboxClient, - type OutputAdapter, - runLoop, - type SandboxEvent, -} from '@tangle-network/agent-runtime/loops' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { - buildEvalKnowledgeBundle, - defineReadinessSpec, - type KnowledgeReadinessSpec, -} from '../../src/eval-readiness' -import { sha256, stableId } from '../../src/ids' -import { buildKnowledgeIndex } from '../../src/indexer' -import { researcherProfile } from '../../src/profiles/researcher' -import { - type KnowledgeResearchLoopDecision, - runKnowledgeResearchLoop, -} from '../../src/research-loop' -import { - type ResearchContribution, - type ResearchDriver, - type ResearchSourceProposal, - type ResearchWorker, - runTwoAgentResearchLoop, - type SourceVerdict, - type WorkerResearchContext, -} from '../../src/two-agent-research-loop' -import { - createTangleRouterClient, - createVerifyingResearchDriver, - createWebResearchWorker, - type RouterClient, -} from '../../src/web-research-worker' - -// =========================================================================== -// THE VALUE A/B: does the verifying-driver (two-agent) loop build a CLEANER -// knowledge base than the single-agent loop AT THE SAME COMPUTE BUDGET? -// -// #28 already proved the MECHANISM (the driver rejects a planted bad source). -// This file answers the VALUE question — and the only way that answer is worth -// anything is if the two loops are budget-MATCHED. The two-agent loop spends -// MORE agent passes per round (worker proposes + driver verifies) than the -// single-agent loop (one pass per iteration). Comparing at equal ROUNDS would -// silently hand the two-agent loop extra compute and rig the result. So we -// budget-match by total AGENT PASSES and give the single-agent loop MORE -// iterations to spend the same budget the two-agent loop burns on verification. -// -// --------------------------------------------------------------------------- -// EQUAL-COMPUTE METHOD (the validity gate) -// --------------------------------------------------------------------------- -// Compute unit = ONE agent pass = one worker/driver invocation that reasons -// over the candidate pool. We count passes, not rounds: -// -// - two-agent round = 1 worker pass + 1 driver-verify pass = 2 passes -// - single-agent iter = 1 pass (no verifier) -// -// We give each arm the SAME budget CEILING B (the max passes it may spend): -// - two-agent : up to floor(B / 2) rounds (each round costs 2 passes) -// - single-agent : up to B iterations (each iter costs 1 pass) -// -// so neither arm can spend more than B passes. Both arms gate on the SAME -// readiness criterion, so each STOPS as soon as the KB is ready — on this pool -// both converge in one effective round (2 passes each), well inside the ceiling. -// Each arm instruments its OWN passes with a counter and the test asserts (a) -// both stayed within the ceiling and (b) the two-agent loop spent NO MORE -// passes than the single-agent loop. That is the machine-checked equal-compute -// gate: the two-agent loop never gets free verification, so any cleanliness -// gain is bought at no extra compute. (It cannot drift to an equal-ROUNDS -// comparison — the two-agent round is charged its full 2 passes.) -// -// The driver's verify pass here is deterministic (a prefix check), so it costs -// no tokens offline — but it is still a real reasoning pass over each round's -// candidates (in the live arm it is an LLM call), and it is exactly the extra -// cost the value question is asking about. We charge it as one pass per round, -// which is budget the single-agent loop is free to spend on extra iterations. -// -// --------------------------------------------------------------------------- -// CONTROLLED LOWER BOUND — read before trusting the assertion -// --------------------------------------------------------------------------- -// The junk in the source pool is PLANTED: `junk/`-prefixed spam whose text -// carries the gap-query tokens so a naive worker proposes it, but which the -// verifying driver rejects on its prefix. That makes this a CONTROLLED -// lower-bound demonstration: "when junk is present and the worker is naive, the -// verifying driver admits no MORE junk than the single-agent loop at equal -// compute, and on this pool admits strictly less." It is NOT a stand-in for -// real research — a real worker's junk is not prefix-detectable and a real -// driver's verifier is an LLM. The LIVE arm (creds-gated, below) is the real -// evidence; this offline arm proves the harness wiring + the floor. -// =========================================================================== - -interface PoolSource { - uri: string - title: string - text: string - /** True for the planted spam sources the verifying driver should reject. */ - junk: boolean -} - -/** A trivial output the offline agent emits: the sources it "found" this turn. */ -interface AgentResearch { - sources: PoolSource[] -} - -const agentOutput: OutputAdapter = { - parse(events: SandboxEvent[]): AgentResearch { - for (let i = events.length - 1; i >= 0; i -= 1) { - const event = events[i] - const data = (event?.data ?? {}) as { result?: AgentResearch } - if (event?.type === 'result' && data.result) return data.result - } - return { sources: [] } - }, -} - -/** A one-shot driver for the inner researcher loop: run once, then done. */ -function oneShotDriver(): Driver<{ prompt: string }, AgentResearch, 'done'> { - return { - name: 'researcher-once', - async plan(task, history) { - return history.length === 0 ? [task] : [] - }, - decide() { - return 'done' - }, - } -} - -/** - * The blocking readiness specs that define "done". Mirrors what the two-agent - * loop's test does: `scoreKnowledgeReadiness` scores PAGES (not raw sources), - * and only `importance: 'blocking'` requirements gate. Coverage below is - * measured as "fraction of these blocking requirements met". - */ -const blockingSpecs: KnowledgeReadinessSpec[] = [ - defineReadinessSpec({ - id: 'topic/ventilation', - description: 'coop ventilation requirements', - query: 'coop ventilation airflow', - requiredFor: ['IntakeAgent'], - importance: 'blocking', - minSources: 1, - minHits: 1, - }), - defineReadinessSpec({ - id: 'topic/predators', - description: 'predator protection requirements', - query: 'predator protection fencing', - requiredFor: ['IntakeAgent'], - importance: 'blocking', - minSources: 1, - minHits: 1, - }), -] - -/** - * A controlled source pool: two GOOD sources that cover the two blocking specs, - * plus planted JUNK whose text carries the same gap-query tokens (so a naive - * worker proposes it) but is `junk/`-prefixed spam. This is the lower-bound - * fixture, not real research. - */ -const sourcePool: PoolSource[] = [ - { - uri: 'web/ventilation-guide', - title: 'Coop Ventilation Guide', - text: 'Coop ventilation airflow keeps the flock healthy. Cross-ventilation near the roof.', - junk: false, - }, - { - uri: 'web/predator-fencing', - title: 'Predator Protection', - text: 'Predator protection fencing and hardware cloth stop raccoons and foxes.', - junk: false, - }, - { - uri: 'junk/ventilation-spam', - title: 'Buy Ventilation Fans Cheap!!!', - text: 'coop ventilation airflow predator protection fencing — SPAM advert, not a reference.', - junk: true, - }, - { - uri: 'junk/predator-clickbait', - title: '10 Predators That Will SHOCK You', - text: 'predator protection fencing coop ventilation clickbait listicle, no real guidance.', - junk: true, - }, -] - -/** - * Build the curated `knowledge/*.md` pages that cite accepted sources. The page - * text repeats the source text so the token-based readiness search hits it. - * Shared by both loops so coverage is measured the same way for each arm. - */ -function pageBuilder(proposals: ResearchSourceProposal[]) { - return (acceptedSources: { id: string; metadata?: Record }[]): string => - acceptedSources - .map((record) => { - const proposal = proposals.find((p) => p.uri === record.metadata?.originalUri) - const slug = (proposal?.uri ?? record.id).replace(/[^a-z0-9]+/gi, '-') - return [ - `---FILE: knowledge/${slug}.md---`, - '---', - `title: ${proposal?.title ?? record.id}`, - `sources: ["${record.id}"]`, - '---', - `# ${proposal?.title ?? record.id}`, - proposal?.text ?? '', - '---END FILE---', - ].join('\n') - }) - .join('\n') -} - -/** - * The shared NAIVE worker: drives the offline researcher AgentProfile through - * the REAL runLoop kernel + inProcessSandboxClient (no creds, no network), and - * proposes EVERY pool source whose token the gap prompt mentions — junk - * included. This is the "weak real agent" both loops are handed identically; - * the ONLY difference between the arms is whether a verifying driver gates the - * proposals. `onPass` ticks once per worker invocation so the test can prove - * equal compute. - */ -function makeNaiveProposals( - ctx: { goal: string; gaps: { description: string; query: string }[]; steer?: string }, - onPass: () => void, -): Promise { - const { profile } = researcherProfile({ harness: 'offline-stub' }) - const sandboxClient = inProcessSandboxClient({ - onPrompt(prompt): SandboxEvent[] { - onPass() - const found = sourcePool.filter((source) => - source.text - .toLowerCase() - .split(/[^a-z0-9]+/i) - .some((token) => token.length > 3 && prompt.toLowerCase().includes(token)), - ) - return [{ type: 'result', data: { result: { sources: found } } }] - }, - }) - const queries = ctx.gaps.map((gap) => `${gap.description} ${gap.query}`).join(' | ') - return runLoop<{ prompt: string }, AgentResearch, 'done'>({ - driver: oneShotDriver(), - agentRun: { profile, taskToPrompt: (task) => task.prompt, name: 'offline-researcher' }, - output: agentOutput, - task: { prompt: `Research these gaps for "${ctx.goal}": ${queries}\n${ctx.steer ?? ''}` }, - ctx: { sandboxClient }, - }).then((result) => - (result.winner?.output.sources ?? []).map((finding) => ({ - uri: finding.uri, - text: finding.text, - title: finding.title, - })), - ) -} - -/** Two-agent worker: the naive proposer wrapped in the two-agent contract. */ -function twoAgentWorker(onPass: () => void): ResearchWorker { - return async (ctx: WorkerResearchContext): Promise => { - const sources = await makeNaiveProposals(ctx, onPass) - return { sources, buildPages: pageBuilder(sources), notes: `proposed ${sources.length}` } - } -} - -/** The verifying driver: rejects the planted `junk/` sources, accepts the rest. */ -function verifyingDriver(): ResearchDriver { - return { - verifySource(source: ResearchSourceProposal): SourceVerdict { - // The differentiating reasoning pass: screen each candidate (a prefix - // check offline; an LLM `verifySource` call live). It is the extra compute - // the value question asks about, charged once per round in the accounting. - if (source.uri.startsWith('junk/')) { - return { accept: false, reason: 'planted junk: spam/clickbait, not a real reference' } - } - return { accept: true } - }, - } -} - -/** How many junk sources reached the KB (registered as sources). */ -async function junkAdmitted(root: string): Promise { - const index = await buildKnowledgeIndex(root) - return index.sources.filter((source) => { - const original = source.metadata?.originalUri - return typeof original === 'string' && original.startsWith('junk/') - }).length -} - -/** - * Coverage = fraction of BLOCKING readiness requirements met, scored the way - * the loops gate: build the readiness report over the current index, count how - * many blocking specs are NOT in `blockingMissingRequirements`. - */ -async function coverage( - root: string, - goal: string, - specs: KnowledgeReadinessSpec[] = blockingSpecs, -): Promise { - const index = await buildKnowledgeIndex(root) - // `buildEvalKnowledgeBundle` searches each spec over the index's PAGES and - // runs the substrate `scoreKnowledgeReadiness` — the SAME scorer both loops - // gate on. Coverage = fraction of blocking specs NOT in the missing set. - const { report } = buildEvalKnowledgeBundle({ taskId: goal, index, specs }) - const blockingCount = specs.filter((s) => s.importance === 'blocking').length - const missing = report.blockingMissingRequirements.length - return blockingCount === 0 ? 1 : (blockingCount - missing) / blockingCount -} - -/** - * Total sources that reached the KB. For REAL research there is no `junk/` - * prefix oracle — the live signal is that the verifying driver admits FEWER - * sources because it rejects the off-topic ones the single-agent loop keeps. So - * the live A/B compares admitted-source COUNTS (cleaner = fewer admitted at - * equal-or-higher coverage), the real-world analogue of the offline junk count. - */ -async function admittedSourceCount(root: string): Promise { - const index = await buildKnowledgeIndex(root) - return index.sources.length -} - -/** - * The source-record id `addSourceText` will assign — deterministic from the - * text+uri (see src/sources.ts: `stableId('src', `${sha256(text)}:${uri}`)`). We - * precompute it so the single-agent arm can write a page that cites the source - * it registers IN THE SAME STEP (the loop registers `sourceTexts` first, then - * applies `proposalText`), exactly mirroring how the two-agent loop cites - * `record.id`. Without this the page would cite a non-existent id and coverage - * would read 0 for reasons unrelated to the A/B. - */ -function predictedSourceId(source: ResearchSourceProposal): string { - return stableId('src', `${sha256(source.text)}:${source.uri}`) -} - -/** - * Run the SINGLE-agent loop on the shared pool with a budget CEILING of - * `maxIterations` passes. Each iteration is one naive-proposer pass; the loop - * applies the proposals with NO verification gate (that is the whole point of - * the single-agent arm). It stops on the SAME readiness gate the two-agent loop - * uses (so both arms converge by the same criterion, not at unequal effort). - * Returns the actual passes spent so the test can compare compute. - */ -/** - * A round of primary research, abstracted so the SAME arm-runners drive both the - * offline naive proposer AND a real web-research worker. Given the goal + open - * gaps + optional steer, return the sources found this pass. The offline default - * is `makeNaiveProposals` over the planted pool; the live arm injects the real - * `createWebResearchWorker`. - */ -type ProposeSources = ( - ctx: { - goal: string - gaps: { description: string; query: string }[] - steer?: string - root: string - }, - onPass: () => void, -) => Promise - -const offlinePropose: ProposeSources = (ctx, onPass) => makeNaiveProposals(ctx, onPass) - -async function runSingleAgentArm( - root: string, - goal: string, - maxIterations: number, - propose: ProposeSources = offlinePropose, - specs: KnowledgeReadinessSpec[] = blockingSpecs, -): Promise<{ passes: number }> { - let passes = 0 - await runKnowledgeResearchLoop({ - root, - goal, - maxIterations, - readinessSpecs: specs, - async step(context): Promise { - passes += 1 - const report = context.readiness?.report - // Single-agent applies the SAME readiness gate the two-agent loop gates on - // (no blocking requirements left ⇒ done). This is what makes the compute - // comparison fair: both stop by the same criterion. - if (report && report.blockingMissingRequirements.length === 0) { - return { done: true, notes: 'readiness gate met' } - } - const gaps = (report?.blockingMissingRequirements ?? []).map((req) => ({ - description: req.description, - query: - typeof req.metadata?.query === 'string' - ? (req.metadata.query as string) - : req.description, - })) - const proposals = await propose({ goal, gaps, root }, () => {}) - if (proposals.length === 0) return { notes: 'no new proposals' } - // Register sources AND write citing pages in one step — exactly like the - // two-agent worker (sources + buildPages), but with NO driver verification - // in between. Pages cite the precomputed real source id. - const proposalText = pageBuilder(proposals)( - proposals.map((p) => ({ id: predictedSourceId(p), metadata: { originalUri: p.uri } })), - ) - return { sourceTexts: proposals, proposalText, notes: `applied ${proposals.length}` } - }, - }) - return { passes } -} - -/** - * Run the TWO-agent loop for `rounds` rounds on the shared pool. Each round is - * one worker pass + one driver-verify pass = 2 passes. Returns the pass count. - */ -async function runTwoAgentArm( - root: string, - goal: string, - rounds: number, - arm?: { worker: ResearchWorker; driver: ResearchDriver }, - specs: KnowledgeReadinessSpec[] = blockingSpecs, -): Promise<{ passes: number }> { - let workerPasses = 0 - // Default arm = the offline naive proposer + prefix-check verifier. The live - // arm injects the real web-research worker + the LLM verifying driver. Either - // way the worker pass is counted once per invocation via a thin wrapper, so - // the equal-compute accounting below holds for both. - const worker: ResearchWorker = - arm?.worker ?? - twoAgentWorker(() => { - workerPasses += 1 - }) - const countedWorker: ResearchWorker = arm - ? async (ctx) => { - workerPasses += 1 - return worker(ctx) - } - : worker - const driver: ResearchDriver = arm?.driver ?? verifyingDriver() - await runTwoAgentResearchLoop({ - root, - goal, - worker: countedWorker, - driver, - readinessSpecs: specs, - maxRounds: rounds, - }) - // EQUAL-COMPUTE ACCOUNTING: every round that ran a worker pass also ran one - // driver-verify pass over that round's candidate batch (a real LLM call in - // the live arm). So passes = 2 × (rounds the worker actually ran). The loop - // stops early on the readiness gate, so workerPasses ≤ `rounds`. - return { passes: workerPasses * 2 } -} - -let twoAgentRoot: string -let singleAgentRoot: string -const goal = 'backyard chicken coop requirements' - -beforeEach(async () => { - twoAgentRoot = await mkdtemp(join(tmpdir(), 'ab-two-agent-')) - singleAgentRoot = await mkdtemp(join(tmpdir(), 'ab-single-agent-')) -}) -afterEach(async () => { - await rm(twoAgentRoot, { recursive: true, force: true }) - await rm(singleAgentRoot, { recursive: true, force: true }) -}) - -describe('research loop A/B at equal compute (offline, controlled lower bound)', () => { - it('two-agent admits <= single-agent junk at the SAME agent-pass budget', async () => { - // Budget CEILING B = 6 agent passes per arm (the MAX each may spend): - // two-agent : up to B/2 = 3 rounds (worker + verify per round = 2 passes) - // single : up to B = 6 iters (1 pass per iter, no verifier) - // Both arms gate on the SAME readiness criterion, so each stops as soon as - // the KB is ready — they converge well inside the ceiling, at EQUAL actual - // passes (asserted below). The ceiling proves neither was starved; the - // equal actual-pass count proves neither was secretly handed more compute. - const budgetPasses = 6 - const twoAgentRounds = budgetPasses / 2 - const singleAgentIters = budgetPasses - - const twoAgent = await runTwoAgentArm(twoAgentRoot, goal, twoAgentRounds) - const single = await runSingleAgentArm(singleAgentRoot, goal, singleAgentIters) - - const twoAgentJunk = await junkAdmitted(twoAgentRoot) - const singleAgentJunk = await junkAdmitted(singleAgentRoot) - const twoAgentCoverage = await coverage(twoAgentRoot, goal) - const singleAgentCoverage = await coverage(singleAgentRoot, goal) - - // Surface the numbers so a CI log shows the A/B result, not just a pass. - console.log( - `[A/B @ B<=${budgetPasses} passes] ` + - `two-agent: passes=${twoAgent.passes} junk=${twoAgentJunk} coverage=${twoAgentCoverage.toFixed(2)} | ` + - `single-agent: passes=${single.passes} junk=${singleAgentJunk} coverage=${singleAgentCoverage.toFixed(2)}`, - ) - - // VALIDITY GATE — equal compute, machine-checked. Both arms stayed within - // the budget ceiling, and the two-agent loop spent NO MORE passes than the - // single-agent loop (in fact equal: both converge in one effective round). - // If this ever fails, the A/B has drifted to unequal compute and the value - // assertion below is meaningless. - expect(twoAgent.passes).toBeLessThanOrEqual(budgetPasses) - expect(single.passes).toBeLessThanOrEqual(budgetPasses) - expect(twoAgent.passes).toBeLessThanOrEqual(single.passes) - - // THE VALUE ASSERTION (controlled lower bound): at equal compute, the - // verifying driver admits no MORE junk than the single-agent loop. On this - // planted pool it admits strictly LESS (the junk is rejectable) while - // keeping coverage at least as high. - expect(twoAgentJunk).toBeLessThanOrEqual(singleAgentJunk) - expect(twoAgentJunk).toBe(0) - expect(singleAgentJunk).toBeGreaterThan(0) - expect(twoAgentCoverage).toBeGreaterThanOrEqual(singleAgentCoverage) - }) - - it('the gap holds across budget ceilings (B in {4, 8})', async () => { - for (const budgetPasses of [4, 8]) { - const twoRoot = await mkdtemp(join(tmpdir(), 'ab2-two-')) - const singleRoot = await mkdtemp(join(tmpdir(), 'ab2-single-')) - try { - const twoAgent = await runTwoAgentArm(twoRoot, goal, budgetPasses / 2) - const single = await runSingleAgentArm(singleRoot, goal, budgetPasses) - // Equal compute: two-agent spends no more passes than single-agent and - // both stay within the ceiling. - expect(twoAgent.passes).toBeLessThanOrEqual(single.passes) - expect(twoAgent.passes).toBeLessThanOrEqual(budgetPasses) - expect(single.passes).toBeLessThanOrEqual(budgetPasses) - // The cleanliness gap holds at every ceiling. - expect(await junkAdmitted(twoRoot)).toBeLessThanOrEqual(await junkAdmitted(singleRoot)) - expect(await junkAdmitted(twoRoot)).toBe(0) - expect(await junkAdmitted(singleRoot)).toBeGreaterThan(0) - } finally { - await rm(twoRoot, { recursive: true, force: true }) - await rm(singleRoot, { recursive: true, force: true }) - } - } - }) -}) - -// =========================================================================== -// LIVE A/B — the real evidence. Skipped offline (no creds), exactly like -// tests/sources-live.test.ts. Runs BOTH loops on a REAL research goal with the -// REAL web-research worker (glm-5.2 query-gen → live `/v1/search` → politeFetch -// → htmlToText) and a REAL LLM verifying driver (glm-5.2 on-topic judgement) at -// equal compute, then reports a PAIRED comparison via agent-eval's -// pairedBootstrap (no hand-rolled significance). -// -// What this adds over the offline arm: there is NO planted `junk/` pool and NO -// prefix-check verifier — the worker fetches whatever the web returns and the -// driver is an LLM. So the live cleanliness signal is admitted-source COUNT: -// the verifying driver rejects off-topic fetches, so the two-agent KB admits -// FEWER sources at equal-or-higher coverage. A win here is evidence the -// verifying driver cleans REAL research, not a planted floor. -// -// Gate: `AGENT_KNOWLEDGE_LIVE=1` + a TANGLE_API_KEY with glm-5.2 credits. -// AGENT_KNOWLEDGE_LIVE_GOALS — `|`-separated goals (default: self-speculative decoding) -// AGENT_KNOWLEDGE_LIVE_BUDGET — agent-pass ceiling B per arm (default 4) -// AGENT_KNOWLEDGE_LIVE_MODEL — router chat model (default glm-5.2) -// =========================================================================== - -/** - * Topic-relevant blocking readiness specs for a live goal. General: the gaps - * are phrased so a real web search can close them, and the search/readiness - * query carries the goal so coverage scores against fetched pages. Override the - * default goal via `AGENT_KNOWLEDGE_LIVE_GOALS` (this builds matching specs). - */ -function liveSpecsForGoal(liveGoal: string): KnowledgeReadinessSpec[] { - return [ - defineReadinessSpec({ - id: 'topic/definition', - description: `what ${liveGoal} is and how it works`, - query: `${liveGoal} how it works method`, - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 1, - minHits: 1, - }), - defineReadinessSpec({ - id: 'topic/results', - description: `reported results, speedups, or trade-offs for ${liveGoal}`, - query: `${liveGoal} speedup results benchmark`, - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 1, - minHits: 1, - }), - ] -} - -describe.skipIf(!process.env.AGENT_KNOWLEDGE_LIVE)( - 'live: research loop A/B at equal compute', - () => { - it('two-agent (real worker + LLM verifier) vs single-agent — paired comparison', async () => { - const goals = (process.env.AGENT_KNOWLEDGE_LIVE_GOALS ?? 'self-speculative decoding') - .split('|') - .map((g) => g.trim()) - .filter(Boolean) - const budgetPasses = Number(process.env.AGENT_KNOWLEDGE_LIVE_BUDGET ?? 4) - const model = process.env.AGENT_KNOWLEDGE_LIVE_MODEL ?? 'glm-5.2' - - // ONE shared router client for the whole run (web search + chat). - const router: RouterClient = createTangleRouterClient({ model }) - - // COST GATE: a cheap glm-5.2 smoke BEFORE the multi-arm burn. Proves the - // key works + the reasoning-token floor returns visible content. If this - // returns empty or throws, the full A/B can't produce real numbers — fail - // fast instead of spending the whole budget to discover it. - const smoke = await router.chat( - [ - { role: 'system', content: 'Reply with exactly the word: OK' }, - { role: 'user', content: 'Say OK.' }, - ], - 1200, - ) - console.log(`[LIVE smoke] glm-5.2 visible content length=${smoke.trim().length}`) - expect(smoke.trim().length).toBeGreaterThan(0) - - const realWorker = createWebResearchWorker({ - router, - resultsPerQuery: 3, - queriesPerGap: 1, - maxSourcesPerRound: 6, - }) - const realDriver = createVerifyingResearchDriver({ router }) - - const twoAdmittedByGoal: number[] = [] - const singleAdmittedByGoal: number[] = [] - let anySourceFetched = false - - for (const liveGoal of goals) { - const specs = liveSpecsForGoal(liveGoal) - const twoRoot = await mkdtemp(join(tmpdir(), 'live-two-')) - const singleRoot = await mkdtemp(join(tmpdir(), 'live-single-')) - try { - // Snapshot cumulative router cost so each arm's token/$/latency/calls - // can be diffed out — the COST half of the cost/quality result. - const u0 = router.usage() - // TWO-AGENT arm: real worker proposes, real LLM driver verifies. - const two = await runTwoAgentArm( - twoRoot, - liveGoal, - budgetPasses / 2, - { worker: realWorker, driver: realDriver }, - specs, - ) - const u1 = router.usage() - // SINGLE-AGENT arm: the SAME real worker, NO verifier gate, more iters - // to spend the same agent-pass budget the two-agent loop burns on - // verification. - const single = await runSingleAgentArm( - singleRoot, - liveGoal, - budgetPasses, - (ctx, onPass) => realWorkerPropose(realWorker, ctx, onPass), - specs, - ) - const u2 = router.usage() - const twoCost = { - chatCalls: u1.chatCalls - u0.chatCalls, - tokens: u1.promptTokens + u1.completionTokens - u0.promptTokens - u0.completionTokens, - usd: u1.usd - u0.usd, - wallMs: u1.wallMs - u0.wallMs, - } - const singleCost = { - chatCalls: u2.chatCalls - u1.chatCalls, - tokens: u2.promptTokens + u2.completionTokens - u1.promptTokens - u1.completionTokens, - usd: u2.usd - u1.usd, - wallMs: u2.wallMs - u1.wallMs, - } - - const twoAdmitted = await admittedSourceCount(twoRoot) - const singleAdmitted = await admittedSourceCount(singleRoot) - const twoCoverage = await coverage(twoRoot, liveGoal, specs) - const singleCoverage = await coverage(singleRoot, liveGoal, specs) - if (twoAdmitted > 0 || singleAdmitted > 0) anySourceFetched = true - - twoAdmittedByGoal.push(twoAdmitted) - singleAdmittedByGoal.push(singleAdmitted) - - console.log( - `[LIVE A/B ${JSON.stringify(liveGoal)} @ B<=${budgetPasses}] ` + - `two-agent: passes=${two.passes} admitted=${twoAdmitted} coverage=${twoCoverage.toFixed(2)} ` + - `calls=${twoCost.chatCalls} tok=${twoCost.tokens} $${twoCost.usd.toFixed(4)} ${twoCost.wallMs}ms | ` + - `single-agent: passes=${single.passes} admitted=${singleAdmitted} coverage=${singleCoverage.toFixed(2)} ` + - `calls=${singleCost.chatCalls} tok=${singleCost.tokens} $${singleCost.usd.toFixed(4)} ${singleCost.wallMs}ms`, - ) - } finally { - await rm(twoRoot, { recursive: true, force: true }) - await rm(singleRoot, { recursive: true, force: true }) - } - } - - // The live arm is only evidence if the worker actually web-searched and - // fetched real pages. Zero sources across both arms = the worker never - // reached the web (creds/network) — that is a FALSE null, fail loud. - expect(anySourceFetched).toBe(true) - - // Paired bootstrap on (single − two) admitted-source deltas: a POSITIVE - // delta means the single-agent loop admitted MORE sources, i.e. the - // verifying driver kept the KB cleaner. `low > 0` is the significance - // gate. Reuse the substrate; do NOT hand-roll significance. - const result = pairedBootstrap(twoAdmittedByGoal, singleAdmittedByGoal, { - statistic: 'mean', - seed: 1, - }) - console.log( - `[LIVE A/B] n=${result.n} mean(single-two admitted)=${result.mean.toFixed(3)} ` + - `CI=[${result.low.toFixed(3)}, ${result.high.toFixed(3)}] — ` + - `two-agent cleaner iff low > 0`, - ) - // At equal compute the verifying driver should admit no MORE sources than - // the ungated single-agent loop on average; the bootstrap lower bound says - // whether the cleanliness gain is significant. - expect(result.mean).toBeGreaterThanOrEqual(0) - }, 600_000) - }, -) - -/** - * Drive the real web-research worker as a single-agent proposer: build a - * `WorkerResearchContext` from the loop context and return the sources it found. - * Charges one pass per invocation via `onPass`, matching the two-agent worker. - */ -async function realWorkerPropose( - worker: ResearchWorker, - ctx: { - goal: string - gaps: { description: string; query: string }[] - root: string - steer?: string - }, - onPass: () => void, -): Promise { - onPass() - const index = await buildKnowledgeIndex(ctx.root) - const readiness = buildEvalKnowledgeBundle({ taskId: ctx.goal, index, specs: [] }) - const contribution = await worker({ - root: ctx.root, - goal: ctx.goal, - round: 1, - index, - gaps: ctx.gaps.map((gap) => ({ - id: gap.description, - description: gap.description, - query: gap.query, - blocking: true, - })), - steer: ctx.steer, - readiness, - }) - return contribution.sources ?? [] -} diff --git a/tests/loops/research-supervisor.test.ts b/tests/loops/research-supervisor.test.ts deleted file mode 100644 index edcb036..0000000 --- a/tests/loops/research-supervisor.test.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -// Capture what runResearchSupervisor hands the runtime's `supervise()` without -// needing a real supervisor brain or worker backend (the LIVE path). We assert -// the wiring: profile assembly + deliverable gate, all offline. The mock is -// created via `vi.hoisted` so it exists when the hoisted `vi.mock` factory runs. -const { superviseMock } = vi.hoisted(() => ({ - superviseMock: vi.fn(async () => ({ - output: undefined, - settled: [], - budgetSpent: { iterations: 0, tokens: 0, usd: 0 }, - })), -})) - -vi.mock('@tangle-network/agent-runtime/loops', async (importOriginal) => { - const actual = await importOriginal() - return { ...actual, supervise: superviseMock } -}) - -import { defineReadinessSpec } from '../../src/eval-readiness' -import { applyKnowledgeWriteBlocks } from '../../src/proposals' -import { - knowledgeReadinessDeliverable, - RESEARCH_SUPERVISOR_SYSTEM_PROMPT, - runResearchSupervisor, -} from '../../src/research-supervisor' -import { addSourceText } from '../../src/sources' -import { initKnowledgeBase } from '../../src/store' - -let root: string -beforeEach(async () => { - root = await mkdtemp(join(tmpdir(), 'research-supervisor-')) - superviseMock.mockClear() -}) -afterEach(async () => { - await rm(root, { recursive: true, force: true }) -}) - -const blockingSpec = defineReadinessSpec({ - id: 'topic/ventilation', - description: 'coop ventilation requirements', - query: 'coop ventilation airflow', - requiredFor: ['IntakeAgent'], - importance: 'blocking', - minSources: 1, - minHits: 1, -}) - -describe('knowledgeReadinessDeliverable.check (offline completion oracle)', () => { - it('reports NOT ready when the blocking gap is unfilled', async () => { - await initKnowledgeBase(root) - const deliverable = knowledgeReadinessDeliverable({ - root, - goal: 'backyard chicken coop requirements', - readinessSpecs: [blockingSpec], - }) - expect(deliverable.describe).toContain(root) - expect(await deliverable.check()).toBe(false) - }) - - it('reports ready once a curated page fills the blocking gap', async () => { - await initKnowledgeBase(root) - // The readiness gate searches curated `knowledge/*.md` PAGES (not raw - // sources). Register the source, then write a page that cites it and repeats - // the query terms so the token search hits it. - const source = await addSourceText(root, { - uri: 'web/ventilation-guide', - title: 'Coop Ventilation Guide', - text: 'Coop ventilation airflow keeps the flock healthy. Cross-ventilation near the roof.', - }) - await applyKnowledgeWriteBlocks( - root, - [ - '---FILE: knowledge/ventilation.md---', - '---', - 'id: topic/ventilation', - 'title: Coop Ventilation', - `sources: ["${source.id}"]`, - '---', - '# Coop Ventilation', - 'Coop ventilation airflow requirements: cross-ventilation near the roof keeps the flock healthy.', - '---END FILE---', - ].join('\n'), - ) - const deliverable = knowledgeReadinessDeliverable({ - root, - goal: 'backyard chicken coop requirements', - readinessSpecs: [blockingSpec], - }) - expect(await deliverable.check()).toBe(true) - }) -}) - -describe('runResearchSupervisor (stub backend, wiring only)', () => { - it('drives supervise() with the goal, backend, budget, and the readiness deliverable', async () => { - const budget = { maxIterations: 4, maxTokens: 1_000, maxUsd: 1 } - const backend = { - backend: 'router' as const, - routerBaseUrl: 'https://router.test', - routerKey: 'test-key', - model: 'test-model', - } - await runResearchSupervisor({ - root, - goal: 'backyard chicken coop requirements', - readinessSpecs: [blockingSpec], - budget, - backend, - }) - - expect(superviseMock).toHaveBeenCalledTimes(1) - const [profile, task, opts] = superviseMock.mock.calls[0] as [ - { name: string; systemPrompt: string }, - string, - { budget: unknown; backend: unknown; deliverable: { check: () => Promise } }, - ] - expect(profile.name).toBe('research-supervisor') - // The supervisor instructions are the base prompt + the worker contract. - expect(profile.systemPrompt).toContain(RESEARCH_SUPERVISOR_SYSTEM_PROMPT) - expect(profile.systemPrompt).toContain('Each researcher worker you spawn follows this contract') - expect(task).toBe('backyard chicken coop requirements') - expect(opts.budget).toBe(budget) - expect(opts.backend).toBe(backend) - // The completion oracle is wired and runs over the real (empty) KB on disk. - expect(await opts.deliverable.check()).toBe(false) - }) -}) diff --git a/tests/loops/researcher-integration.test.ts b/tests/loops/researcher-integration.test.ts deleted file mode 100644 index 1057e3a..0000000 --- a/tests/loops/researcher-integration.test.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { runLoop } from '@tangle-network/agent-runtime/loops' -import type { CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' -import { describe, expect, it } from 'vitest' -import { - multiHarnessResearcherFanout, - type ResearchOutput, - type ResearchTask, - researcherProfile, -} from '../../src/profiles/researcher' - -function ageMs(days: number): number { - return Date.now() - days * 24 * 60 * 60 * 1000 -} - -function researchPayload(overrides: { namespace?: string; quality?: number } = {}): ResearchOutput { - const namespace = overrides.namespace ?? 'cust_42' - const quality = overrides.quality ?? 1 - const item = { - id: 'item-1', - namespace, - claim: 'cpg-founder ICP engages with founder-pov threads', - evidence: [ - { - source: 'twitter', - quote: quality >= 0.5 ? 'Engagement rate 4.2x' : '', - url: 'https://x.com/example/status/1', - capturedAt: ageMs(7), - }, - ], - confidence: 0.82 * quality, - authoredBy: { kind: 'agent' as const, id: 'researcher-stub' }, - } - return { - items: [item], - citations: - quality >= 0.5 - ? [ - { - url: 'https://x.com/example/status/1', - quote: 'Engagement rate 4.2x', - confidence: 0.82, - }, - ] - : [], - proposedWrites: [{ kind: 'insert', namespace, item }], - } -} - -function stubClient(perCall: Array<() => ResearchOutput | { __error: string }>): { - client: { create(opts?: CreateSandboxOptions): Promise } - callCount: () => number -} { - let i = 0 - return { - callCount: () => i, - client: { - async create() { - const idx = i - i += 1 - const factory = perCall[idx] ?? perCall[perCall.length - 1] - return { - async *streamPrompt() { - if (!factory) { - yield { type: 'noise', data: {} } satisfies SandboxEvent - return - } - const value = factory() - if ('__error' in value) { - throw new Error(value.__error) - } - yield { - type: 'result', - data: { result: value }, - } satisfies SandboxEvent - }, - } as unknown as SandboxInstance - }, - }, - } -} - -const task: ResearchTask = { - question: 'What does cpg-founder ICP engage with on Twitter?', - knowledgeNamespace: 'cust_42', - sources: ['twitter', 'web'], - maxItems: 10, - minConfidence: 0.6, -} - -describe('researcherProfile end-to-end through runLoop', () => { - it('drives a single AgentRunSpec via fanout-vote and selects a clean winner', async () => { - const { agentRuns, output, validator, driver } = multiHarnessResearcherFanout({ - harnesses: ['researcher-a', 'researcher-b', 'researcher-c'], - task, - }) - const { client } = stubClient([ - () => researchPayload({ quality: 1 }), - () => researchPayload({ quality: 1 }), - () => researchPayload({ quality: 1 }), - ]) - - const result = await runLoop({ - driver, - agentRuns, - output, - validator, - task, - ctx: { sandboxClient: client }, - }) - - expect(result.decision).toBe('done') - expect(result.iterations).toHaveLength(3) - expect(result.winner).toBeDefined() - expect(result.winner?.output.items).toHaveLength(1) - expect(result.winner?.output.proposedWrites).toHaveLength(1) - expect(result.winner?.verdict?.valid).toBe(true) - }) - - it('fails-out when every harness emits a cross-namespace leak', async () => { - const { agentRuns, output, validator, driver } = multiHarnessResearcherFanout({ - harnesses: ['researcher-a', 'researcher-b'], - task, - }) - const { client } = stubClient([ - () => researchPayload({ namespace: 'cust_99' }), - () => researchPayload({ namespace: 'cust_77' }), - ]) - - const result = await runLoop({ - driver, - agentRuns, - output, - validator, - task, - ctx: { sandboxClient: client }, - }) - - expect(result.decision).toBe('done') - expect(result.iterations).toHaveLength(2) - for (const iter of result.iterations) { - expect(iter.verdict?.valid).toBe(false) - expect(iter.verdict?.notes).toMatch(/namespace violation/) - } - // The kernel may surface a structural top-of-attempts even when nothing - // validates. The contract is: never a winner with `valid === true` when - // every output leaked across namespaces. - if (result.winner) { - expect(result.winner.verdict?.valid).toBe(false) - } - }) - - it('selects the higher-quality output across heterogeneous harnesses', async () => { - const { agentRuns, output, validator, driver } = multiHarnessResearcherFanout({ - harnesses: ['low-quality', 'high-quality'], - task, - }) - const { client } = stubClient([ - () => researchPayload({ quality: 0.1 }), // no quoted citation → density floor fails - () => researchPayload({ quality: 1 }), - ]) - - const result = await runLoop({ - driver, - agentRuns, - output, - validator, - task, - ctx: { sandboxClient: client }, - }) - - expect(result.decision).toBe('done') - expect(result.winner?.iterationIndex).toBe(1) - expect(result.winner?.agentRunName).toBe('researcher-high-quality') - }) - - it('passes proposedWrites through unchanged — caller is responsible for materialize', async () => { - const single = researcherProfile({ harness: 'researcher-stub' }) - const driver = multiHarnessResearcherFanout({ harnesses: ['researcher-stub'], task }).driver - const { client } = stubClient([() => researchPayload({ quality: 1 })]) - - const result = await runLoop({ - driver, - agentRuns: [single.agentRunSpec], - output: single.output, - validator: multiHarnessResearcherFanout({ harnesses: ['researcher-stub'], task }).validator, - task, - ctx: { sandboxClient: client }, - }) - - // The winner carries proposedWrites that the caller must explicitly - // route to applyKnowledgeWriteBlocks (or a KbStore put). The profile - // itself never wrote anything to disk — that's the invariant. - const writes = result.winner?.output.proposedWrites ?? [] - expect(writes).toHaveLength(1) - expect(writes[0]?.kind).toBe('insert') - expect(writes[0]?.namespace).toBe('cust_42') - }) -}) diff --git a/tests/loops/two-agent-research-loop.test.ts b/tests/loops/two-agent-research-loop.test.ts deleted file mode 100644 index 29b72de..0000000 --- a/tests/loops/two-agent-research-loop.test.ts +++ /dev/null @@ -1,399 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { scoreKnowledgeReadiness } from '@tangle-network/agent-eval' -import { - type Driver, - inProcessSandboxClient, - type OutputAdapter, - runLoop, - type SandboxEvent, -} from '@tangle-network/agent-runtime/loops' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { defineReadinessSpec } from '../../src/eval-readiness' -import { buildKnowledgeIndex } from '../../src/indexer' -import { researcherProfile } from '../../src/profiles/researcher' -import { searchKnowledge } from '../../src/search' -import { - type ResearchContribution, - type ResearchDriver, - type ResearchSourceProposal, - type ResearchWorker, - runTwoAgentResearchLoop, - type SourceVerdict, - type WorkerResearchContext, -} from '../../src/two-agent-research-loop' - -// --------------------------------------------------------------------------- -// Offline agent scaffolding: a researcher AgentProfile driven through the REAL -// runLoop kernel + inProcessSandboxClient (no creds, no network). The agent's -// per-prompt behaviour is a deterministic stub that emits research sources for -// the gaps it is told about, encoded in the prompt the worker hands it. -// --------------------------------------------------------------------------- - -interface StubFinding { - uri: string - title: string - text: string -} - -/** A trivial output the offline agent emits: the sources it "found" this turn. */ -interface AgentResearch { - sources: StubFinding[] -} - -const agentOutput: OutputAdapter = { - parse(events: SandboxEvent[]): AgentResearch { - for (let i = events.length - 1; i >= 0; i -= 1) { - const event = events[i] - const data = (event?.data ?? {}) as { result?: AgentResearch } - if (event?.type === 'result' && data.result) return data.result - } - return { sources: [] } - }, -} - -/** A one-shot driver for the inner researcher loop: run once, then done. */ -function oneShotDriver(): Driver<{ prompt: string }, AgentResearch, 'done'> { - return { - name: 'researcher-once', - async plan(_task, history) { - return history.length === 0 ? [_task] : [] - }, - decide() { - return 'done' - }, - } -} - -/** - * Build a `ResearchWorker` backed by the offline researcher agent. The agent is - * a corpus keyed by query token: when a gap mentions a token in `corpus`, the - * agent returns the matching finding. This proves the loop drives a real - * AgentProfile through runLoop + inProcessSandboxClient end-to-end offline. - */ -function offlineResearchWorker(corpus: StubFinding[]): ResearchWorker { - const { profile } = researcherProfile({ harness: 'offline-stub' }) - return async (ctx: WorkerResearchContext): Promise => { - const sandboxClient = inProcessSandboxClient({ - onPrompt(prompt): SandboxEvent[] { - // The agent "researches": return every corpus finding whose token the - // prompt (gap queries the worker folded in) mentions. - const found = corpus.filter((finding) => - finding.uri - .split(/[^a-z0-9]+/i) - .some( - (token) => token.length > 2 && prompt.toLowerCase().includes(token.toLowerCase()), - ), - ) - return [{ type: 'result', data: { result: { sources: found } } }] - }, - }) - - const queries = ctx.gaps.map((gap) => `${gap.description} ${gap.query}`).join(' | ') - const result = await runLoop<{ prompt: string }, AgentResearch, 'done'>({ - driver: oneShotDriver(), - agentRun: { - profile, - taskToPrompt: (task) => task.prompt, - name: 'offline-researcher', - }, - output: agentOutput, - task: { prompt: `Research these gaps for "${ctx.goal}": ${queries}\n${ctx.steer ?? ''}` }, - ctx: { sandboxClient }, - }) - - const sources: ResearchSourceProposal[] = (result.winner?.output.sources ?? []).map( - (finding) => ({ uri: finding.uri, text: finding.text, title: finding.title }), - ) - return { - sources, - // Curated, searchable pages cite the sources the driver accepted. The - // readiness gate searches PAGES, so growing the KB = sources + citing pages. - buildPages: pageBuilder(sources), - notes: `offline worker found ${sources.length} source(s)`, - } - } -} - -/** - * Build curated `knowledge/*.md` pages that cite the accepted sources, matched - * back to their proposals by `metadata.originalUri`. The page text repeats the - * source text so the token-based readiness search hits it. - */ -function pageBuilder(proposals: ResearchSourceProposal[]) { - return (acceptedSources: { id: string; metadata?: Record }[]): string => { - return acceptedSources - .map((record) => { - const proposal = proposals.find((p) => p.uri === record.metadata?.originalUri) - const slug = (proposal?.uri ?? record.id).replace(/[^a-z0-9]+/gi, '-') - return [ - `---FILE: knowledge/${slug}.md---`, - '---', - `title: ${proposal?.title ?? record.id}`, - `sources: ["${record.id}"]`, - '---', - `# ${proposal?.title ?? record.id}`, - proposal?.text ?? '', - '---END FILE---', - ].join('\n') - }) - .join('\n') - } -} - -/** - * A verifying driver: rejects any source whose uri starts with `bad/` (the - * deliberately-bad source), accepts the rest. Gates on readiness (the loop does - * that), and does NOT research (pure coordinator) unless `research` is supplied. - */ -function verifyingDriver(): ResearchDriver { - return { - verifySource(source: ResearchSourceProposal): SourceVerdict { - if (source.uri.startsWith('bad/')) { - return { accept: false, reason: 'source is not a real/relevant reference' } - } - return { accept: true } - }, - } -} - -let root: string - -beforeEach(async () => { - root = await mkdtemp(join(tmpdir(), 'two-agent-kb-')) -}) -afterEach(async () => { - await rm(root, { recursive: true, force: true }) -}) - -const specs = [ - defineReadinessSpec({ - id: 'topic/ventilation', - description: 'coop ventilation requirements', - query: 'coop ventilation airflow', - requiredFor: ['IntakeAgent'], - // `blocking` importance makes the gate actually gate: an unmet requirement - // lands in `blockingMissingRequirements` (a soft `high` requirement would - // only be a non-blocking gap and never stop the loop). - importance: 'blocking', - minSources: 1, - minHits: 1, - }), - defineReadinessSpec({ - id: 'topic/predators', - description: 'predator protection requirements', - query: 'predator protection fencing', - requiredFor: ['IntakeAgent'], - importance: 'blocking', - minSources: 1, - minHits: 1, - }), -] - -describe('runTwoAgentResearchLoop (offline, real FileSystemKbStore temp dir)', () => { - it('(a) GROWS the knowledge base across rounds via the offline agent', async () => { - const corpus: StubFinding[] = [ - { - uri: 'web/ventilation-guide', - title: 'Coop Ventilation Guide', - text: 'Coop ventilation airflow keeps the flock healthy. Cross-ventilation near the roof.', - }, - { - uri: 'web/predator-fencing', - title: 'Predator Protection', - text: 'Predator protection fencing and hardware cloth stop raccoons and foxes.', - }, - ] - - const result = await runTwoAgentResearchLoop({ - root, - goal: 'backyard chicken coop requirements', - worker: offlineResearchWorker(corpus), - driver: verifyingDriver(), - readinessSpecs: specs, - maxRounds: 3, - }) - - const index = await buildKnowledgeIndex(root) - // The KB grew: both corpus sources were registered. - expect(index.sources.length).toBeGreaterThanOrEqual(2) - const originalUris = index.sources.map((s) => s.metadata?.originalUri) - expect(originalUris).toContain('web/ventilation-guide') - expect(originalUris).toContain('web/predator-fencing') - // Sources are searchable through the existing search atom. - expect(searchKnowledge(index, 'ventilation airflow', 5).length).toBeGreaterThan(0) - // At least one round ran and recorded accepted sources. - expect(result.steps.length).toBeGreaterThanOrEqual(1) - expect( - result.steps.reduce((n, step) => n + step.acceptedWorkerSources.length, 0), - ).toBeGreaterThanOrEqual(2) - }) - - it("(b) the driver's verification REJECTS a deliberately-bad source the worker proposes", async () => { - const corpus: StubFinding[] = [ - { - uri: 'web/ventilation-guide', - title: 'Coop Ventilation Guide', - text: 'Coop ventilation airflow keeps the flock healthy.', - }, - { - // The deliberately-bad source: relevant-looking query tokens, but the - // verifier rejects anything under `bad/`. - uri: 'bad/ventilation-spam', - title: 'Buy Ventilation Fans Cheap', - text: 'ventilation airflow predator — SPAM advertisement, not a real reference.', - }, - ] - - const result = await runTwoAgentResearchLoop({ - root, - goal: 'backyard chicken coop requirements', - worker: offlineResearchWorker(corpus), - driver: verifyingDriver(), - readinessSpecs: specs, - maxRounds: 2, - }) - - const rejected = result.steps.flatMap((step) => step.rejectedWorkerSources) - expect(rejected.length).toBeGreaterThanOrEqual(1) - expect(rejected.some((r) => r.source.uri === 'bad/ventilation-spam')).toBe(true) - expect(rejected[0]?.reason).toMatch(/real\/relevant|reference/) - - // The bad source NEVER reached the knowledge base. - const index = await buildKnowledgeIndex(root) - const originalUris = index.sources.map((s) => s.metadata?.originalUri) - expect(originalUris).not.toContain('bad/ventilation-spam') - expect(originalUris).toContain('web/ventilation-guide') - }) - - it('(c) STOPS as soon as scoreKnowledgeReadiness reports the KB is ready', async () => { - const corpus: StubFinding[] = [ - { - uri: 'web/ventilation-guide', - title: 'Coop Ventilation Guide', - text: 'Coop ventilation airflow keeps the flock healthy.', - }, - { - uri: 'web/predator-fencing', - title: 'Predator Protection', - text: 'Predator protection fencing stops raccoons and foxes.', - }, - ] - - let rounds = 0 - const result = await runTwoAgentResearchLoop({ - root, - goal: 'backyard chicken coop requirements', - worker: offlineResearchWorker(corpus), - driver: verifyingDriver(), - readinessSpecs: specs, - maxRounds: 5, - onRound: () => { - rounds += 1 - }, - }) - - // The gate ended the loop BEFORE maxRounds. - expect(result.ready).toBe(true) - expect(result.rounds).toBeLessThan(5) - expect(rounds).toBe(result.rounds) - - // Ground-truth the stop with the same scorer the loop gates on. - const index = await buildKnowledgeIndex(root) - const report = scoreKnowledgeReadiness({ - taskId: 'backyard chicken coop requirements', - requirements: result.readiness?.requirements ?? [], - }) - expect(report.blockingMissingRequirements.length).toBe(0) - // The last recorded round is the one that flipped `ready`. - expect(result.steps.at(-1)?.ready).toBe(true) - }) - - it('driverResearches=false is the pure-coordinator mode (driver adds no sources)', async () => { - const corpus: StubFinding[] = [ - { - uri: 'web/ventilation-guide', - title: 'Coop Ventilation Guide', - text: 'Coop ventilation airflow keeps the flock healthy.', - }, - { - uri: 'web/predator-fencing', - title: 'Predator Protection', - text: 'Predator protection fencing stops raccoons.', - }, - ] - const result = await runTwoAgentResearchLoop({ - root, - goal: 'backyard chicken coop requirements', - worker: offlineResearchWorker(corpus), - driver: verifyingDriver(), - driverResearches: false, - readinessSpecs: specs, - maxRounds: 3, - }) - // Coordinator-only: the driver contributed zero sources of its own. - expect(result.steps.every((step) => step.driverSources.length === 0)).toBe(true) - }) - - it('driverResearches=true lets the driver gap-fill what the worker missed', async () => { - // The worker only ever finds the ventilation source; the predator gap stays - // open until the DRIVER fills it in its own research pass. - const workerCorpus: StubFinding[] = [ - { - uri: 'web/ventilation-guide', - title: 'Coop Ventilation Guide', - text: 'Coop ventilation airflow keeps the flock healthy.', - }, - ] - const driverFinding: ResearchSourceProposal = { - uri: 'web/predator-fencing', - title: 'Predator Protection', - text: 'Predator protection fencing stops raccoons and foxes.', - } - const driver: ResearchDriver = { - ...verifyingDriver(), - research(ctx) { - // Fill the remaining (predator) gap if it is still open. - const needsPredator = ctx.remainingGaps.some((gap) => gap.id === 'topic/predators') - return needsPredator - ? { sources: [driverFinding], buildPages: pageBuilder([driverFinding]) } - : {} - }, - } - - const result = await runTwoAgentResearchLoop({ - root, - goal: 'backyard chicken coop requirements', - worker: offlineResearchWorker(workerCorpus), - driver, - driverResearches: true, - readinessSpecs: specs, - maxRounds: 3, - }) - - expect(result.steps.reduce((n, s) => n + s.driverSources.length, 0)).toBeGreaterThanOrEqual(1) - const index = await buildKnowledgeIndex(root) - const originalUris = index.sources.map((s) => s.metadata?.originalUri) - expect(originalUris).toContain('web/predator-fencing') - expect(result.ready).toBe(true) - }) - - // Live variant — gated on creds; runs a real harness instead of the offline - // inProcessSandboxClient. Skipped unless AGENT_KNOWLEDGE_LIVE is set. - it.skipIf(!process.env.AGENT_KNOWLEDGE_LIVE)( - 'live: drives a real researcher harness through the two-agent loop', - async () => { - const result = await runTwoAgentResearchLoop({ - root, - goal: 'backyard chicken coop requirements', - // A live worker would drive researcherProfile over a real backend. - worker: offlineResearchWorker([]), - driver: verifyingDriver(), - readinessSpecs: specs, - maxRounds: 2, - }) - expect(result.rounds).toBeGreaterThanOrEqual(1) - }, - ) -}) diff --git a/tests/profiles/researcher.test.ts b/tests/profiles/researcher.test.ts deleted file mode 100644 index b14d0f7..0000000 --- a/tests/profiles/researcher.test.ts +++ /dev/null @@ -1,483 +0,0 @@ -import type { SandboxEvent } from '@tangle-network/sandbox' -import { describe, expect, it } from 'vitest' -import { - createResearcherValidator, - type KnowledgeItem, - multiHarnessResearcherFanout, - type ResearchOutput, - type ResearchTask, - researcherProfile, -} from '../../src/profiles/researcher' - -function ageMs(days: number): number { - return Date.now() - days * 24 * 60 * 60 * 1000 -} - -function item(overrides: Partial = {}): KnowledgeItem { - return { - id: overrides.id ?? 'item-1', - namespace: overrides.namespace ?? 'cust_42', - claim: overrides.claim ?? 'cpg-founder ICP engages with founder-pov threads', - evidence: overrides.evidence ?? [ - { - source: 'twitter', - quote: 'Engagement rate 4.2x', - url: 'https://x.com/example/status/1', - capturedAt: ageMs(7), - }, - ], - confidence: overrides.confidence ?? 0.82, - authoredBy: overrides.authoredBy ?? { kind: 'agent', id: 'researcher-glm' }, - ...overrides, - } -} - -function output(overrides: Partial = {}): ResearchOutput { - const items = overrides.items ?? [item()] - return { - items, - citations: overrides.citations ?? [ - { url: 'https://x.com/example/status/1', quote: 'Engagement rate 4.2x', confidence: 0.82 }, - ], - proposedWrites: - overrides.proposedWrites ?? - items.map((i) => ({ - kind: 'insert' as const, - namespace: i.namespace, - item: i, - })), - gaps: overrides.gaps, - notes: overrides.notes, - raw: overrides.raw, - } -} - -const task: ResearchTask = { - question: 'What does cpg-founder ICP engage with on Twitter?', - knowledgeNamespace: 'cust_42', - sources: ['twitter', 'web'], - maxItems: 10, - minConfidence: 0.6, -} - -describe('researcherProfile()', () => { - it('builds an AgentProfile + AgentRunSpec with role=researcher metadata', () => { - const { profile, agentRunSpec } = researcherProfile({ - harness: 'opencode/zai-coding-plan/glm-5.1', - }) - expect(profile.name).toBe('researcher-opencode/zai-coding-plan/glm-5.1') - expect(profile.metadata?.role).toBe('researcher') - expect(profile.metadata?.backendType).toBe('opencode/zai-coding-plan/glm-5.1') - expect(profile.tools).toMatchObject({ web_search: true, fs: true }) - expect(agentRunSpec.name).toBe(profile.name) - expect(typeof agentRunSpec.taskToPrompt).toBe('function') - }) - - it('formats the prompt with namespace + scope + recency window', () => { - const { taskToPrompt } = researcherProfile() - const prompt = taskToPrompt({ - question: 'q?', - knowledgeNamespace: 'ws_1', - scope: 'b2b SaaS', - sources: ['web', 'twitter'], - recencyWindow: { since: new Date('2026-01-01'), until: new Date('2026-05-01') }, - maxItems: 5, - minConfidence: 0.7, - }) - expect(prompt).toContain('Knowledge namespace (DO NOT cross): ws_1') - expect(prompt).toContain('Scope: b2b SaaS') - expect(prompt).toContain('Preferred sources: web, twitter') - expect(prompt).toContain('Recency window: 2026-01-01T00:00:00.000Z .. 2026-05-01T00:00:00.000Z') - expect(prompt).toContain('Max items: 5') - }) -}) - -describe('validator scoring', () => { - it('passes a fully-grounded, in-namespace output', async () => { - const validator = createResearcherValidator(task) - const verdict = await validator.validate(output(), { - iteration: 0, - signal: new AbortController().signal, - }) - expect(verdict.valid).toBe(true) - expect(verdict.score).toBeGreaterThan(0.5) - expect(verdict.scores?.citation_density).toBe(1) - expect(verdict.scores?.provenance).toBe(1) - expect(verdict.scores?.namespace).toBe(1) - }) - - it('hard-fails when items.length === 0', async () => { - const validator = createResearcherValidator(task) - const verdict = await validator.validate( - output({ items: [], proposedWrites: [], citations: [] }), - { iteration: 0, signal: new AbortController().signal }, - ) - expect(verdict.valid).toBe(false) - expect(verdict.notes).toContain('no items') - }) - - it('hard-fails when any item lacks evidence (provenance missing)', async () => { - const validator = createResearcherValidator(task) - const verdict = await validator.validate(output({ items: [item({ evidence: [] })] }), { - iteration: 0, - signal: new AbortController().signal, - }) - expect(verdict.valid).toBe(false) - expect(verdict.notes).toContain('without evidence') - expect(verdict.scores?.provenance).toBe(0) - }) - - it('hard-fails on cross-namespace item leak', async () => { - const validator = createResearcherValidator(task) - const verdict = await validator.validate( - output({ items: [item({ namespace: 'cust_99' })], proposedWrites: [] }), - { iteration: 0, signal: new AbortController().signal }, - ) - expect(verdict.valid).toBe(false) - expect(verdict.notes).toMatch(/namespace violation/) - expect(verdict.scores?.namespace).toBe(0) - }) - - it('hard-fails on cross-namespace proposedWrite leak (even when items are clean)', async () => { - const validator = createResearcherValidator(task) - const cleanItem = item({ namespace: 'cust_42' }) - const verdict = await validator.validate( - output({ - items: [cleanItem], - proposedWrites: [{ kind: 'insert', namespace: 'cust_99', item: cleanItem }], - }), - { iteration: 0, signal: new AbortController().signal }, - ) - expect(verdict.valid).toBe(false) - expect(verdict.notes).toMatch(/namespace violation/) - }) - - it('hard-fails when citation density falls below the floor', async () => { - const validator = createResearcherValidator(task, { citationDensityMin: 0.8 }) - const itemsArr = [ - item({ id: 'a' }), - item({ id: 'b' }), - item({ id: 'c' }), - item({ id: 'd' }), - item({ id: 'e' }), - ] - const verdict = await validator.validate( - output({ - items: itemsArr, - citations: [{ url: 'https://x.com/1', quote: 'q1', confidence: 0.8 }], - proposedWrites: itemsArr.map((i) => ({ - kind: 'insert' as const, - namespace: i.namespace, - item: i, - })), - }), - { iteration: 0, signal: new AbortController().signal }, - ) - expect(verdict.valid).toBe(false) - expect(verdict.notes).toMatch(/citation density 0\.20 below floor 0\.80/) - }) - - it('rewards source diversity in the aggregate score', async () => { - const validator = createResearcherValidator(task) - const single = await validator.validate( - output({ - items: [ - item({ id: 'a', evidence: [{ source: 'twitter', capturedAt: ageMs(1) }] }), - item({ id: 'b', evidence: [{ source: 'twitter', capturedAt: ageMs(2) }] }), - ], - citations: [ - { url: 'https://x.com/1', quote: 'q', confidence: 0.8 }, - { url: 'https://x.com/2', quote: 'q', confidence: 0.8 }, - ], - proposedWrites: [], - }), - { iteration: 0, signal: new AbortController().signal }, - ) - const diverse = await validator.validate( - output({ - items: [ - item({ id: 'a', evidence: [{ source: 'twitter', capturedAt: ageMs(1) }] }), - item({ id: 'b', evidence: [{ source: 'github', capturedAt: ageMs(2) }] }), - ], - citations: [ - { url: 'https://x.com/1', quote: 'q', confidence: 0.8 }, - { url: 'https://github.com/1', quote: 'q', confidence: 0.8 }, - ], - proposedWrites: [], - }), - { iteration: 0, signal: new AbortController().signal }, - ) - expect(diverse.scores?.source_diversity ?? 0).toBeGreaterThan( - single.scores?.source_diversity ?? 0, - ) - expect(diverse.score).toBeGreaterThan(single.score) - }) - - it('penalises recency mismatches when a recencyWindow is supplied', async () => { - const windowed: ResearchTask = { - ...task, - recencyWindow: { since: new Date(ageMs(5)) }, - } - const validator = createResearcherValidator(windowed) - const stale = await validator.validate( - output({ - items: [item({ evidence: [{ source: 'twitter', capturedAt: ageMs(100) }] })], - }), - { iteration: 0, signal: new AbortController().signal }, - ) - const fresh = await validator.validate( - output({ - items: [item({ evidence: [{ source: 'twitter', capturedAt: ageMs(1) }] })], - }), - { iteration: 0, signal: new AbortController().signal }, - ) - expect(stale.scores?.recency_match).toBe(0) - expect(fresh.scores?.recency_match).toBe(1) - }) - - it('penalises gaps in the score, never auto-fails on gaps alone', async () => { - const validator = createResearcherValidator(task) - const noGaps = await validator.validate(output(), { - iteration: 0, - signal: new AbortController().signal, - }) - const withGaps = await validator.validate( - output({ gaps: ['no data on Q4 2025 engagement window'] }), - { iteration: 0, signal: new AbortController().signal }, - ) - expect(withGaps.valid).toBe(true) - expect(withGaps.scores?.gap_coverage ?? 0).toBeLessThan(noGaps.scores?.gap_coverage ?? 1) - }) - - it('namespaceCheck=false disables the namespace gate (for shared validators)', async () => { - const validator = createResearcherValidator(task, { namespaceCheck: false }) - const verdict = await validator.validate( - output({ items: [item({ namespace: 'cust_99' })], proposedWrites: [] }), - { iteration: 0, signal: new AbortController().signal }, - ) - expect(verdict.valid).toBe(true) - expect(verdict.scores?.namespace).toBeUndefined() - }) -}) - -describe('propose-without-materialize invariant', () => { - it('emits proposedWrites but never invokes any FS / KB primitive', async () => { - // The profile has no FS dependency by construction. We verify by - // checking that `researcherProfile()` returns the same shape on - // every call without touching disk — and by inspecting the static - // imports of the module. - const { profile, output: adapter, validator, taskToPrompt, agentRunSpec } = researcherProfile() - expect(profile).toBeDefined() - expect(adapter.parse([])).toEqual({ items: [], citations: [], proposedWrites: [] }) - expect(typeof validator.validate).toBe('function') - expect(typeof taskToPrompt).toBe('function') - expect(agentRunSpec.profile).toBe(profile) - // The caller — not the profile — applies updates. Sanity-check that - // the profile module does not import any FS / KB primitive that would - // let it materialize writes itself. - const modText = await import('node:fs/promises').then((fs) => - fs.readFile(new URL('../../src/profiles/researcher.ts', import.meta.url), 'utf8'), - ) - // Forbid imports from the materialize / FS surface. - expect(modText).not.toMatch(/from ['"]\.\.\/proposals['"]/) - expect(modText).not.toMatch(/from ['"]\.\.\/store['"]/) - expect(modText).not.toMatch(/from ['"]\.\.\/kb-store['"]/) - expect(modText).not.toMatch(/from ['"]\.\.\/sources['"]/) - expect(modText).not.toMatch(/from ['"]node:fs(\/promises)?['"]/) - // Forbid actually calling the materialize / write APIs anywhere - // outside an `@example` or fenced-prose docblock. We accept the - // identifier inside source-level prose (audit context) but never as - // a callable identifier — a callable is `name(` with parens. - expect(modText).not.toMatch(/applyKnowledgeWriteBlocks\s*\(/) - expect(modText).not.toMatch(/writeFile\s*\(/) - expect(modText).not.toMatch(/addSourcePath\s*\(/) - expect(modText).not.toMatch(/addSourceText\s*\(/) - }) -}) - -describe('loose-output passthrough', () => { - it('parses a result event with the typed shape', () => { - const { output: adapter } = researcherProfile() - const items = [item()] - const events: SandboxEvent[] = [ - { - type: 'result', - data: { - result: { - items, - citations: [{ url: 'https://x.com/1', quote: 'q', confidence: 0.8 }], - proposedWrites: [{ kind: 'insert', namespace: 'cust_42', item: items[0] }], - gaps: ['no Q4 data'], - notes: 'inspected 4 sources', - }, - }, - }, - ] - const parsed = adapter.parse(events) - expect(parsed.items).toHaveLength(1) - expect(parsed.items[0]?.namespace).toBe('cust_42') - expect(parsed.citations).toHaveLength(1) - expect(parsed.proposedWrites).toHaveLength(1) - expect(parsed.gaps).toEqual(['no Q4 data']) - expect(parsed.notes).toBe('inspected 4 sources') - }) - - it('preserves agent extras under raw — never drops free-form intelligence', () => { - const { output: adapter } = researcherProfile() - const events: SandboxEvent[] = [ - { - type: 'final', - data: { - result: { - items: [item()], - citations: [{ url: 'https://x.com/1', quote: 'q', confidence: 0.8 }], - proposedWrites: [], - customField: { agentMood: 'curious', extraInsight: 42 }, - anotherUnknown: ['a', 'b'], - }, - }, - }, - ] - const parsed = adapter.parse(events) - expect(parsed.items).toHaveLength(1) - expect(parsed.raw).toMatchObject({ - customField: { agentMood: 'curious', extraInsight: 42 }, - anotherUnknown: ['a', 'b'], - }) - }) - - it('parses a fenced JSON block from a text delta when no structured result exists', () => { - const { output: adapter } = researcherProfile() - const payload = { - items: [item()], - citations: [{ url: 'https://x.com/1', quote: 'q', confidence: 0.8 }], - proposedWrites: [], - } - const events: SandboxEvent[] = [ - { - type: 'text', - data: { delta: `here is my answer:\n\`\`\`json\n${JSON.stringify(payload)}\n\`\`\`\n` }, - }, - ] - const parsed = adapter.parse(events) - expect(parsed.items).toHaveLength(1) - expect(parsed.citations).toHaveLength(1) - }) - - it('drops items that lack required fields rather than crashing', () => { - const { output: adapter } = researcherProfile() - const events: SandboxEvent[] = [ - { - type: 'result', - data: { - result: { - items: [ - { - id: 'good', - namespace: 'cust_42', - claim: 'ok', - evidence: [{ source: 'twitter', capturedAt: 0 }], - confidence: 0.5, - authoredBy: { kind: 'agent', id: 'r' }, - }, - { id: 'no-claim', namespace: 'cust_42' }, - null, - ], - citations: [], - proposedWrites: [], - }, - }, - }, - ] - const parsed = adapter.parse(events) - expect(parsed.items).toHaveLength(1) - expect(parsed.items[0]?.id).toBe('good') - }) - - it('returns an empty result when no events match the expected shape', () => { - const { output: adapter } = researcherProfile() - const parsed = adapter.parse([{ type: 'noise', data: { random: 1 } }]) - expect(parsed).toEqual({ items: [], citations: [], proposedWrites: [] }) - }) - - it('parses a result event whose answer is only in finalText (opencode)', () => { - const { output: adapter } = researcherProfile() - const payload = { - items: [item()], - citations: [{ url: 'https://x.com/1', quote: 'q', confidence: 0.8 }], - proposedWrites: [{ kind: 'insert', namespace: 'cust_42', item: item() }], - } - // opencode puts the terminal answer in `finalText`, not `result`/`output`. - const events: SandboxEvent[] = [ - { - type: 'result', - data: { finalText: `done:\n\`\`\`json\n${JSON.stringify(payload)}\n\`\`\`\n` }, - }, - ] - const parsed = adapter.parse(events) - expect(parsed.items).toHaveLength(1) - expect(parsed.citations).toHaveLength(1) - expect(parsed.proposedWrites).toHaveLength(1) - }) - - it('mines fenced JSON from finalText in the text-scan fallback', () => { - const { output: adapter } = researcherProfile() - const payload = { items: [item()], citations: [], proposedWrites: [] } - const events: SandboxEvent[] = [ - { type: 'message', data: { finalText: `\`\`\`json\n${JSON.stringify(payload)}\n\`\`\`` } }, - ] - const parsed = adapter.parse(events) - expect(parsed.items).toHaveLength(1) - expect(parsed.citations).toHaveLength(0) - expect(parsed.proposedWrites).toHaveLength(0) - }) - - it('falls through to empty when finalText carries plain prose (no fenced JSON)', () => { - const { output: adapter } = researcherProfile() - const events: SandboxEvent[] = [ - { type: 'result', data: { finalText: 'I could not find anything relevant.' } }, - ] - expect(adapter.parse(events)).toEqual({ items: [], citations: [], proposedWrites: [] }) - }) - - it('prefers a rich finalText over an all-empty direct result shape on the same event', () => { - const { output: adapter } = researcherProfile() - const payload = { items: [item()], citations: [], proposedWrites: [] } - // Same event carries BOTH an empty typed shape and a rich finalText — the empty - // shape must not shadow the real answer. - const events: SandboxEvent[] = [ - { - type: 'result', - data: { - items: [], - citations: [], - proposedWrites: [], - finalText: `\`\`\`json\n${JSON.stringify(payload)}\n\`\`\``, - }, - }, - ] - expect(adapter.parse(events).items).toHaveLength(1) - }) -}) - -describe('multiHarnessResearcherFanout', () => { - it('builds N AgentRunSpecs with a single-fanout-then-stop driver', () => { - const fan = multiHarnessResearcherFanout({ - harnesses: ['claude-code', 'codex', 'opencode/zai-coding-plan/glm-5.1'], - }) - expect(fan.agentRuns).toHaveLength(3) - expect(fan.agentRuns.map((spec) => spec.name)).toEqual([ - 'researcher-claude-code', - 'researcher-codex', - 'researcher-opencode/zai-coding-plan/glm-5.1', - ]) - expect(typeof fan.driver.plan).toBe('function') - expect(typeof fan.driver.decide).toBe('function') - expect(fan.driver.name).toBe('researcher-fanout') - }) - - it('falls back to three default harnesses when none supplied', () => { - const fan = multiHarnessResearcherFanout() - expect(fan.agentRuns).toHaveLength(3) - }) -}) diff --git a/tsup.config.ts b/tsup.config.ts index 84f7eab..12c204f 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -7,8 +7,6 @@ export default defineConfig({ cli: 'src/cli.ts', 'memory/index': 'src/memory/index.ts', 'sources/index': 'src/sources/index.ts', - 'profiles/index': 'src/profiles/index.ts', - 'autodata/index': 'src/autodata/index.ts', 'benchmarks/index': 'src/benchmarks/index.ts', }, format: ['esm'],