diff --git a/docs/api/index.md b/docs/api/index.md index 40e89d8..287d684 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -2698,11 +2698,24 @@ CODE-surface wiring with prompt-parity DX: name `surface: 'code'`, point at a `opts.generator` is supplied. Without either, `surface: 'code'` still fails loud — there is no safe zero-config repo to invent. +##### skills? + +> `optional` **skills?**: `ImproveSkillsOptions` + +Defined in: [improvement/improve.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L111) + +SKILLS-surface wiring for real skill-DOCUMENT optimization. Without this, + `surface: 'skills'` optimizes the profile's skills REFS array (file pointers) + — which `skillOptProposer` (a document patcher) cannot meaningfully edit. + Provide the document CONTENT to optimize + a `writeBack` to persist the + shipped winner (the profile ref points at a file the caller owns). This is + what makes skillOpt reachable through improve(). + ##### storage? > `optional` **storage?**: `CampaignStorage` -Defined in: [improvement/improve.ts:106](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L106) +Defined in: [improvement/improve.ts:113](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L113) Storage passthrough to `selfImprove`; overrides the default chosen from `runDir`. @@ -2710,7 +2723,7 @@ Storage passthrough to `selfImprove`; overrides the default chosen from `runDir` ### ImproveResult -Defined in: [improvement/improve.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L128) +Defined in: [improvement/improve.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L144) #### Type Parameters @@ -2728,7 +2741,7 @@ Defined in: [improvement/improve.ts:128](https://github.com/tangle-network/agent > **profile**: `AgentProfile` -Defined in: [improvement/improve.ts:131](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L131) +Defined in: [improvement/improve.ts:147](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L147) The profile after improvement: the winner surface applied back into the matching field when the gate shipped, else the input profile unchanged. @@ -2737,7 +2750,7 @@ The profile after improvement: the winner surface applied back into the > **shipped**: `boolean` -Defined in: [improvement/improve.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L133) +Defined in: [improvement/improve.ts:149](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L149) True when `gateDecision === 'ship'`. @@ -2745,7 +2758,7 @@ True when `gateDecision === 'ship'`. > **lift**: `number` -Defined in: [improvement/improve.ts:135](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L135) +Defined in: [improvement/improve.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L151) Held-out lift (`winner − baseline` composite). @@ -2753,7 +2766,7 @@ Held-out lift (`winner − baseline` composite). > **gateDecision**: `"ship"` \| `"hold"` \| `"need_more_work"` \| `"model_ceiling"` \| `"arch_ceiling"` -Defined in: [improvement/improve.ts:137](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L137) +Defined in: [improvement/improve.ts:153](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L153) The five-valued gate verdict from `selfImprove`. @@ -2761,7 +2774,7 @@ The five-valued gate verdict from `selfImprove`. > **raw**: `SelfImproveResult`\<`TScenario`, `TArtifact`\> -Defined in: [improvement/improve.ts:139](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L139) +Defined in: [improvement/improve.ts:155](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L155) Full `selfImprove` result for advanced inspection. @@ -7913,7 +7926,7 @@ Build the starting instruction for a coder agent tasked with implementing a new > **improve**\<`TScenario`, `TArtifact`\>(`profile`, `findings`, `opts`): `Promise`\<[`ImproveResult`](#improveresult)\<`TScenario`, `TArtifact`\>\> -Defined in: [improvement/improve.ts:325](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L325) +Defined in: [improvement/improve.ts:347](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L347) Run the held-out-gated self-improvement loop on ONE profile surface. diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index d1715f0..ea71e5e 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.89.0` and `@tangle-network/agent-eval@0.103.2` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.89.0` and `@tangle-network/agent-eval@0.106.2` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -949,7 +949,7 @@ Import from `@tangle-network/agent-eval` — 49 exports. ### CAMPAIGN — profile matrix, gates, improvement loop -Import from `@tangle-network/agent-eval/campaign` — 226 exports. +Import from `@tangle-network/agent-eval/campaign` — 229 exports. | Symbol | Kind | Summary | |---|---|---| @@ -981,7 +981,7 @@ Import from `@tangle-network/agent-eval/campaign` — 226 exports. | `gepaReflectionEntry` | function | GEPA, reflection-only (single-parent, no Pareto combine). | | `gitWorktreeAdapter` | function | Git-backed `WorktreeAdapter`: creates isolated worktrees on fresh branches, commits agent changes, and discards losers. | | `haloProposer` | function | Wrap the real halo-engine CLI as a SurfaceProposer (prompt-tier). | -| `heldOutGate` | function | Composable held-out delta gate: ships only when the candidate's mean composite on `scenarios` beats the baseline by at least `deltaThreshold`. | +| `heldOutGate` | function | Composable held-out gate: ships only when the PAIRED bootstrap CI lower bound | | `heldoutSignificance` | function | Significance of the held-out composite lift: ship only when the paired | | `inMemoryCampaignStorage` | function | In-memory storage for filesystem-less runtimes. Artifacts + trace spans | | `isProposedCandidate` | function | Type guard: a proposal carrying its rationale vs a bare | @@ -1001,6 +1001,7 @@ Import from `@tangle-network/agent-eval/campaign` — 226 exports. | `planCampaignRun` | function | Plan a campaign WITHOUT dispatching: computes the manifest hash and the per-cell | | `planEvalFixtureRun` | function | Dry-run planner for a fixture campaign: loads the scenarios, delegates to `planCampaignRun`, | | `policyEditProposer` | function | `SurfaceProposer` that admission-checks typed analyst `PolicyEdit`s and applies each | +| `powerPreflight` | function | Estimate the minimum detectable lift a paired-holdout improvement run can | | `provenanceRecordPath` | function | Canonical durable paths under the run dir. | | `provenanceSpansPath` | function | Canonical path for the durable OTLP spans JSONL file under a loop run directory. | | `renderScoreboardMarkdown` | function | Render the scoreboard as a launch-readiness Markdown document — the literal | @@ -1060,6 +1061,7 @@ Import from `@tangle-network/agent-eval/campaign` — 226 exports. | `PlaybackDriver` | interface | Drives the real product through a story and returns the runtime event stream | | `PlaybackStep` | interface | One step of a user story — what the user does. The driver interprets | | `PolicyEditProposerOptions` | interface | `policyEditProposer` turns typed analyst policy edits into measured candidate | +| `PowerPreflightOptions` | interface | Power preflight — "can this budget detect the effect you are hunting?" | | `ProposeContext` | interface | Everything a proposer may read to plan the next | | `ProposedCandidate` | interface | A proposer output carrying the surface AND the WHY behind | | `ProposerEntry` | interface | What an optimizer produced: the surface it promoted + what it cost to get | @@ -1095,7 +1097,7 @@ Import from `@tangle-network/agent-eval/campaign` — 226 exports. | `SequentialDecision` | type | Anytime-valid sequential promotion gate — an e-process (betting | | `SkillPatchOp` | type | A single bounded edit against a skill surface. | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcceptedEdit`, `ApplySkillPatchResult`, `AxisEvidence`, `BuildAnalystSurfaceDispatchOptions`, `BuildEvidenceVectorOptions`, `BuildLoopProvenanceArgs`, `CampaignAggregates`, `CampaignBreakdown`, `CampaignCellResult`, `CampaignResult`, `CampaignRunPlan`, `CampaignRunPlanCell`, `CompareProposersOptions`, `DimensionRegression`, `EmitLoopProvenanceArgs`, `EmitLoopProvenanceResult`, `EvalFixture`, `EvalFixtureFile`, `EvalFixtureLoadOptions`, `EvalFixtureScenario`, `EvidenceVector`, `FailureModeRecallJudgeOptions`, `FapoAttributionSignals`, `FapoFailureCluster`, `FapoProposerOptions`, `FapoReviewInput`, `FapoReviewIssue`, `FapoReviewResult`, `FapoScopeContract`, `GateContext`, `GateResult`, `GenerationRecord`, `GepaProposerOptions`, `GitWorktreeAdapterOptions`, `HeldOutGateOptions`, `HeldoutSignificance`, `HeldoutSignificanceOptions`, `JudgeAggregate`, `JudgeDimension`, `LabeledScenarioRecord`, `LabeledScenarioSampleArgs`, `LabeledScenarioStore`, `LlmJudgeOptions`, `LoadEvalFixtureScenariosOptions`, `LoopProvenanceBackend`, `LoopProvenanceCandidate`, `OpenAutoPrResult`, `OptimizerConfig`, `ParameterCandidate`, `ParameterChange`, `ParameterSweepProposerOptions`, `ParetoSignificanceGateOptions`, `PlanCampaignRunOptions`, `PlanEvalFixtureRunOptions`, `ProfileSummary`, `PromotionObjective`, `ProposePatchesArgs`, `ProposerComparison`, `ProposerPairwise`, `ProposerScore`, `RunImprovementLoopResult`, `RunOptimizationResult`, `RunProfileMatrixOptions`, `RunProfileMatrixResult`, `RunSkillOptResult`, `ScenarioAggregate`, `ScenarioRollup`, `ScoreboardRenderOptions`, `SequentialDecideFn`, `SequentialDecideOptions`, `SequentialObservation`, `SequentialPairedGate`, `SequentialPairedGateOptions`, `SkillOptEpochRecord`, `SkillOptProposer`, `SkillOptProposerOptions`, `SkillPatchRejection`, `TraceSpan`, `WorktreeAdapter`, `EvalFixtureRunPlan`, `EvalFixtureValidationMode`, `JsonPrimitive`, `JsonValue`, `RedactionStatus`, `RunOptimizationOptions`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcceptedEdit`, `ApplySkillPatchResult`, `AxisEvidence`, `BuildAnalystSurfaceDispatchOptions`, `BuildEvidenceVectorOptions`, `BuildLoopProvenanceArgs`, `CampaignAggregates`, `CampaignBreakdown`, `CampaignCellResult`, `CampaignResult`, `CampaignRunPlan`, `CampaignRunPlanCell`, `CompareProposersOptions`, `DimensionRegression`, `EmitLoopProvenanceArgs`, `EmitLoopProvenanceResult`, `EvalFixture`, `EvalFixtureFile`, `EvalFixtureLoadOptions`, `EvalFixtureScenario`, `EvidenceVector`, `FailureModeRecallJudgeOptions`, `FapoAttributionSignals`, `FapoFailureCluster`, `FapoProposerOptions`, `FapoReviewInput`, `FapoReviewIssue`, `FapoReviewResult`, `FapoScopeContract`, `GateContext`, `GateResult`, `GenerationRecord`, `GepaProposerOptions`, `GitWorktreeAdapterOptions`, `HeldOutGateOptions`, `HeldoutSignificance`, `HeldoutSignificanceOptions`, `JudgeAggregate`, `JudgeDimension`, `LabeledScenarioRecord`, `LabeledScenarioSampleArgs`, `LabeledScenarioStore`, `LlmJudgeOptions`, `LoadEvalFixtureScenariosOptions`, `LoopProvenanceBackend`, `LoopProvenanceCandidate`, `OpenAutoPrResult`, `OptimizerConfig`, `ParameterCandidate`, `ParameterChange`, `ParameterSweepProposerOptions`, `ParetoSignificanceGateOptions`, `PlanCampaignRunOptions`, `PlanEvalFixtureRunOptions`, `PowerPreflight`, `ProfileSummary`, `PromotionObjective`, `ProposePatchesArgs`, `ProposerComparison`, `ProposerPairwise`, `ProposerScore`, `RunImprovementLoopResult`, `RunOptimizationResult`, `RunProfileMatrixOptions`, `RunProfileMatrixResult`, `RunSkillOptResult`, `ScenarioAggregate`, `ScenarioRollup`, `ScoreboardRenderOptions`, `SequentialDecideFn`, `SequentialDecideOptions`, `SequentialObservation`, `SequentialPairedGate`, `SequentialPairedGateOptions`, `SkillOptEpochRecord`, `SkillOptProposer`, `SkillOptProposerOptions`, `SkillPatchRejection`, `TraceSpan`, `WorktreeAdapter`, `EvalFixtureRunPlan`, `EvalFixtureValidationMode`, `JsonPrimitive`, `JsonValue`, `RedactionStatus`, `RunOptimizationOptions`. ### TOKEN / USAGE — usage extraction + run-record usage types diff --git a/src/improvement/improve.test.ts b/src/improvement/improve.test.ts index b97aa24..758770f 100644 --- a/src/improvement/improve.test.ts +++ b/src/improvement/improve.test.ts @@ -119,6 +119,67 @@ describe('improve() — default proposer resolution (substrate export drift guar } }) + it("surface 'skills' with a document optimizes CONTENT and writes back the shipped winner", async () => { + // Baseline document; a scenario whose judge rewards the presence of a rule the + // skillOpt proposer will add. A deterministic stub proposer stands in for the LLM. + const baselineDoc = '# OR skills\n- always run a solver\n' + let writtenBack: string | null = null + const stubProposer = { + kind: 'stub-skillopt', + async propose(ctx: { currentSurface: unknown }) { + // Prove the baseline surface is the DOCUMENT, not a refs array. + expect(ctx.currentSurface).toBe(baselineDoc) + return [ + { + surface: `${baselineDoc}- recompute the objective before writing\n`, + label: 'add-recompute-rule', + rationale: 'stub', + }, + ] + }, + } + // Judge: reward the document that contains the added rule. + const docJudge: JudgeConfig<{ doc: string }, Scenario> = { + name: 'doc-judge', + dimensions: [{ key: 'q', description: 'has recompute rule' }], + score: ({ artifact }) => { + const has = artifact.doc.includes('recompute the objective') + return { dimensions: { q: has ? 1 : 0 }, composite: has ? 1 : 0, notes: '' } + }, + } + const skillProfileWithRef = (): AgentProfile => ({ + name: 'fixture-agent', + resources: { skills: [{ path: 'or-skills.md' } as never] }, + }) + + const result = await improve(skillProfileWithRef(), [], { + surface: 'skills', + scenarios, + judge: docJudge, + agent: async (surface, _s, ctx) => { + ctx.cost.observe(0.0001, 'stub') + ctx.cost.observeTokens({ input: 1, output: 1 }) + return { doc: String(surface) } + }, + generator: stubProposer as never, + skills: { + document: baselineDoc, + writeBack: (winner) => { + writtenBack = winner + }, + }, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.25 }, + }) + + expect(typeof result.gateDecision).toBe('string') + if (result.shipped) { + // The winner document (content, not a refs array) was written back. + expect(writtenBack).toContain('recompute the objective') + // The profile ref is unchanged (writeBack owns the file, not the profile). + expect(result.profile.resources?.skills).toEqual(skillProfileWithRef().resources?.skills) + } + }) + it('a surface with no zero-config default still fails loud with ConfigError', async () => { // The default-proposer map covers prompt + skills only; the config surfaces // (tools/mcp/hooks/code) require a caller-supplied generator. This is the diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index 1b7ad35..a6718fb 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -102,10 +102,26 @@ export interface ImproveOptions { * `opts.generator` is supplied. Without either, `surface: 'code'` still fails * loud — there is no safe zero-config repo to invent. */ code?: ImproveCodeOptions + /** SKILLS-surface wiring for real skill-DOCUMENT optimization. Without this, + * `surface: 'skills'` optimizes the profile's skills REFS array (file pointers) + * — which `skillOptProposer` (a document patcher) cannot meaningfully edit. + * Provide the document CONTENT to optimize + a `writeBack` to persist the + * shipped winner (the profile ref points at a file the caller owns). This is + * what makes skillOpt reachable through improve(). */ + skills?: ImproveSkillsOptions /** Storage passthrough to `selfImprove`; overrides the default chosen from `runDir`. */ storage?: SelfImproveOptions['storage'] } +export interface ImproveSkillsOptions { + /** The skill document's current text — the baseline `skillOptProposer` patches. */ + document: string + /** Persist the shipped winner document (write the file the profile ref points at). + * Called only on a ship verdict. When omitted, the winner is still returned in + * `result.raw.winner.surface` for the caller to materialize. */ + writeBack?: (winnerDocument: string) => void +} + export interface ImproveCodeOptions { /** Repo root candidate worktrees fork from. */ repoRoot: string @@ -169,12 +185,18 @@ function defaultGeneratorFor( /** Extract the baseline surface a driver mutates from the profile field that * backs `surface`. `prompt`/`skills` are string surfaces; the config surfaces * serialize the matching profile record. */ -function baselineSurfaceFor(profile: AgentProfile, surface: ImproveSurface): MutableSurface { +function baselineSurfaceFor( + profile: AgentProfile, + surface: ImproveSurface, + skills?: ImproveSkillsOptions, +): MutableSurface { switch (surface) { case 'prompt': return profile.prompt?.systemPrompt ?? '' case 'skills': - return JSON.stringify(profile.resources?.skills ?? []) + // With a document supplied, optimize its CONTENT (the real skillOpt path); + // otherwise fall back to the refs-array surface for back-compat. + return skills?.document ?? JSON.stringify(profile.resources?.skills ?? []) case 'tools': return JSON.stringify(profile.tools ?? {}) case 'mcp': @@ -351,7 +373,7 @@ export async function improve( agent: opts.agent, scenarios: opts.scenarios, judge: opts.judge, - baselineSurface: baselineSurfaceFor(profile, surface), + baselineSurface: baselineSurfaceFor(profile, surface, opts.skills), proposer, budget, llm: opts.llm, @@ -367,7 +389,17 @@ export async function improve( }) const shipped = raw.gateDecision === 'ship' - const nextProfile = shipped ? applyWinnerToProfile(profile, surface, raw.winner.surface) : profile + // When a skill DOCUMENT was optimized, the winner is document text — persist it + // via writeBack (the profile ref points at the caller's file, unchanged) rather + // than parsing it as a refs array. Otherwise use the standard field write-back. + const usedSkillDocument = surface === 'skills' && opts.skills !== undefined + if (shipped && usedSkillDocument && typeof raw.winner.surface === 'string') { + opts.skills?.writeBack?.(raw.winner.surface) + } + const nextProfile = + shipped && !usedSkillDocument + ? applyWinnerToProfile(profile, surface, raw.winner.surface) + : profile return { profile: nextProfile, shipped, lift: raw.lift, gateDecision: raw.gateDecision, raw } }