From 7ac3516df4764b9c2fb0fdbd553875df79f105be Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Thu, 9 Jul 2026 11:26:03 -0400 Subject: [PATCH] feat(harness): experiment-iteration reframe prototype (B4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prove one experiment iteration can run driven by a score + deterministic TS driver rather than an LLM orchestrator (spec-20260708-112732 §3.2/§4.3, plan slice B4, amicode#109). Stands on amicode#107's G-1 ruling: score-first for flow + a thin TS driver only where the stage model can't express iteration. What lands: - `amico-run` harness module (src/harness/): an iteration-score parser (the data-defined flow) + `runExperimentIteration`, the deterministic control-flow loop that replaces the orchestrator prompt for one iteration — select target → dispatch ONE flat/depth-1 experimenter leaf → launch via the amico-run CLI → re-rollout verify → record + gate promotion on `agree`. The single model seam (`dispatchExperimenter`) is pluggable; production wires it to `opencode run --agent experimenter`. - Re-rollout gate extended to TIER-2 (composed), not just tier-3/free (§4.3): `isVerifiedTier` is the single source of truth; cli.ts gates on it. - Example harness score at extension/scores/experiment-iteration/ITERATION.toml (ignored by the interview repertoire loader, which only reads SCORE.md). - Runnable demo (`pnpm --filter @amicode/amico-run demo:harness`) + tests that run the iteration with NO LLM and NO Julia (deterministic fakes), asserting: exactly one flat leaf, tier-2 re-rollout runs, promotion gated on agree. - s31 grep-guard now recurses src/ so it covers the new harness subdir (no fetch/MCP/HTTP in the tool layer). amico-run: 115 tests pass (was 106); typecheck clean; demo PASS. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/amico-run/esbuild.config.mjs | 26 +- packages/amico-run/harness-demo/fixtures.ts | 124 ++++++++ packages/amico-run/harness-demo/run_demo.ts | 93 ++++++ packages/amico-run/package.json | 3 +- packages/amico-run/src/cli.ts | 19 +- .../src/harness/experiment_iteration.ts | 272 ++++++++++++++++++ packages/amico-run/src/harness/index.ts | 5 + .../amico-run/src/harness/iteration_score.ts | 123 ++++++++ packages/amico-run/src/index.ts | 2 + packages/amico-run/src/verify.ts | 21 +- packages/amico-run/test/cli.test.ts | 62 ++++ .../test/experiment_iteration.test.ts | 142 +++++++++ packages/amico-run/test/s31.test.ts | 17 +- packages/amico-run/test/verify.test.ts | 11 +- packages/amico-run/tsconfig.json | 2 +- .../experiment-iteration/ITERATION.toml | 31 ++ .../scores/experiment-iteration/README.md | 56 ++++ 17 files changed, 988 insertions(+), 21 deletions(-) create mode 100644 packages/amico-run/harness-demo/fixtures.ts create mode 100644 packages/amico-run/harness-demo/run_demo.ts create mode 100644 packages/amico-run/src/harness/experiment_iteration.ts create mode 100644 packages/amico-run/src/harness/index.ts create mode 100644 packages/amico-run/src/harness/iteration_score.ts create mode 100644 packages/amico-run/test/experiment_iteration.test.ts create mode 100644 packages/extension/scores/experiment-iteration/ITERATION.toml create mode 100644 packages/extension/scores/experiment-iteration/README.md diff --git a/packages/amico-run/esbuild.config.mjs b/packages/amico-run/esbuild.config.mjs index 69a8d9b7..82039569 100644 --- a/packages/amico-run/esbuild.config.mjs +++ b/packages/amico-run/esbuild.config.mjs @@ -1,17 +1,33 @@ import { build } from "esbuild"; import { chmodSync } from "node:fs"; -await build({ - entryPoints: ["src/cli.ts"], +const common = { bundle: true, platform: "node", target: "node20", - // ESM, not CJS: the package is "type": "module", so node executes dist/amico-run.js + // ESM, not CJS: the package is "type": "module", so node executes the bundles // as ESM — a CJS bundle would die on `require is not defined in ES module scope`. format: "esm", - outfile: "dist/amico-run.js", - banner: { js: "#!/usr/bin/env node" }, sourcemap: true, logLevel: "info", +}; + +await build({ + ...common, + entryPoints: ["src/cli.ts"], + outfile: "dist/amico-run.js", + banner: { js: "#!/usr/bin/env node" }, }); chmodSync("dist/amico-run.js", 0o755); + +// Harness-reframe prototype demo (amicode#109, slice B4): bundles the harness +// driver + its deterministic fake experimenter leaf so `demo:harness` runs one +// iteration end-to-end with NO LLM and NO Julia. It shells out to the sibling +// dist/amico-run.js built above. +await build({ + ...common, + entryPoints: ["harness-demo/run_demo.ts"], + outfile: "dist/harness-demo.js", + banner: { js: "#!/usr/bin/env node" }, +}); +chmodSync("dist/harness-demo.js", 0o755); diff --git a/packages/amico-run/harness-demo/fixtures.ts b/packages/amico-run/harness-demo/fixtures.ts new file mode 100644 index 00000000..ca09ae44 --- /dev/null +++ b/packages/amico-run/harness-demo/fixtures.ts @@ -0,0 +1,124 @@ +// Deterministic fixtures for the harness-reframe prototype (amicode#109, B4). +// Shared by the runnable demo (run_demo.ts) and the test +// (test/experiment_iteration.test.ts). The whole point: the experimenter leaf +// and the re-rollout harness are FAKES, so the CONTROL FLOW is exercised with NO +// model and NO Julia — proving the iteration runs without an LLM orchestrator. +import { chmodSync, mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { maskedHash } from "../src/baseline.js"; +import type { ExperimenterDispatch } from "../src/harness/index.js"; + +/** The exemplar the (fake) experimenter leaf splices — a minimal Piccolo script + * with the template convention's default fill-point markers. */ +export const EXEMPLAR_SCRIPT = + `using Piccolo\n` + + `using JLD2, TOML\n` + + `# ── FILL IN ──────\n` + + `T = 10.0\n` + + `# ─────────────────\n` + + `# (a real leaf would build + serialize system_verify.jld2 here)\n` + + `solve()\n`; + +export interface FakeEnv { + juliaBin: string; + verifyHarness: string; + authoringFile: string; + /** Env for the amico-run CLI child (authoring path + verify runner). */ + env: Record; + /** The deterministic experimenter leaf — a plain function, NOT an LLM. */ + dispatchExperimenter: ExperimenterDispatch; + exemplarId: string; +} + +function writeExec(dir: string, name: string, body: string): string { + const p = join(dir, name); + writeFileSync(p, `#!/usr/bin/env node\n${body}\n`); + chmodSync(p, 0o755); + return p; +} + +/** + * Build a deterministic environment for one harness iteration: + * - a fake `julia` that writes result.toml into the run dir and exits 0; + * - a fake re-rollout harness that writes verification.toml with the requested + * `agree` (stands in for the VETTED Julia harness — the trust anchor); + * - the authoring.json the CLI gate + verify read; + * - a matching tier-2 exemplars index so the composed launch gate passes. + * The returned `dispatchExperimenter` authors the exemplar with an inside-fill- + * point edit — a pure function so the loop has no model in it. + */ +export function setupFakeIterationEnv(dir: string, opts: { agree: boolean } = { agree: true }): FakeEnv { + mkdirSync(dir, { recursive: true }); + const exemplarId = "demo-cz"; + + const juliaBin = writeExec( + dir, + "fake-julia", + `const fs=require('fs');\n` + + `fs.writeFileSync('result.toml','schema_version = "1"\\nfidelity = 0.9995\\niterations = 42\\n');\n` + + `console.log('AMICODE_ITER iter=1 f=0.5');\n` + + `console.log('DONE f=0.9995');`, + ); + + // Run by the CLI as `node ` (AMICO_VERIFY_RUNNER=node), + // so argv[2] is the run dir. + const verifyHarness = writeExec( + dir, + "fake-verify.js", + `const fs=require('fs'),p=require('path');\n` + + `const runDir=process.argv[2];\n` + + `fs.writeFileSync(p.join(runDir,'verification.toml'),\n` + + ` 'schema_version = "1"\\nagree = ${opts.agree}\\nfidelity_rerolled = 0.9994\\n' +\n` + + ` 'fidelity_reported = 0.9995\\ntolerance = 0.01\\nintegrator = "fake"\\n');`, + ); + + // exemplars index — baseline hash of the UNEDITED exemplar (masked outside its fill points). + const exemplarsIndex = join(dir, "exemplars-index.json"); + writeFileSync( + exemplarsIndex, + JSON.stringify({ + schema_version: 1, + exemplars: [ + { + id: exemplarId, + platform: "rydberg", + kind: "gate_synthesis", + size: 2, + path: `${exemplarId}/script.jl`, + packages: ["Piccolo", "JLD2", "TOML"], + baseline_hash: maskedHash(EXEMPLAR_SCRIPT), + }, + ], + }), + ); + + const authoringFile = join(dir, "authoring.json"); + writeFileSync( + authoringFile, + JSON.stringify({ + schema_version: 1, + allowlist: ["Piccolo", "Legato"], + support_set: ["JLD2", "CairoMakie", "TOML", "Printf"], + exemplars: exemplarsIndex, + verify_harness: verifyHarness, + verify_tolerance: 0.01, + }), + ); + + // The deterministic experimenter leaf. In production this is + // `opencode run --agent experimenter` (headless, flat); here it is a function. + const dispatchExperimenter: ExperimenterDispatch = async (_target, ctx) => { + const scriptPath = join(ctx.workdir, "solve.jl"); + writeFileSync(scriptPath, EXEMPLAR_SCRIPT.replace("T = 10.0", "T = 25.0")); + return { scriptPath, note: "spliced T=25.0 into demo-cz (fake leaf, no LLM)" }; + }; + + return { + juliaBin, + verifyHarness, + authoringFile, + env: { AMICO_AUTHORING_FILE: authoringFile, AMICO_VERIFY_RUNNER: "node" }, + dispatchExperimenter, + exemplarId, + }; +} diff --git a/packages/amico-run/harness-demo/run_demo.ts b/packages/amico-run/harness-demo/run_demo.ts new file mode 100644 index 00000000..19cf3132 --- /dev/null +++ b/packages/amico-run/harness-demo/run_demo.ts @@ -0,0 +1,93 @@ +// Runnable demo of the harness-reframe prototype (amicode#109, slice B4). +// +// pnpm --filter @amicode/amico-run build +// pnpm --filter @amicode/amico-run demo:harness +// +// Runs ONE experiment iteration driven entirely by code + data (an iteration +// score) — NO LLM in the control-flow loop and NO Julia. The experimenter leaf +// and the re-rollout harness are deterministic fakes; what is exercised is the +// deterministic control flow that replaces the 1117-line orchestrator prompt: +// select target → dispatch one flat leaf → launch via amico-run → tier-2 +// re-rollout verify → record + gate promotion on `agree`. +import { existsSync, mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseIterationScore, runExperimentIteration } from "../src/harness/index.js"; +import { setupFakeIterationEnv } from "./fixtures.js"; + +async function main(): Promise { + const here = dirname(fileURLToPath(import.meta.url)); // dist/ after bundling + const amicoRunBundle = resolve(here, "amico-run.js"); + if (!existsSync(amicoRunBundle)) { + console.error(`[demo] missing ${amicoRunBundle} — run: pnpm --filter @amicode/amico-run build`); + process.exit(1); + } + + // 1) The shipped harness score (data) parses — the flow lives as data. + const shipped = resolve(here, "..", "..", "extension", "scores", "experiment-iteration", "ITERATION.toml"); + if (existsSync(shipped)) { + const r = parseIterationScore(readFileSync(shipped, "utf8")); + console.log( + r.ok + ? `[demo] shipped ITERATION.toml parses → ${r.score.target.platform}/${r.score.target.gate} tier=${r.score.target.tier}, promote_on=${r.score.verify.promote_on}` + : `[demo] shipped score error: ${r.error}`, + ); + } + + // 2) Run one iteration against a demo-tuned score (exemplar matches the fakes). + const work = mkdtempSync(join(tmpdir(), "harness-demo-")); + const fake = setupFakeIterationEnv(join(work, "fixtures")); + const parsed = parseIterationScore( + [ + `schema_version = 1`, + `id = "experiment-iteration-demo"`, + `[target]`, + `platform = "rydberg"`, + `gate = "CZ"`, + `kind = "gate_synthesis"`, + `size = 2`, + `tier = "composed"`, + `exemplar_id = "${fake.exemplarId}"`, + `env = { kind = "provisioned" }`, + `[verify]`, + `promote_on = "agree"`, + ].join("\n"), + ); + if (!parsed.ok) { + console.error(`[demo] score error: ${parsed.error}`); + process.exit(1); + } + + console.log("\n[demo] ── running one iteration (control flow = code, one flat leaf) ──"); + const iterDir = join(work, "iter"); + const outcome = await runExperimentIteration(parsed.score, { + dispatchExperimenter: fake.dispatchExperimenter, // ← the ONLY model seam (here: a fake) + workdir: iterDir, + runsRoot: join(work, "runs"), + amicoRunBundle, + juliaBin: fake.juliaBin, + env: fake.env, + logger: (l) => console.log(l), + }); + + console.log("\n[demo] ── outcome ──"); + console.log(JSON.stringify(outcome, null, 2)); + console.log(`\n[demo] iteration.toml (${join(iterDir, "iteration.toml")}):`); + console.log(readFileSync(join(iterDir, "iteration.toml"), "utf8")); + + const ok = + outcome.dispatched === 1 && outcome.status === "completed" && outcome.verified === true && outcome.promoted === true; + if (!ok) { + console.error("[demo] UNEXPECTED outcome — the prototype did not behave as designed"); + process.exit(1); + } + console.log( + "[demo] PASS — one iteration ran: exactly one flat experimenter leaf, tier-2 re-rollout agreed, promotion gated on agree. No LLM in the control-flow loop.", + ); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/amico-run/package.json b/packages/amico-run/package.json index e2a197bc..4fa2a165 100644 --- a/packages/amico-run/package.json +++ b/packages/amico-run/package.json @@ -15,7 +15,8 @@ "build": "node esbuild.config.mjs", "typecheck": "tsc --noEmit", "test": "vitest run --passWithNoTests --exclude '**/slow/**'", - "test:slow": "vitest run test/slow" + "test:slow": "vitest run test/slow", + "demo:harness": "node esbuild.config.mjs && node dist/harness-demo.js" }, "dependencies": { "@amicode/schema": "workspace:*", diff --git a/packages/amico-run/src/cli.ts b/packages/amico-run/src/cli.ts index 2ef2ddc0..2e8225d0 100644 --- a/packages/amico-run/src/cli.ts +++ b/packages/amico-run/src/cli.ts @@ -5,7 +5,7 @@ import { LocalExecutor } from "./local_executor.js"; import { ConfigError, type Finished, type SubmitOpts } from "./types.js"; import { readAuthoring } from "./authoring.js"; import { runGate } from "./gate.js"; -import { runVerification } from "./verify.js"; +import { isVerifiedTier, runVerification } from "./verify.js"; import { trySubcommand } from "./subcommands.js"; function readTomlSafe(fp: string): Record | undefined { @@ -173,11 +173,18 @@ export async function main(argv: string[]): Promise { console.error(`amico-run: FINISHED missing in ${handle.runDir} (write fault)`); return 64; } - // spec C: free-tier re-rollout verification runs AFTER FINISHED, BEFORE the - // AMICODE_FINISHED line — so consumers see a settled verification state. The - // harness (or the fallback) always writes verification.toml; the promote gate - // keys off agree==true. - if (opts.spec?.tier === "free") { + // spec C + spec-20260708-112732 §4.3: re-rollout verification runs AFTER + // FINISHED, BEFORE the AMICODE_FINISHED line — so consumers see a settled + // verification state. The harness (or the fallback) always writes + // verification.toml; the promote gate keys off agree==true. + // + // TIERS VERIFIED: "free" (author-first, the original tier-3 trust anchor) AND + // "composed" (tier-2). The depth-1 redesign extends the gate to tier-2 because + // splicing params into an exemplar is exactly where a subtle mangle survives + // the masked-baseline check — a wrong fill can still re-roll to a different + // pulse. "vetted" (tier-1) skips it (the template is the trust anchor). Keep + // this set in sync with SolveSpec.tier (schema enum: vetted|composed|free). + if (opts.spec && isVerifiedTier(opts.spec.tier)) { const { config: authoring } = readAuthoring(); await runVerification(handle.runDir, opts.spec, authoring); const verified = readTomlSafe(join(handle.runDir, "verification.toml")); diff --git a/packages/amico-run/src/harness/experiment_iteration.ts b/packages/amico-run/src/harness/experiment_iteration.ts new file mode 100644 index 00000000..2d28ce19 --- /dev/null +++ b/packages/amico-run/src/harness/experiment_iteration.ts @@ -0,0 +1,272 @@ +// Experiment-iteration harness driver — the CONTROL-FLOW half of the harness +// reframe (spec-20260708-112732 §3.2/§4.3, plan slice B4). This is the module +// that DISSOLVES the 1117-line LLM `orchestrator` for one iteration: the loop +// (select target → dispatch one experimenter leaf → run the solve → re-rollout +// verify → record + gate promotion) is deterministic CODE, not an agent prompt. +// +// The ONLY place a model enters is `dispatchExperimenter` — a single, flat, +// depth-1 leaf that authors the Julia script. Everything around it is code: +// - the SolveSpec is DERIVED from the iteration score (data), not authored by +// the model — so tier/source/env are not the LLM's judgment; +// - the launch goes through the `amico-run` CLI (the §7.3 "harness calls the +// CLI directly" spine), which runs the launch gate and the tier-2/3 +// re-rollout verification; +// - promotion is gated on the re-rollout `agree` verdict, in code. +// +// The leaf dispatch is a pluggable seam so the loop is testable with NO model in +// the control-flow path. In production it is wired to `opencode run --agent +// experimenter` (headless, fire-and-forget → returns the authored script path) +// per §3.2; in tests/the demo it is a deterministic fake. +import { spawn } from "node:child_process"; +import { mkdirSync } from "node:fs"; +import { atomicWriteFile } from "../run_dir.js"; +import { isVerifiedTier } from "../verify.js"; +import type { IterationScore, IterationTarget, Tier } from "./iteration_score.js"; + +/** What the experimenter leaf returns: the Julia script it authored. The leaf's + * job is authoring ONLY — the SolveSpec (tier/source/env) is derived from the + * score by the driver, so trust-tier selection is never the model's call. */ +export interface AuthoredScript { + scriptPath: string; + /** Optional free text the leaf reports (logged for provenance; never control). */ + note?: string; +} + +export interface DispatchContext { + /** Scratch dir the leaf may author its script into. */ + workdir: string; + /** 1-based iteration index (this prototype runs exactly one). */ + iteration: number; +} + +/** The single model seam. Flat / depth-1: the dispatched leaf holds no `task`/ + * `Agent` grant, so it cannot itself spawn children. */ +export type ExperimenterDispatch = (target: IterationTarget, ctx: DispatchContext) => Promise; + +export interface IterationDeps { + /** The one model call in the whole loop. */ + dispatchExperimenter: ExperimenterDispatch; + /** Scratch/work dir for the authored script + the iteration record. */ + workdir: string; + /** Where the solve run dir is created (amico-run --runs-root). */ + runsRoot: string; + /** Absolute path to the built amico-run CLI bundle (dist/amico-run.js). */ + amicoRunBundle: string; + /** Julia binary the solve runs under. */ + juliaBin: string; + /** Lab pointer for the SolveSpec + run dir. Default "default". */ + labId?: string; + /** node binary that runs the bundle. Default "node". */ + nodeBin?: string; + /** Extra env for the CLI child (AMICO_AUTHORING_FILE, AMICO_VERIFY_RUNNER, …). */ + env?: Record; + /** Structured log sink (the demo prints these to show the deterministic steps). */ + logger?: (line: string) => void; +} + +export interface IterationOutcome { + scoreId: string; + target: IterationTarget; + /** How many experimenter leaves were dispatched — always 1 (flat, depth-1). */ + dispatched: number; + authoredScript?: string; + runDir?: string; + status?: "completed" | "failed" | "aborted"; + /** Re-rollout verdict: true/false for a verified tier, null when the tier is + * not verified (vetted) or no verification line was emitted. */ + verified: boolean | null; + /** Promotion decision, gated on `agree` for verified tiers (verify.promote_on). */ + promoted: boolean; + promoteReason: string; + /** Set when the iteration could not complete (dispatch/launch fault). */ + error?: string; +} + +/** SolveSpec object derived from the score target — NOT authored by the model. */ +function buildSolveSpec(target: IterationTarget, scriptPath: string, labId: string): Record { + const spec: Record = { + schema_version: "2", + script_path: scriptPath, + lab_id: labId, + executor: "local", + tier: target.tier, + env: target.env ?? { kind: "provisioned" }, + }; + if (target.gate) spec.gate = target.gate; + const source: Record = {}; + if (target.tier === "composed" && target.exemplar_id) source.exemplar_id = target.exemplar_id; + if (target.tier === "vetted" && target.template_id) source.template_id = target.template_id; + if (Object.keys(source).length) spec.source = source; + return spec; +} + +interface LaunchResult { + code: number; + status?: "completed" | "failed" | "aborted"; + exitCode?: number; + runDir?: string; + verified: boolean | null; + stdout: string; + stderr: string; +} + +/** Launch the authored solve through the amico-run CLI and parse its stdout + * protocol lines (AMICODE_FINISHED / AMICODE_VERIFIED). The CLI runs the launch + * gate and, for verified tiers, the independent re-rollout — the harness does + * not re-implement either; it calls the spine and reads the verdicts. */ +function launchSolve(specPath: string, scriptPath: string, deps: IterationDeps): Promise { + const args = [ + deps.amicoRunBundle, + scriptPath, + "--spec", + specPath, + "--runs-root", + deps.runsRoot, + "--julia", + deps.juliaBin, + ]; + return new Promise((resolvePromise) => { + const child = spawn(deps.nodeBin ?? "node", args, { + env: { ...process.env, ...(deps.env ?? {}) }, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (d: Buffer) => (stdout += d.toString())); + child.stderr.on("data", (d: Buffer) => (stderr += d.toString())); + child.on("error", (e) => resolvePromise({ code: 127, verified: null, stdout, stderr: stderr + String(e) })); + child.on("close", (code) => { + const fin = /AMICODE_FINISHED status=(\w+) exitCode=(-?\d+) runDir=(\S+)/.exec(stdout); + const ver = /AMICODE_VERIFIED agree=(true|false)/.exec(stdout); + resolvePromise({ + code: code ?? 1, + status: fin ? (fin[1] as LaunchResult["status"]) : undefined, + exitCode: fin ? Number(fin[2]) : undefined, + runDir: fin ? fin[3] : undefined, + verified: ver ? ver[1] === "true" : null, + stdout, + stderr, + }); + }); + }); +} + +const tq = (s: string): string => JSON.stringify(s); // JSON escaping is valid TOML basic-string + +/** Bookkeeping (spec §2: bookkeeping → tools): the deterministic outcome record. + * This is what a librarian/distiller would consume; no model writes it. */ +function writeIterationRecord(deps: IterationDeps, o: IterationOutcome): void { + const lines = [ + `schema_version = 1`, + `score_id = ${tq(o.scoreId)}`, + `platform = ${tq(o.target.platform)}`, + ...(o.target.gate ? [`gate = ${tq(o.target.gate)}`] : []), + `kind = ${tq(o.target.kind)}`, + `size = ${o.target.size}`, + `tier = ${tq(o.target.tier)}`, + `dispatched_experimenters = ${o.dispatched}`, + ...(o.authoredScript ? [`authored_script = ${tq(o.authoredScript)}`] : []), + ...(o.runDir ? [`run_dir = ${tq(o.runDir)}`] : []), + ...(o.status ? [`status = ${tq(o.status)}`] : []), + `verified = ${o.verified === null ? tq("none") : o.verified}`, + `promoted = ${o.promoted}`, + `promote_reason = ${tq(o.promoteReason)}`, + ...(o.error ? [`error = ${tq(o.error)}`] : []), + ]; + atomicWriteFile(deps.workdir, "iteration.toml", lines.join("\n") + "\n"); +} + +/** + * Run ONE experiment iteration, driven end-to-end by this code (the score + this + * driver), with a single flat experimenter leaf. Returns the outcome; also + * writes `iteration.toml` into `deps.workdir`. Never throws — a dispatch or + * launch fault becomes a recorded, un-promoted outcome (an orchestrator that + * crashes is worse than one that records a failure). + */ +export async function runExperimentIteration(score: IterationScore, deps: IterationDeps): Promise { + const log = deps.logger ?? (() => {}); + const labId = deps.labId ?? "default"; + const tier: Tier = score.target.tier; + mkdirSync(deps.workdir, { recursive: true }); + mkdirSync(deps.runsRoot, { recursive: true }); + + // ── step 1: select/receive the target (deterministic, from the score data) ── + const target = score.target; + log(`[harness] iteration for score "${score.id}": ${target.platform}/${target.gate ?? target.kind} tier=${tier}`); + + const base: IterationOutcome = { + scoreId: score.id, + target, + dispatched: 0, + verified: null, + promoted: false, + promoteReason: "", + }; + + // ── step 2: dispatch ONE experimenter leaf (flat / depth-1) — the sole model call ── + let authored: AuthoredScript; + try { + log(`[harness] dispatching experimenter leaf (flat, depth-1)…`); + authored = await deps.dispatchExperimenter(target, { workdir: deps.workdir, iteration: 1 }); + } catch (e) { + const outcome: IterationOutcome = { + ...base, + dispatched: 1, + promoteReason: "experimenter dispatch failed", + error: `dispatch: ${(e as Error).message}`, + }; + log(`[harness] experimenter dispatch FAILED: ${outcome.error}`); + writeIterationRecord(deps, outcome); + return outcome; + } + base.dispatched = 1; + base.authoredScript = authored.scriptPath; + log(`[harness] experimenter authored ${authored.scriptPath}${authored.note ? ` — ${authored.note}` : ""}`); + + // ── step 3: derive the SolveSpec from the score (NOT the model) + launch via the CLI ── + const spec = buildSolveSpec(target, authored.scriptPath, labId); + const specPath = `${deps.workdir}/solvespec.json`; + atomicWriteFile(deps.workdir, "solvespec.json", JSON.stringify(spec, null, 2) + "\n"); + log(`[harness] launching solve via amico-run (tier=${tier}, verify=${isVerifiedTier(tier)})…`); + const launch = await launchSolve(specPath, authored.scriptPath, deps); + if (!launch.runDir) { + const outcome: IterationOutcome = { + ...base, + promoteReason: "solve launch produced no run dir", + error: `launch exit ${launch.code}: ${(launch.stderr || launch.stdout).trim().split("\n").slice(-1)[0] ?? ""}`, + }; + log(`[harness] solve launch FAILED: ${outcome.error}`); + writeIterationRecord(deps, outcome); + return outcome; + } + log(`[harness] solve ${launch.status} in ${launch.runDir} (re-rollout agree=${launch.verified})`); + + // ── step 4: record outcome + gate promotion on the re-rollout verdict ── + const verifiedTier = isVerifiedTier(tier); + let promoted = false; + let promoteReason: string; + if (launch.status !== "completed") { + promoteReason = `not promoted: solve ${launch.status ?? "unknown"}`; + } else if (verifiedTier && launch.verified !== true) { + promoteReason = "not promoted: re-rollout did not agree (promote_on=agree)"; + } else if (verifiedTier) { + promoted = true; + promoteReason = "promoted: re-rollout agreed"; + } else { + // vetted tier: trusted by its template, no re-rollout gate + promoted = true; + promoteReason = "promoted: vetted tier (template-trusted, no re-rollout)"; + } + + const outcome: IterationOutcome = { + ...base, + runDir: launch.runDir, + status: launch.status, + verified: launch.verified, + promoted, + promoteReason, + }; + log(`[harness] outcome: promoted=${promoted} (${promoteReason})`); + writeIterationRecord(deps, outcome); + return outcome; +} diff --git a/packages/amico-run/src/harness/index.ts b/packages/amico-run/src/harness/index.ts new file mode 100644 index 00000000..27e273d0 --- /dev/null +++ b/packages/amico-run/src/harness/index.ts @@ -0,0 +1,5 @@ +// Experiment-iteration harness (spec-20260708-112732 §3.2, plan slice B4). +// The DATA (iteration score) + the CODE (deterministic driver) that together +// run one experiment iteration without an LLM orchestrator. +export * from "./iteration_score.js"; +export * from "./experiment_iteration.js"; diff --git a/packages/amico-run/src/harness/iteration_score.ts b/packages/amico-run/src/harness/iteration_score.ts new file mode 100644 index 00000000..0161f5e1 --- /dev/null +++ b/packages/amico-run/src/harness/iteration_score.ts @@ -0,0 +1,123 @@ +// Iteration score — the DATA half of the harness reframe (spec-20260708-112732 +// §3.2, plan slice B4). An "iteration score" declares ONE experiment iteration +// as data: which target to solve for, at which trust tier, and how promotion is +// gated. It is deliberately NOT an interview SCORE.md (no questions/choices) — +// the interview repertoire loader only reads `SCORE.md`, so a harness score +// (ITERATION.toml) never enters that machinery. The control flow that walks +// these fields lives in experiment_iteration.ts as CODE, not in an LLM prompt. +// +// G-1 ruling (amicode#107): score-first for flow; a thin TS DRIVER only for what +// the stage model can't express. The experiment-iteration loop is exactly that +// case — the flow (target + gate policy) is data here; the loop is the driver. +import { parse as parseToml } from "smol-toml"; + +export type Tier = "vetted" | "composed" | "free"; +const TIERS: readonly Tier[] = ["vetted", "composed", "free"]; + +export type EnvKind = "provisioned" | "project" | "sandbox"; + +/** The target the iteration solves for. Mirrors the fields the experimenter + * leaf needs to author + the SolveSpec the gate reads (tier/source/env). */ +export interface IterationTarget { + platform: string; + gate?: string; + kind: string; + size: number; + tier: Tier; + target_fidelity?: number; + /** tier-2: the exemplars-index entry the leaf splices; the gate REQUIRES this + * when tier=composed (see gate.ts step 4). Carried through to the SolveSpec. */ + exemplar_id?: string; + /** tier-1: the registry template id (informational for the driver). */ + template_id?: string; + /** Julia env for the SolveSpec (kind=sandbox is mandatory for free). */ + env?: { kind: EnvKind; project?: string }; +} + +export interface IterationScore { + schema_version: number; + id: string; + name?: string; + target: IterationTarget; + /** Promotion policy. "agree" = only promote (save to catalog) when the + * independent re-rollout agrees with the optimizer-reported fidelity. This is + * the trust anchor for author-first tiers (spec §4.3). */ + verify: { promote_on: "agree" }; +} + +export type ParseResult = { ok: true; score: IterationScore } | { ok: false; error: string }; + +function str(v: unknown): string | undefined { + return typeof v === "string" && v.trim() !== "" ? v : undefined; +} +function num(v: unknown): number | undefined { + return typeof v === "number" && Number.isFinite(v) ? v : undefined; +} + +/** Parse + validate an iteration score from TOML text. Never throws — a malformed + * score returns { ok:false, error } so the harness reports it, exactly like the + * interview repertoire treats a broken SCORE.md (report, don't crash). */ +export function parseIterationScore(tomlText: string): ParseResult { + let raw: Record; + try { + raw = parseToml(tomlText) as Record; + } catch (e) { + return { ok: false, error: `iteration score: unparseable TOML (${(e as Error).message})` }; + } + + const id = str(raw.id); + if (!id) return { ok: false, error: "iteration score: missing `id`" }; + + const t = raw.target; + if (typeof t !== "object" || t === null) return { ok: false, error: "iteration score: missing `[target]` table" }; + const tt = t as Record; + + const platform = str(tt.platform); + const kind = str(tt.kind); + const size = num(tt.size); + const tier = str(tt.tier) as Tier | undefined; + if (!platform) return { ok: false, error: "iteration score: `target.platform` required" }; + if (!kind) return { ok: false, error: "iteration score: `target.kind` required" }; + if (size === undefined || size <= 0) return { ok: false, error: "iteration score: `target.size` must be a positive number" }; + if (!tier || !TIERS.includes(tier)) + return { ok: false, error: `iteration score: \`target.tier\` must be one of ${TIERS.join("|")}` }; + + const exemplar_id = str(tt.exemplar_id); + if (tier === "composed" && !exemplar_id) + return { ok: false, error: 'iteration score: tier "composed" requires `target.exemplar_id` (the gate needs it)' }; + + let env: IterationTarget["env"]; + if (typeof tt.env === "object" && tt.env !== null) { + const e = tt.env as Record; + const kindE = str(e.kind) as EnvKind | undefined; + if (kindE) env = { kind: kindE, project: str(e.project) }; + } + if (tier === "free" && env?.kind !== "sandbox") + return { ok: false, error: 'iteration score: tier "free" requires `target.env.kind = "sandbox"` (gate step 3)' }; + + const verifyRaw = (typeof raw.verify === "object" && raw.verify !== null ? raw.verify : {}) as Record; + const promote_on = str(verifyRaw.promote_on) ?? "agree"; + if (promote_on !== "agree") + return { ok: false, error: `iteration score: \`verify.promote_on\` must be "agree" (got "${promote_on}")` }; + + return { + ok: true, + score: { + schema_version: num(raw.schema_version) ?? 1, + id, + name: str(raw.name), + target: { + platform, + gate: str(tt.gate), + kind, + size, + tier, + target_fidelity: num(tt.target_fidelity), + exemplar_id, + template_id: str(tt.template_id), + env, + }, + verify: { promote_on: "agree" }, + }, + }; +} diff --git a/packages/amico-run/src/index.ts b/packages/amico-run/src/index.ts index dcd0ab44..35522b1e 100644 --- a/packages/amico-run/src/index.ts +++ b/packages/amico-run/src/index.ts @@ -5,3 +5,5 @@ export * from "./schemas.js"; export * from "./event_queue.js"; export * from "./local_executor.js"; export * from "./scheduler.js"; +export { isVerifiedTier } from "./verify.js"; +export * from "./harness/index.js"; diff --git a/packages/amico-run/src/verify.ts b/packages/amico-run/src/verify.ts index e168bd83..d8a6f29d 100644 --- a/packages/amico-run/src/verify.ts +++ b/packages/amico-run/src/verify.ts @@ -1,17 +1,30 @@ -// Free-tier re-rollout verification invoke (spec C). After FINISHED, when the -// SolveSpec is tier "free", amico-run runs the FIXED, VETTED re-rollout harness +// Re-rollout verification invoke (spec C; extended to tier-2 by +// spec-20260708-112732 §4.3). After FINISHED, when the SolveSpec is a verified +// tier (see isVerifiedTier), amico-run runs the FIXED, VETTED re-rollout harness // (a Julia asset shipped with the extension, path from authoring.json) against // the run dir's system_verify.jld2 + pulse.jld2. The harness writes // verification.toml itself; if it is missing, fails to run, or exits without // writing, we write a fallback verification.toml with agree=false + a reason — -// a free run must NEVER end verification-less (absence would read as "pending" -// forever and mask a failure, and the auto-promote gate keys off agree==true). +// a verified run must NEVER end verification-less (absence would read as +// "pending" forever and mask a failure, and the auto-promote gate keys off +// agree==true). import { spawn } from "node:child_process"; import { existsSync, renameSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import type { AuthoringConfig } from "./authoring.js"; import type { SpecStamp } from "./types.js"; +/** Which trust tiers get the independent re-rollout gate after FINISHED. + * free = author-first (tier-3, the original trust anchor); composed = exemplar- + * spliced (tier-2 — added by spec-20260708-112732 §4.3, because a wrong fill can + * still re-roll to a different pulse and pass the masked-baseline check). vetted + * (tier-1) is trusted by its template and skips it. Single source of truth for + * the policy — cli.ts gates on it, and the harness driver reads it to know + * whether a run will emit an AMICODE_VERIFIED line. */ +export function isVerifiedTier(tier: string | undefined): boolean { + return tier === "free" || tier === "composed"; +} + function tomlEscape(s: string): string { return JSON.stringify(s); } diff --git a/packages/amico-run/test/cli.test.ts b/packages/amico-run/test/cli.test.ts index b814297f..c85032f8 100644 --- a/packages/amico-run/test/cli.test.ts +++ b/packages/amico-run/test/cli.test.ts @@ -3,6 +3,7 @@ import { execFileSync, execFile } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { tmpRoot, fakeJulia, readToml } from "./helpers.js"; +import { maskedHash } from "../src/baseline.js"; const BUNDLE = join(__dirname, "..", "dist", "amico-run.js"); beforeAll(() => { @@ -188,6 +189,67 @@ describe("amico-run CLI", () => { const vetDir = /runDir=(\S+)/.exec(rVet.stdout)![1]; expect(existsSync(join(vetDir, "verification.toml"))).toBe(false); }); + it("--spec tier=composed: verification runs too (spec-20260708-112732 §4.3 tier-2 extension)", () => { + const root = tmpRoot(); + // exemplar with the template's default fill-point markers + const exemplar = `using Piccolo\nusing JLD2, TOML\n# ── FILL IN ──────\nT = 10.0\n# ─────────────────\nsolve()\n`; + const index = join(root, "exemplars.json"); + writeFileSync( + index, + JSON.stringify({ + schema_version: 1, + exemplars: [ + { + id: "ex-cz", + platform: "rydberg", + kind: "gate_synthesis", + size: 2, + path: "ex-cz/script.jl", + packages: ["Piccolo", "JLD2", "TOML"], + baseline_hash: maskedHash(exemplar), + }, + ], + }), + ); + const harness = fakeJulia( + root, + "h.js", + `const fs=require('fs'),p=require('path');fs.writeFileSync(p.join(process.argv[process.argv.length-2],'verification.toml'),'schema_version = "1"\\nagree = true\\n')`, + ); + writeFileSync( + join(root, "authoring.json"), + JSON.stringify({ + schema_version: 1, + allowlist: ["Piccolo"], + support_set: ["JLD2", "TOML"], + exemplars: index, + verify_harness: harness, + verify_tolerance: 0.01, + }), + ); + // authored script: an inside-fill-point edit → passes the masked-baseline gate + const script = fakeJulia(root, "s.jl", ""); + writeFileSync(script, exemplar.replace("T = 10.0", "T = 25.0")); + const julia = fakeJulia(root, "j", `console.log('DONE f=0.99')`); + const spec = { + schema_version: "2", + script_path: script, + lab_id: "default", + executor: "local", + tier: "composed", + env: { kind: "provisioned" }, + source: { exemplar_id: "ex-cz" }, + }; + writeFileSync(join(root, "composed.json"), JSON.stringify(spec)); + const r = run([script, "--runs-root", join(root, "runs"), "--spec", join(root, "composed.json"), "--julia", julia], { + AMICO_AUTHORING_FILE: join(root, "authoring.json"), + AMICO_VERIFY_RUNNER: harness, + }); + expect(r.code).toBe(0); + expect(r.stdout).toMatch(/AMICODE_VERIFIED agree=true/); + const dir = /runDir=(\S+)/.exec(r.stdout)![1]; + expect(existsSync(join(dir, "verification.toml"))).toBe(true); + }); it("SIGTERM to the CLI → abort lane, exit 130", async () => { const root = tmpRoot(); const julia = fakeJulia(root, "j", `console.log('READY'); setInterval(() => {}, 1000)`); diff --git a/packages/amico-run/test/experiment_iteration.test.ts b/packages/amico-run/test/experiment_iteration.test.ts new file mode 100644 index 00000000..a415a152 --- /dev/null +++ b/packages/amico-run/test/experiment_iteration.test.ts @@ -0,0 +1,142 @@ +// B4 harness-reframe prototype acceptance (amicode#109, spec-20260708-112732 +// §3.2/§4.3). Proves ONE experiment iteration runs end-to-end driven by an +// iteration score + the deterministic driver — NOT an LLM orchestrator. The +// experimenter leaf and the re-rollout harness are deterministic fakes, so the +// control-flow path has NO model in it and NO Julia. +import { describe, it, expect, beforeAll } from "vitest"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, existsSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parse as parseToml } from "smol-toml"; +import { parseIterationScore, runExperimentIteration, type IterationScore } from "../src/harness/index.js"; +import { setupFakeIterationEnv } from "../harness-demo/fixtures.js"; + +const PKG = join(__dirname, ".."); +const BUNDLE = join(PKG, "dist", "amico-run.js"); + +// The driver shells out to the real amico-run CLI, so build the bundle first — +// this exercises the REAL launch gate + tier-2 re-rollout invocation, not a mock. +beforeAll(() => { + execFileSync("node", [join(PKG, "esbuild.config.mjs")], { cwd: PKG }); +}); + +function composedScore(exemplarId: string): IterationScore { + const r = parseIterationScore( + [ + `schema_version = 1`, + `id = "test-iteration"`, + `[target]`, + `platform = "rydberg"`, + `gate = "CZ"`, + `kind = "gate_synthesis"`, + `size = 2`, + `tier = "composed"`, + `exemplar_id = "${exemplarId}"`, + `env = { kind = "provisioned" }`, + `[verify]`, + `promote_on = "agree"`, + ].join("\n"), + ); + if (!r.ok) throw new Error(r.error); + return r.score; +} + +describe("experiment-iteration harness (B4 prototype)", () => { + it("runs ONE iteration: flat leaf → solve → tier-2 re-rollout → promote, no LLM in the loop", async () => { + const work = mkdtempSync(join(tmpdir(), "b4-iter-")); + const fake = setupFakeIterationEnv(join(work, "fx"), { agree: true }); + let dispatchCalls = 0; + const outcome = await runExperimentIteration(composedScore(fake.exemplarId), { + dispatchExperimenter: async (t, ctx) => { + dispatchCalls++; + return fake.dispatchExperimenter(t, ctx); + }, + workdir: join(work, "iter"), + runsRoot: join(work, "runs"), + amicoRunBundle: BUNDLE, + juliaBin: fake.juliaBin, + env: fake.env, + }); + + // exactly one experimenter leaf, dispatched flat (depth-1) + expect(dispatchCalls).toBe(1); + expect(outcome.dispatched).toBe(1); + // the solve completed + expect(outcome.status).toBe("completed"); + // the re-rollout gate ran for TIER-2 (composed) and agreed — the §4.3 extension + expect(outcome.verified).toBe(true); + // promotion is gated on that agreement + expect(outcome.promoted).toBe(true); + // the run dir carries the verification verdict + expect(outcome.runDir && existsSync(join(outcome.runDir, "verification.toml"))).toBeTruthy(); + // the deterministic outcome record was written (bookkeeping, in code) + const rec = parseToml(readFileSync(join(work, "iter", "iteration.toml"), "utf8")) as Record; + expect(rec.promoted).toBe(true); + expect(rec.tier).toBe("composed"); + expect(rec.dispatched_experimenters).toBe(1); + expect(rec.verified).toBe(true); + }); + + it("promotion is gated on agree: re-rollout DISAGREES → not promoted", async () => { + const work = mkdtempSync(join(tmpdir(), "b4-iter-")); + const fake = setupFakeIterationEnv(join(work, "fx"), { agree: false }); + const outcome = await runExperimentIteration(composedScore(fake.exemplarId), { + dispatchExperimenter: fake.dispatchExperimenter, + workdir: join(work, "iter"), + runsRoot: join(work, "runs"), + amicoRunBundle: BUNDLE, + juliaBin: fake.juliaBin, + env: fake.env, + }); + expect(outcome.status).toBe("completed"); // the solve itself is fine… + expect(outcome.verified).toBe(false); // …but the independent re-rollout disagreed… + expect(outcome.promoted).toBe(false); // …so promotion is withheld. + expect(outcome.promoteReason).toMatch(/did not agree/); + }); + + it("a dispatch fault is recorded, not thrown (the loop never crashes)", async () => { + const work = mkdtempSync(join(tmpdir(), "b4-iter-")); + const fake = setupFakeIterationEnv(join(work, "fx")); + const outcome = await runExperimentIteration(composedScore(fake.exemplarId), { + dispatchExperimenter: async () => { + throw new Error("leaf boom"); + }, + workdir: join(work, "iter"), + runsRoot: join(work, "runs"), + amicoRunBundle: BUNDLE, + juliaBin: fake.juliaBin, + env: fake.env, + }); + expect(outcome.promoted).toBe(false); + expect(outcome.error).toMatch(/boom/); + expect(existsSync(join(work, "iter", "iteration.toml"))).toBe(true); + }); +}); + +describe("parseIterationScore", () => { + it("parses the shipped ITERATION.toml (the data-defined flow)", () => { + const shipped = join(PKG, "..", "extension", "scores", "experiment-iteration", "ITERATION.toml"); + const r = parseIterationScore(readFileSync(shipped, "utf8")); + expect(r.ok).toBe(true); + if (r.ok) { + expect(r.score.target.tier).toBe("composed"); + expect(r.score.target.exemplar_id).toBe("rydberg-cz"); + expect(r.score.verify.promote_on).toBe("agree"); + } + }); + it("rejects composed without exemplar_id (the gate needs it)", () => { + const r = parseIterationScore(`id="x"\n[target]\nplatform="p"\nkind="k"\nsize=1\ntier="composed"\n`); + expect(r.ok).toBe(false); + if (!r.ok) expect(r.error).toMatch(/exemplar_id/); + }); + it("rejects free without a sandbox env (gate step 3)", () => { + const r = parseIterationScore(`id="x"\n[target]\nplatform="p"\nkind="k"\nsize=1\ntier="free"\n`); + expect(r.ok).toBe(false); + if (!r.ok) expect(r.error).toMatch(/sandbox/); + }); + it("rejects an unknown tier", () => { + const r = parseIterationScore(`id="x"\n[target]\nplatform="p"\nkind="k"\nsize=1\ntier="premium"\n`); + expect(r.ok).toBe(false); + }); +}); diff --git a/packages/amico-run/test/s31.test.ts b/packages/amico-run/test/s31.test.ts index 89f6a08b..ca9a4598 100644 --- a/packages/amico-run/test/s31.test.ts +++ b/packages/amico-run/test/s31.test.ts @@ -9,11 +9,22 @@ import { join } from "node:path"; // NOT a physics knob; all physics stays in the script.) const FORBIDDEN = [/--gate\b/, /--system\b/, /--pulse\b/, /modelcontextprotocol/i, /node:https?\b/, /\bfetch\s*\(/]; +/** Every .ts under src/, recursively — the guard must cover subdirs (e.g. + * src/harness/) too, so the tool layer stays CLI/spawn: no HTTP, no MCP. */ +function srcFiles(dir: string): string[] { + const out: string[] = []; + for (const e of readdirSync(dir, { withFileTypes: true })) { + const p = join(dir, e.name); + if (e.isDirectory()) out.push(...srcFiles(p)); + else if (e.name.endsWith(".ts")) out.push(p); + } + return out; +} + describe("S31 grep rule", () => { it("src/ contains no forbidden tool-layer patterns", () => { - const srcDir = join(__dirname, "..", "src"); - for (const f of readdirSync(srcDir)) { - const text = readFileSync(join(srcDir, f), "utf8"); + for (const f of srcFiles(join(__dirname, "..", "src"))) { + const text = readFileSync(f, "utf8"); for (const re of FORBIDDEN) { expect(text, `${f} matches forbidden ${re}`).not.toMatch(re); } diff --git a/packages/amico-run/test/verify.test.ts b/packages/amico-run/test/verify.test.ts index 2b25c1e0..2bbcde4f 100644 --- a/packages/amico-run/test/verify.test.ts +++ b/packages/amico-run/test/verify.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest"; import { mkdtempSync, mkdirSync, existsSync, readFileSync, writeFileSync, chmodSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { runVerification } from "../src/verify.js"; +import { isVerifiedTier, runVerification } from "../src/verify.js"; import { readToml } from "./helpers.js"; import type { AuthoringConfig } from "../src/authoring.js"; import type { SpecStamp } from "../src/types.js"; @@ -74,3 +74,12 @@ describe("runVerification", () => { expect(readToml(join(runDir, "verification.toml")).agree).toBe(false); }); }); + +describe("isVerifiedTier (spec-20260708-112732 §4.3)", () => { + it("verifies free (tier-3) AND composed (tier-2); vetted (tier-1) is trusted", () => { + expect(isVerifiedTier("free")).toBe(true); + expect(isVerifiedTier("composed")).toBe(true); // the §4.3 extension + expect(isVerifiedTier("vetted")).toBe(false); + expect(isVerifiedTier(undefined)).toBe(false); + }); +}); diff --git a/packages/amico-run/tsconfig.json b/packages/amico-run/tsconfig.json index b9ee75a8..73fb300f 100644 --- a/packages/amico-run/tsconfig.json +++ b/packages/amico-run/tsconfig.json @@ -8,5 +8,5 @@ "resolveJsonModule": true, "types": ["node"] }, - "include": ["src", "test"] + "include": ["src", "test", "harness-demo"] } diff --git a/packages/extension/scores/experiment-iteration/ITERATION.toml b/packages/extension/scores/experiment-iteration/ITERATION.toml new file mode 100644 index 00000000..b4dc0958 --- /dev/null +++ b/packages/extension/scores/experiment-iteration/ITERATION.toml @@ -0,0 +1,31 @@ +# Experiment-iteration score — the DATA half of the harness reframe. +# +# spec-20260708-112732 §3.2/§4.3 · plan slice B4 · amicode#109 (prototype). +# +# This is a HARNESS score, not an interview SCORE.md: it declares one autonomous +# experiment iteration as data. The interview repertoire loader only reads +# `SCORE.md`, so this file never enters that machinery. The deterministic driver +# that WALKS these fields is code — `@amicode/amico-run`'s harness module +# (src/harness/experiment_iteration.ts) — NOT an LLM orchestrator. +# +# G-1 (amicode#107): score-first for flow + a thin TS driver only for what the +# stage model can't express. The iteration loop is that exception. + +schema_version = 1 +id = "experiment-iteration" +name = "Single experiment iteration (tier-2, re-rollout gated)" + +# ── the target the iteration solves for (the harness receives it; no LLM picks it) ── +[target] +platform = "rydberg" +gate = "CZ" +kind = "gate_synthesis" +size = 2 +tier = "composed" # tier-2 → the re-rollout gate MUST run (§4.3) +exemplar_id = "rydberg-cz" # the shipped exemplar the experimenter leaf splices +target_fidelity = 0.9999 +env = { kind = "provisioned" } # tier-2 runs in the provisioned project + +# ── promotion policy (enforced in code, not prose) ── +[verify] +promote_on = "agree" # save to catalog ONLY when the re-rollout agrees diff --git a/packages/extension/scores/experiment-iteration/README.md b/packages/extension/scores/experiment-iteration/README.md new file mode 100644 index 00000000..3af67af3 --- /dev/null +++ b/packages/extension/scores/experiment-iteration/README.md @@ -0,0 +1,56 @@ +# Experiment-iteration score (harness reframe prototype) + +**Status:** prototype for review — `amicode#109` (slice B4), spec `spec-20260708-112732` +§3.2/§4.3. Stands on `amicode#107`'s G-1 ruling (score-first + a thin TS driver +where the stage model can't express iteration). + +## What this is + +This directory holds a **harness score** — the *data* half of dissolving the +1117-line LLM `orchestrator` for a single experiment iteration. Unlike an +interview `SCORE.md` (questions the user rides), an `ITERATION.toml` declares one +autonomous experiment iteration: which target to solve for, at which trust tier, +and how promotion is gated. The interview repertoire loader only reads +`SCORE.md`, so this file is invisible to that machinery by design. + +The **control flow** that walks this data is *code*, not an agent prompt: +[`packages/amico-run/src/harness/experiment_iteration.ts`](../../../amico-run/src/harness/experiment_iteration.ts). + +## The iteration, as deterministic code + +``` +select target (from ITERATION.toml) ← data, no LLM + │ +dispatch ONE experimenter leaf (flat/depth-1) ← the ONLY model call; it authors the Julia script + │ +derive SolveSpec from the score + launch ← amico-run --spec runs the launch gate… + │ …and, for tier-2/3, the independent re-rollout +record outcome + gate promotion on `agree` ← bookkeeping, in code +``` + +Nothing in that loop is an LLM's judgment except the single flat leaf that +authors the script. The trust-tier, the source exemplar, the env, and the +promote decision are all derived from this score or computed in code. + +## Why the driver lives in `amico-run`, not here + +Per spec §7.3 the deterministic harness "calls the CLI directly — no LLM," and +`amico-run` is that CLI lineage. The driver shells out to `amico-run --spec`, so +the launch gate and the re-rollout verification are reused verbatim, not +re-implemented. In production the experimenter seam is wired to +`opencode run --agent experimenter` (headless); the driver is UI-agnostic. + +## Run the prototype + +The prototype runs with **no LLM and no Julia** — the experimenter leaf and the +re-rollout harness are deterministic fakes, so the *control flow* is what's +exercised: + +```bash +pnpm --filter @amicode/amico-run build +pnpm --filter @amicode/amico-run demo:harness # prints each deterministic step + the outcome +pnpm --filter @amicode/amico-run test # includes the no-LLM iteration test +``` + +See `packages/amico-run/harness-demo/` for the runnable demo and +`packages/amico-run/test/experiment_iteration.test.ts` for the test.