From f82a076cf49e5d3ed0bab5b6a738f425c031548f Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 23 Aug 2026 06:53:33 -0700 Subject: [PATCH] feat(episode): stop an episode when the game reports it is over A consumer measured 163 of 300 decisions (54.3%) on ale-breakout played after lives reached 0, with the engine's own terminal flag set. The ALE worker breaks out of its action-repeat loop once that flag holds, so those inputs never reached the emulator. The episode still reported 300 of 300 answered. Game gains an optional over(state). A game that omits it is never over, so every existing adapter keeps its behaviour with no edit. playEpisode, runCampaign, and executeBenchmark take stopAtGameOver, off by default, because episode length is the denominator a study divides by. The stop is an exit from the decision loop, not an abort. finalizeRecord runs, and the record verifies by replay exactly as a turn-limited record does. EpisodeRecord gains stoppedBy and gameOver, so a reader of an artifact never infers why a run ended. --- CHANGELOG.md | 26 ++++ README.md | 60 ++++++++++ adapters/ale.ts | 5 + adapters/gymnasium.ts | 5 + adapters/native-2048.ts | 3 + adapters/screen-puzzle.ts | 3 + adapters/stable-retro.ts | 3 + ale.test.mts | 54 +++++++++ campaign.ts | 43 ++++++- docs/adapters.md | 24 ++++ episode-loop.ts | 31 ++++- episode.ts | 52 +++++++- execute.ts | 3 + game-over.test.mts | 243 ++++++++++++++++++++++++++++++++++++++ gymnasium.test.mts | 29 ++++- package.json | 2 +- runtime.ts | 37 ++++++ 17 files changed, 608 insertions(+), 15 deletions(-) create mode 100644 game-over.test.mts diff --git a/CHANGELOG.md b/CHANGELOG.md index 887307a..23e80aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,32 @@ All notable changes to Playproof are documented here. +## Unreleased + +### An episode can end because the game ended + +- **The measurement.** A consumer running `ale-breakout` at `maxTurns: 300` counted **163 of 300 decisions (54.3%) taken after lives reached 0**, with the engine's own `terminal` flag set. The ALE worker breaks out of its action-repeat loop once that flag holds, so none of those inputs reached the emulator: the decisions were inert, not merely unproductive. The episode still reported 300 of 300 answered and looked healthy. +- **Why the consumer could not fix it.** `playEpisode` had two stop conditions, the turn limit and the dollar budget, and no terminal concept in the published API. The only other exit was an abort through `signal`, which throws inside the decision loop before `finalizeRecord` and destroys the attestation the grade is made of. +- `Game` gains an OPTIONAL `over(state): boolean`. A game that omits it is never over, so every existing adapter keeps its behaviour with no edit. It must be pure like `step`, because a verifier recomputes the final state from the seed and the input log and asks again. There is no shared spelling of the terminal flag across substrates — ALE writes `terminal`, Gymnasium `terminated` and `truncated`, stable-retro `episodeDone`, the 2048 core `gameOver` — so the mapping belongs to the adapter, not to a guess the harness makes over field names. `adapters/ale`, `adapters/gymnasium`, `adapters/stable-retro`, `adapters/native-2048`, and the `screen-puzzle` fixture implement it over evidence they already published. +- `playEpisode`, `runCampaign`, and `executeBenchmark` take `stopAtGameOver`. **It is off by default**: episode length is the denominator a study divides by, and rounds compare only while every round played to the same turn limit. A default that shortened episodes would retroactively break a running comparison, so arming the stop is the caller's decision, taken once for a series. +- The stop is an exit from the decision loop, never a thrown abort. `finalizeRecord` runs, and the record verifies by replay exactly as a turn-limited record does. +- `EpisodeRecord` gains `stoppedBy` and `gameOver`, so a reader of an artifact never infers why a run ended. `stoppedBy` is `maxTurns`, `budget`, `gameOver`, or — for a campaign — `steering` or `analyst`. `gameOver` is `true`, `false`, or `null` when the game declares no terminal state at all. Game over outranks the limits: a run that reaches its last allowed turn and a finished game at the same instant reports `gameOver`. +- The two fields together state which mode produced a record. `stoppedBy: 'maxTurns'` next to `gameOver: true` can only come from a run played past the end, so the stop was not armed. Where the game never ended, the two modes produce the same length and the same record. +- `CampaignStop` gains `gameOver`, and a campaign segment records it. A campaign resumed from a ledger whose game already ended plays no further segment and writes no empty one. A ledger written by 0.6.0 still loads; the ledger schema is unchanged. + +### Measured + +ALE Breakout, ale-py 0.12.1, seed 0, `maxTurns: 300`, one scripted policy that opens four milestones and then loses every life. Both runs are gates in `pnpm test:ale`. + +| Run | Decisions | `stoppedBy` | `gameOver` | Milestones | Replay | +|---|---|---|---|---|---| +| turn limit (today's behaviour) | 300 | `maxTurns` | `true` | 4 of 6 | clean | +| `stopAtGameOver: true` | 150 | `gameOver` | `true` | 4 of 6 | clean | + +The 150 dropped decisions are inert: every evidence channel — screen hash, save-state hash, and engine state — is byte-identical from decision 150 to decision 300, while decision 149 to 150 did move the emulator. + +Gymnasium FrozenLake-v1, same shape on a second real substrate: 6 decisions instead of 26, 3 of 3 milestones either way, replay clean. It is a gate in `pnpm test:gym`. + ## 0.6.0 ### A hash check identifies a state, not a trajectory diff --git a/README.md b/README.md index aff5312..cb0329e 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,66 @@ const driver = createCliAgentDriver({ The CLI driver spawns without a shell and bounds time and output. It can invoke Claude Code, Codex CLI, OpenCode, Pi, a local executable, or a container entrypoint. See [Agent drivers](docs/agent-drivers.md) and `examples/` for complete integrations, including a caller-supplied Tangle Agent Runtime backend. +## When an episode ends + +An episode has three stop conditions, and the record names the one that fired. + +| `record.stoppedBy` | What happened | +|---|---| +| `maxTurns` | The turn limit was reached. | +| `budget` | The dollar budget was reached. | +| `gameOver` | The game reported that it is finished. | + +The game-over stop is opt-in. + +```ts +const { record } = await playEpisode(game, contract, driver, budgetUsd, maxTurns, seed, signal, { + stopAtGameOver: true, +}) +// record.stoppedBy === 'gameOver' +// record.gameOver === true +``` + +`runCampaign` and `executeBenchmark` take the same `stopAtGameOver` option. + +It is off by default because episode length is a denominator. +Rounds of one study compare only while every round played to the same turn limit, so shortening an episode is the caller's decision, taken once for a whole series. + +Every record also carries `gameOver`, armed or not. +It is `true` when the game was finished at the last state of the run, `false` when it was not, and `null` when the game declares no terminal state at all. +A record that reports `stoppedBy: 'maxTurns'` next to `gameOver: true` is a run that kept paying for decisions after the game ended. + +A game declares the end of play with the optional `over(state)` member. + +```ts +const game: Game = { + // ... + evidence: (s) => s.evidence, + over: (s) => s.evidence.engineState?.terminal === 1, +} +``` + +`over` must be pure, like `step`, because a verifier recomputes the final state from the seed and the input log and asks again. +A game that omits it is never over, so every adapter written before this member keeps its behaviour. +`adapters/ale`, `adapters/gymnasium`, `adapters/stable-retro`, and `adapters/native-2048` implement it over the terminal flag their worker already publishes. + +The stop is an exit from the decision loop, not an abort. +The attestation runs, and the record verifies by replay exactly as a turn-limited one does. +Aborting through `signal` is a different thing: it throws inside the loop, before the record is built, and leaves nothing to grade. + +### Measured + +ALE Breakout, ale-py 0.12.1, seed 0, 300 turns, one scripted policy that opens four milestones and then loses every life: + +| Run | Decisions | `stoppedBy` | `gameOver` | Milestones | +|---|---|---|---|---| +| turn limit | 300 | `maxTurns` | `true` | 4 of 6 | +| game-over stop | 150 | `gameOver` | `true` | 4 of 6 | + +The 150 dropped decisions are inert, not merely unproductive. +The ALE worker breaks out of its action-repeat loop once the game is over, so every evidence channel is byte-identical from decision 150 to decision 300. +Gymnasium FrozenLake behaves the same way: 6 decisions instead of 26, with 3 of 3 milestones either way. + ## The observation channel: text always, pixels optional A game declares what the agent perceives. Text is always there. Images are opt-in. diff --git a/adapters/ale.ts b/adapters/ale.ts index 98a7ac1..f1453e5 100644 --- a/adapters/ale.ts +++ b/adapters/ale.ts @@ -298,6 +298,11 @@ export function makeAle(options: AleOptions): Ale { ? { text: s.frameText } : { text: s.frameText, images: [s.frameImage] }), evidence: (s) => s.evidence, + // The worker sets `terminal` when `ale.game_over()` first holds, and from + // that instant its step loop breaks out before it acts, so no further + // input reaches the emulator. Reading the flag the worker already + // publishes keeps one definition of the end of the game. + over: (s) => s.evidence.engineState?.terminal === 1, } try { diff --git a/adapters/gymnasium.ts b/adapters/gymnasium.ts index 3288493..fe6304a 100644 --- a/adapters/gymnasium.ts +++ b/adapters/gymnasium.ts @@ -240,6 +240,11 @@ export function makeGymnasium(options: GymnasiumOptions): Gymnasium { }, frame: (s) => s.frameText, evidence: (s) => s.evidence, + // Gymnasium ends an episode two ways and the worker publishes both: + // `terminated` is the environment's own goal or failure, `truncated` is + // its time limit. The worker freezes the environment on either, so both + // mean no further input changes anything. + over: (s) => s.evidence.engineState?.terminated === 1 || s.evidence.engineState?.truncated === 1, } try { diff --git a/adapters/native-2048.ts b/adapters/native-2048.ts index a6df2c7..32cc51f 100644 --- a/adapters/native-2048.ts +++ b/adapters/native-2048.ts @@ -184,6 +184,9 @@ export function makeNative2048(seed = 0): Native2048Adapter { }, frame: (state) => state.frameText, evidence: (state) => state.evidence, + // The core publishes `gameOver` when no move can change the board, which + // is exactly the end of a 2048 game. + over: (state) => state.evidence.engineState?.gameOver === 1, } const reference = [...NATIVE_2048_REFERENCE] const contract = deriveContract(game, seed, reference, marks()) diff --git a/adapters/screen-puzzle.ts b/adapters/screen-puzzle.ts index 12492a4..62d7c46 100644 --- a/adapters/screen-puzzle.ts +++ b/adapters/screen-puzzle.ts @@ -34,6 +34,9 @@ export const screenPuzzle: Game = { engineState: { x: s.x }, frameHash: hashString(render(s)), }), + // The far gate is the end of the puzzle: at the last column no input moves + // the piece any more, so the state machine is finished. + over: (s) => s.x >= PUZZLE_WIDTH - 1, } export const screenPuzzleContract = (): MilestoneContract => diff --git a/adapters/stable-retro.ts b/adapters/stable-retro.ts index 0ec2808..9883109 100644 --- a/adapters/stable-retro.ts +++ b/adapters/stable-retro.ts @@ -287,6 +287,9 @@ export function makeStableRetro(options: StableRetroOptions): StableRetro { ? { text: s.frameText } : { text: s.frameText, images: [s.frameImage] }), evidence: (s) => s.evidence, + // The worker publishes `episodeDone` when the environment reported + // terminated or truncated, and stops acting from that point. + over: (s) => s.evidence.engineState?.episodeDone === 1, } try { diff --git a/ale.test.mts b/ale.test.mts index 61c8147..6576c26 100644 --- a/ale.test.mts +++ b/ale.test.mts @@ -18,6 +18,7 @@ import { spawnSync } from 'node:child_process' import { createHash } from 'node:crypto' import { attestRun } from './attestation' import { assertContractSeparates, assertOpaqueChecksDeclared, calibrateContract } from './calibration' +import { playEpisode, scriptedDriver } from './episode' import { logFrom, observationOf } from './runtime' import { decodePng, unscale } from './test-png.mts' import { contractLegibility, formatMilestoneScore, scoreMilestones, validateContract } from './schema' @@ -313,6 +314,59 @@ if (!pythonHasAle()) { vision.dispose() } + // The game-over stop, on the real emulator. + // + // Motivation, measured by a consumer at 300 turns on this ROM: 163 of 300 + // decisions (54.3%) were taken after lives reached 0 with `terminal` set. + // The worker breaks out of its action-repeat loop once that flag holds, so + // none of those inputs reached the emulator. + // + // The script reproduces the shape deterministically — 40 inputs of the + // reference, which open four milestones, then FIRE until the last life is + // gone. A regression in these turn counts means ale-py, the ROM, or the + // reference moved. + const GAME_OVER_TURNS = 300 + const REFERENCE_PREFIX = 40 + const dying = () => scriptedDriver([ + ...adapter.reference.slice(0, REFERENCE_PREFIX), + ...Array.from({ length: GAME_OVER_TURNS - REFERENCE_PREFIX }, () => 'FIRE'), + ]) + const fullLength = await playEpisode(adapter.game, adapter.contract, dying(), 1, GAME_OVER_TURNS, adapter.seed) + const atGameOver = await playEpisode( + adapter.game, adapter.contract, dying(), 1, GAME_OVER_TURNS, adapter.seed, undefined, { stopAtGameOver: true }, + ) + assert.equal(fullLength.record.turns, GAME_OVER_TURNS) + assert.equal(fullLength.record.stoppedBy, 'maxTurns') + assert.equal(fullLength.record.gameOver, true, 'the full-length run ended past a finished game') + assert.equal(atGameOver.record.turns, 150) + assert.equal(atGameOver.record.stoppedBy, 'gameOver') + assert.equal(atGameOver.record.gameOver, true) + // The milestone verdict is unchanged: the 150 dropped decisions bought + // nothing, on either the score or the two pinned hashes. + assert.deepEqual(atGameOver.record.verified, fullLength.record.verified) + assert.deepEqual(atGameOver.record.score, fullLength.record.score) + assert.equal(atGameOver.record.verified.length > 0, true) + // The short record verifies by replay exactly as the full-length one does. + for (const run of [atGameOver, fullLength]) { + assert.equal(run.record.verdict, 'clean') + assert.equal(run.record.replayDivergence, false) + const recomputed = attestRun(adapter.game, adapter.contract, adapter.seed, run.log, [...run.record.verified]) + assert.equal(recomputed.verdict, 'clean', recomputed.reasons.join('; ')) + assert.deepEqual(recomputed.verified, run.record.verified) + } + // The dropped decisions were inert, not merely unproductive: every + // evidence channel is byte-identical from the game-over snapshot to the + // 300th, while the decision before it did move the emulator. + const dyingTrace = trace(adapter, [...fullLength.log.inputs()]) + assert.equal(dyingTrace[atGameOver.record.turns], dyingTrace[GAME_OVER_TURNS]) + assert.notEqual(dyingTrace[atGameOver.record.turns - 1], dyingTrace[atGameOver.record.turns]) + console.log( + `ale: game-over stop — ${atGameOver.record.turns} of ${GAME_OVER_TURNS} decisions played, ` + + `${fullLength.record.turns - atGameOver.record.turns} dropped as inert ` + + `(${Math.round((1 - atGameOver.record.turns / GAME_OVER_TURNS) * 1000) / 10}% of the episode); ` + + `milestones ${formatMilestoneScore(atGameOver.record.score)}, unchanged from the full-length run`, + ) + // Teardown: dispose kills the worker and every later call fails loudly // instead of silently reading a dead transport. second.dispose() diff --git a/campaign.ts b/campaign.ts index 7cb4f1e..d2a5e5f 100644 --- a/campaign.ts +++ b/campaign.ts @@ -26,9 +26,10 @@ import type { AgentDriver, AgentHistoryEntry, EpisodeRecord, + EpisodeStop, MilestoneCostRow, } from './episode' -import { observationTextOf, type Game, type InputLog } from './runtime' +import { isGameOver, observationTextOf, type Game, type InputLog } from './runtime' import { contractHash, scoreMilestones, type MilestoneContract, type MilestoneScore } from './schema' /** @@ -47,6 +48,7 @@ export type CampaignStop = | 'segmentLimit' | 'maxTurns' | 'budget' + | 'gameOver' | 'steering' | 'analyst' | 'abort' @@ -160,6 +162,14 @@ export interface CampaignOptions { steer?: (report: SegmentReport, analysis: Analysis | null) => Promise /** Persistence hook. Save the ledger here so a killed process can resume. */ onLedger?: (ledger: CampaignLedger) => void | Promise + /** + * End the campaign as soon as the game declares itself over. Off by default, + * for the reason `EpisodeOptions.stopAtGameOver` states: episode length is a + * denominator, and shortening it is the caller's decision. + * + * A resumed campaign whose game already ended plays no further segment. + */ + stopAtGameOver?: boolean signal?: AbortSignal } @@ -207,8 +217,18 @@ export async function runCampaign( let guidance = resumeGuidance(ledger) let stopped = false + // Why the whole campaign ended, once it is known. Null while it is still the + // limits that decide, which the tail of this function reads off the rollout. + let endedBy: EpisodeStop | null = null while (!stopped && rollout.turns < maxTurns && rollout.spent < budgetUsd) { options.signal?.throwIfAborted() + // A campaign resumed from a ledger whose game already ended starts no new + // segment and records none. An empty segment would report a boundary the + // run never played to. + if (options.stopAtGameOver === true && isGameOver(game, rollout.state)) { + endedBy = 'gameOver' + break + } const segment = ledger.segments.length const startTurn = rollout.turns + 1 const guidanceInEffect = guidance @@ -225,6 +245,7 @@ export async function runCampaign( maxTurns, maxDecisions: segmentTurns, ...(guidance === undefined ? {} : { guidance }), + ...(options.stopAtGameOver === undefined ? {} : { stopAtGameOver: options.stopAtGameOver }), ...(options.signal === undefined ? {} : { signal: options.signal }), }) } catch (error) { @@ -301,12 +322,19 @@ export async function runCampaign( } // Explicit steering outranks the analyst; a stop from either ends the run. guidance = nextGuidance(guidance, steering, analysis) - stopped = steering?.stop === true || analysis?.recommendation === 'stop' + const hookStop = steering?.stop === true + ? 'steering' + : analysis?.recommendation === 'stop' ? 'analyst' : null // A hard limit that already ended the segment keeps its own reason. - if (stopped && stoppedBy === 'segmentLimit') { - stoppedBy = steering?.stop === true ? 'steering' : 'analyst' - segmentRecord.stoppedBy = stoppedBy + if (hookStop !== null && stoppedBy === 'segmentLimit') { + stoppedBy = hookStop + segmentRecord.stoppedBy = hookStop } + // A finished game ends the campaign whatever a hook advises: there is + // nothing left to play, and no note can restart it. + if (stoppedBy === 'gameOver') endedBy = 'gameOver' + else if (hookStop !== null) endedBy = hookStop + stopped = endedBy !== null } } finally { ledger.updatedAt = new Date().toISOString() @@ -315,7 +343,9 @@ export async function runCampaign( if (abortError !== undefined) throw abortError } - const record = finalizeRecord(game, contract, seed, rollout, budgetUsd, started) + // The record states why the CAMPAIGN ended, not why its last segment paused. + const runStop: EpisodeStop = endedBy ?? (rollout.turns >= maxTurns ? 'maxTurns' : 'budget') + const record = finalizeRecord(game, contract, seed, rollout, budgetUsd, started, runStop) syncLedger(ledger, rollout) // Replay-verified progress supersedes the live tracker once the run is over. ledger.verified = [...record.verified] @@ -479,6 +509,7 @@ function isStop(value: unknown): value is CampaignStop { return value === 'segmentLimit' || value === 'maxTurns' || value === 'budget' + || value === 'gameOver' || value === 'steering' || value === 'analyst' || value === 'abort' diff --git a/docs/adapters.md b/docs/adapters.md index cb40d54..a9c5f8a 100644 --- a/docs/adapters.md +++ b/docs/adapters.md @@ -24,6 +24,30 @@ Read the mode column strictly. `trusted-recorder` means a named recorder signed what it captured and nothing was independently reproduced. `platform-attested` means a signed recorder read normalized progress from a platform API; it is not a signature from that platform. +## Which adapters declare the end of play + +`Game.over(state)` is optional, and it is what an episode reads when the caller +passes `stopAtGameOver`. A game that omits it is never over. + +| Adapter | `over(state)` reads | +|---|---| +| `adapters/ale` | `engineState.terminal`, set when `ale.game_over()` first holds | +| `adapters/gymnasium` | `engineState.terminated` or `engineState.truncated` | +| `adapters/stable-retro` | `engineState.episodeDone` | +| `adapters/native-2048` | `engineState.gameOver`, set when no move changes the board | +| `adapters/pyboy-generic`, `adapters/pyboy-tetris`, `adapters/retroarch` | Nothing; these workers publish no terminal flag, so the episode runs to a limit | +| `platforms/steam`, `platforms/xbox` | Nothing; the title runs elsewhere | + +Each adapter maps its own substrate, because there is no shared spelling of the +flag. It must be pure, like `step`: a verifier recomputes the final state from +the seed and the input log and asks the same question. + +Every emulator worker here already stops acting once its own terminal flag is +set, so the decisions an episode takes after that point change nothing at all. +Measured on ALE Breakout at 300 turns: from the game-over decision onward the +screen hash, the save-state hash, and the engine state are byte-identical to +the end of the run. + ## The observation image channel Every emulator adapter here was already capturing the screen for evidence and then showing the agent an ASCII downsample of it. diff --git a/episode-loop.ts b/episode-loop.ts index 119d712..0d1146e 100644 --- a/episode-loop.ts +++ b/episode-loop.ts @@ -10,13 +10,14 @@ * This module is internal. It is not exported from `index.ts`. */ import { attestRun, inputStatistics, MilestoneTracker } from './attestation' -import { InputLog, observationOf, observationTextOf, type Game } from './runtime' +import { InputLog, isGameOver, observationOf, observationTextOf, type Game } from './runtime' import type { MilestoneContract } from './schema' import type { AgentDecisionContext, AgentDriver, AgentHistoryEntry, EpisodeRecord, + EpisodeStop, MilestoneCostRow, } from './episode' @@ -105,7 +106,7 @@ export function applyInput( rollout.turns += 1 } -export type RolloutStop = 'segmentLimit' | 'maxTurns' | 'budget' +export type RolloutStop = 'segmentLimit' | 'maxTurns' | 'budget' | 'gameOver' export interface RolloutLimits { budgetUsd: number @@ -114,10 +115,18 @@ export interface RolloutLimits { maxDecisions?: number /** Latest supervisor or analyst note, passed to every decision in this call. */ guidance?: string + /** Stop as soon as the game declares itself over. Off by default. */ + stopAtGameOver?: boolean signal?: AbortSignal } -/** Drive the agent until a limit stops it. Returns the limit that stopped it. */ +/** + * Drive the agent until a stop condition holds. Returns the one that held. + * + * Every stop is an ordinary exit from the loop, so the caller finalizes the + * rollout the same way whichever one fired. Nothing here throws to end a run: + * an exception would leave the caller without the record the run is graded on. + */ export async function advanceRollout( game: Game, driver: AgentDriver, @@ -126,7 +135,14 @@ export async function advanceRollout( limits: RolloutLimits, ): Promise { let taken = 0 - while (rollout.turns < limits.maxTurns && rollout.spent < limits.budgetUsd) { + // The conditions are asked in one place, before every decision, so game over + // is seen at the start of the run as well as after the last input. Order is + // precedence: a finished game outranks the harness limits, and both outrank + // a segment boundary, which only pauses a campaign. + while (true) { + if (limits.stopAtGameOver === true && isGameOver(game, rollout.state)) return 'gameOver' + if (rollout.turns >= limits.maxTurns) return 'maxTurns' + if (rollout.spent >= limits.budgetUsd) return 'budget' if (limits.maxDecisions !== undefined && taken >= limits.maxDecisions) return 'segmentLimit' limits.signal?.throwIfAborted() // One observation per decision. An over-cap image throws here, which fails @@ -163,7 +179,6 @@ export async function advanceRollout( applyInput(game, rollout, turn.input, turn.costUsd, latencyMs) taken += 1 } - return rollout.turns >= limits.maxTurns ? 'maxTurns' : 'budget' } /** Attest the whole rollout and build the record both entrypoints return. */ @@ -174,6 +189,7 @@ export function finalizeRecord( rollout: Rollout, budgetUsd: number, startedAtMs: number, + stoppedBy: EpisodeStop, ): EpisodeRecord { const attestation = attestRun(game, contract, seed, rollout.log, []) return { @@ -182,6 +198,11 @@ export function finalizeRecord( spentUsd: round4(rollout.spent), budgetUsd, budgetExhausted: rollout.spent >= budgetUsd, + stoppedBy, + // Asked of every run, not only of a run that stopped for it: a turn-limited + // episode that reports `gameOver: true` is one that kept paying for + // decisions the game could no longer act on. + gameOver: game.over === undefined ? null : isGameOver(game, rollout.state), verified: attestation.verified, score: attestation.score, milestones: [...rollout.milestones], diff --git a/episode.ts b/episode.ts index 6b295f5..19ea6e0 100644 --- a/episode.ts +++ b/episode.ts @@ -91,12 +91,54 @@ export interface MilestoneCostRow { costUsd: number } +/** + * Why an episode ended. + * + * `maxTurns` and `budget` are harness limits. `gameOver` is the game itself: + * the state machine reported that no input can change anything any more. + * `steering` and `analyst` are campaign hooks and never end a single episode. + * + * Game over outranks the limits. A run that reaches its last allowed turn and + * a finished game at the same instant reports `gameOver`, because a finished + * game had no turn left to give whatever the limit said. + */ +export type EpisodeStop = 'maxTurns' | 'budget' | 'gameOver' | 'steering' | 'analyst' + +/** Optional behaviour of one episode. Every field is off by default. */ +export interface EpisodeOptions { + /** + * End the episode as soon as the game declares itself over. + * + * Off by default, and deliberately so. Episode length is the denominator a + * study divides by, and rounds of one study compare only while that + * denominator is fixed at the turn limit. Turning this on shortens an + * episode, so it is the caller's decision, taken once, for a whole series. + * + * It has no effect on a game that implements no `over()`, and none on the + * attestation: the run stops between two decisions and is finalized by the + * same path a turn-limited run takes. + */ + stopAtGameOver?: boolean +} + export interface EpisodeRecord { game: string turns: number spentUsd: number budgetUsd: number budgetExhausted: boolean + /** Which condition ended the run, stated rather than inferred by a reader. */ + stoppedBy: EpisodeStop + /** + * Whether the game was over at the last state of the run. `null` means the + * game declares no terminal state at all, which is a different fact from + * "the game was still playable". + * + * Read with `stoppedBy` it also states which mode produced the record: + * `stoppedBy: 'maxTurns'` next to `gameOver: true` can only come from a run + * that played on past the end, so `stopAtGameOver` was off. + */ + gameOver: boolean | null /** Every milestone the replay reproduced. */ verified: string[] /** `verified` over the contract's milestone count. */ @@ -117,6 +159,7 @@ export async function playEpisode( maxTurns: number, seed = 0, signal?: AbortSignal, + options: EpisodeOptions = {}, ): Promise<{ record: EpisodeRecord; log: InputLog }> { if (!Number.isFinite(budgetUsd) || budgetUsd < 0) throw new Error('budgetUsd must be non-negative') if (!Number.isInteger(maxTurns) || maxTurns < 0) throw new Error('maxTurns must be a non-negative integer') @@ -124,13 +167,18 @@ export async function playEpisode( const started = Date.now() const rollout = startRollout(game, contract, seed) - await advanceRollout(game, driver, rollout, seed, { + // The stop is a loop exit, never a thrown abort: the run falls out of the + // decision loop and is finalized below, so the attestation of an episode + // that ended at game over is built by the same code as any other. + const stop = await advanceRollout(game, driver, rollout, seed, { budgetUsd, maxTurns, ...(signal === undefined ? {} : { signal }), + ...(options.stopAtGameOver === undefined ? {} : { stopAtGameOver: options.stopAtGameOver }), }) + if (stop === 'segmentLimit') throw new Error('single episode stopped at a segment limit it never set') return { - record: finalizeRecord(game, contract, seed, rollout, budgetUsd, started), + record: finalizeRecord(game, contract, seed, rollout, budgetUsd, started, stop), log: rollout.log, } } diff --git a/execute.ts b/execute.ts index d1fc446..fac07ad 100644 --- a/execute.ts +++ b/execute.ts @@ -14,6 +14,8 @@ export interface ExecuteBenchmarkOptions { budgetUsd: number maxTurns: number seed?: number + /** End the run as soon as the game declares itself over. Off by default. */ + stopAtGameOver?: boolean signal?: AbortSignal actor: RunEnvelope['actor'] signer: { @@ -51,6 +53,7 @@ export async function executeBenchmark( options.maxTurns, seed, options.signal, + { ...(options.stopAtGameOver === undefined ? {} : { stopAtGameOver: options.stopAtGameOver }) }, ) if (turns.length !== record.latencyMs.length || turns.length !== log.inputs().length) { throw new Error(`decision capture mismatch: turns=${turns.length} latency=${record.latencyMs.length} log=${log.inputs().length}`) diff --git a/game-over.test.mts b/game-over.test.mts new file mode 100644 index 0000000..ed416d0 --- /dev/null +++ b/game-over.test.mts @@ -0,0 +1,243 @@ +/** + * Game-over stop gates — an episode may end because the GAME ended, and the + * record it produces is a complete one. + * + * The measurement that motivated the stop: on `ale-breakout` at 300 turns, a + * consumer counted 163 of 300 decisions (54.3%) taken after lives reached 0 + * with the engine's own `terminal` flag set. The ALE worker breaks out of its + * action-repeat loop once that flag holds, so those inputs never reached the + * emulator; the episode still reported 300 of 300 answered. + * + * Every gate here is deterministic and offline. `screen-puzzle` is a shipped + * adapter whose far gate is a real terminal state; the two local fixtures cover + * the ends of the range no shipped adapter reaches in one input. + */ +import { strict as assert } from 'node:assert' +import { deriveContract } from './authoring' +import { verifyRunArtifact } from './attestation' +import { runCampaign, type CampaignLedger } from './campaign' +import { playEpisode, scriptedDriver, type EpisodeRecord } from './episode' +import { engineCrawler, engineCrawlerContract, ENGINE_CRAWLER_REFERENCE } from './adapters/engine-crawler' +import { screenPuzzle, screenPuzzleContract, SCREEN_PUZZLE_REFERENCE, PUZZLE_WIDTH } from './adapters/screen-puzzle' +import { contractHash, type MilestoneContract } from './schema' +import type { Game, InputLog } from './runtime' + +const puzzle = screenPuzzleContract() +/** Reaching the gate takes every reference input; `noop` after that. */ +const GATE_TURN = SCREEN_PUZZLE_REFERENCE.length +const drive = () => scriptedDriver([...SCREEN_PUZZLE_REFERENCE]) + +/** Replay the log against a fresh verifier, the way an artifact is checked. */ +function replayVerifies(game: Game, contract: MilestoneContract, seed: number, record: EpisodeRecord, log: InputLog): void { + const verdict = verifyRunArtifact(game, contract, { + gameId: game.id, + contractHash: contractHash(contract), + seed, + inputs: [...log.inputs()], + claimed: [...record.verified], + }) + assert.equal(verdict.verdict, 'clean', `replay rejected the record: ${verdict.reasons.join(', ')}`) + assert.deepEqual(verdict.recomputed, record.verified) + assert.equal(record.verdict, 'clean') + assert.equal(record.replayDivergence, false) +} + +// --------------------------------------------------------------------------- +// The pathology, reproduced offline: without the stop, an episode keeps paying +// for decisions after the game has ended, and the extra decisions buy nothing. +{ + const long = await playEpisode(screenPuzzle, puzzle, drive(), 1, 20) + assert.equal(long.record.turns, 20) + assert.equal(long.record.stoppedBy, 'maxTurns') + assert.equal(long.record.gameOver, true, 'the puzzle was over long before turn 20') + + const stopped = await playEpisode(screenPuzzle, puzzle, drive(), 1, 20, 0, undefined, { stopAtGameOver: true }) + assert.equal(stopped.record.turns, GATE_TURN) + assert.equal(stopped.record.stoppedBy, 'gameOver') + assert.equal(stopped.record.gameOver, true) + // 12 of 20 decisions were played past the end, and dropping them costs no + // progress: the shorter run verifies exactly the same milestones. + assert.equal(long.record.turns - stopped.record.turns, 12) + assert.deepEqual(stopped.record.verified, long.record.verified) + assert.deepEqual(stopped.record.score, long.record.score) + + // The attestation survives the early stop and is checked the same way. + replayVerifies(screenPuzzle, puzzle, 0, stopped.record, stopped.log) + replayVerifies(screenPuzzle, puzzle, 0, long.record, long.log) +} + +// --------------------------------------------------------------------------- +// The three stop reasons are recorded and distinct, on one game and one driver. +{ + const limit = await playEpisode(screenPuzzle, puzzle, drive(), 1, 3, 0, undefined, { stopAtGameOver: true }) + const budget = await playEpisode(screenPuzzle, puzzle, drive(), 0, 20, 0, undefined, { stopAtGameOver: true }) + const ended = await playEpisode(screenPuzzle, puzzle, drive(), 1, 20, 0, undefined, { stopAtGameOver: true }) + + assert.equal(limit.record.stoppedBy, 'maxTurns') + assert.equal(budget.record.stoppedBy, 'budget') + assert.equal(ended.record.stoppedBy, 'gameOver') + assert.equal(new Set([limit.record.stoppedBy, budget.record.stoppedBy, ended.record.stoppedBy]).size, 3) + + // A limit still ends a run while the game-over stop is armed. + assert.equal(limit.record.turns, 3) + assert.equal(limit.record.gameOver, false) + assert.equal(budget.record.turns, 0) + assert.equal(budget.record.budgetExhausted, true) +} + +// --------------------------------------------------------------------------- +// COMPATIBILITY GATE. An adapter that implements no `over()` behaves exactly as +// it does today, whether or not the caller arms the stop. +{ + assert.equal(engineCrawler.over, undefined, 'this gate is only meaningful while the crawler declares no terminal state') + const contract = engineCrawlerContract() + const script = () => scriptedDriver([...ENGINE_CRAWLER_REFERENCE]) + const today = await playEpisode(engineCrawler, contract, script(), 1, 12) + const armed = await playEpisode(engineCrawler, contract, script(), 1, 12, 0, undefined, { stopAtGameOver: true }) + + // `ms` is wall time; everything a reader grades is identical. + const { ms: _today, ...todayFields } = today.record + const { ms: _armed, ...armedFields } = armed.record + assert.deepEqual(armedFields, todayFields) + assert.deepEqual([...armed.log.inputs()], [...today.log.inputs()]) + assert.equal(armed.record.turns, 12) + assert.equal(armed.record.stoppedBy, 'maxTurns') + // Null, not false: the game states no terminal condition, which is a + // different fact from stating that it is still playable. + assert.equal(armed.record.gameOver, null) + assert.equal(today.record.gameOver, null) +} + +// --------------------------------------------------------------------------- +// Game over on the LAST allowed decision. The game ended and the turn limit was +// reached at the same instant; the record names the game, because a finished +// game had no further turn to give whatever the limit said. +{ + const armed = await playEpisode(screenPuzzle, puzzle, drive(), 1, GATE_TURN, 0, undefined, { stopAtGameOver: true }) + const bare = await playEpisode(screenPuzzle, puzzle, drive(), 1, GATE_TURN) + assert.equal(armed.record.turns, GATE_TURN) + assert.equal(bare.record.turns, GATE_TURN) + assert.equal(armed.record.stoppedBy, 'gameOver') + assert.equal(bare.record.stoppedBy, 'maxTurns') + assert.equal(armed.record.gameOver, true) + assert.equal(bare.record.gameOver, true) + // Same inputs, same progression, same attestation: only the stated reason + // differs, so a reader can tell the two modes apart at equal length. + assert.deepEqual([...armed.log.inputs()], [...bare.log.inputs()]) + assert.deepEqual(armed.record.verified, bare.record.verified) + replayVerifies(screenPuzzle, puzzle, 0, armed.record, armed.log) +} + +// --------------------------------------------------------------------------- +// Game over on the FIRST decision, and before it. +interface SprintState { + steps: number +} + +/** One input finishes it — the shortest playable episode. */ +const sprint: Game = { + id: 'game-over-sprint', + init: () => ({ steps: 0 }), + step: (s) => ({ steps: s.steps + 1 }), + frame: (s) => `steps ${s.steps}`, + evidence: (s) => ({ engineState: { steps: s.steps } }), + over: (s) => s.steps >= 1, +} +const sprintContract = deriveContract(sprint, 0, ['go'], [ + { + afterInputs: 1, + id: 'moved', + tier: 'engine-state', + glitchClass: 'legal', + sample: () => ({ kind: 'state-path', path: 'steps', op: '>=', value: 1 }), + }, +]) + +{ + const { record, log } = await playEpisode(sprint, sprintContract, scriptedDriver(['go']), 1, 10, 0, undefined, { stopAtGameOver: true }) + assert.equal(record.turns, 1) + assert.equal(record.stoppedBy, 'gameOver') + assert.equal(record.gameOver, true) + assert.deepEqual(record.verified, ['moved']) + replayVerifies(sprint, sprintContract, 0, record, log) +} + +/** Already finished at `init`, so the episode never takes a decision. */ +const spent: Game = { ...sprint, id: 'game-over-at-init', over: () => true } +const spentContract = deriveContract(spent, 0, ['go'], [ + { + afterInputs: 0, + id: 'booted', + tier: 'engine-state', + glitchClass: 'legal', + sample: () => ({ kind: 'state-path', path: 'steps', op: '>=', value: 0 }), + }, +]) + +{ + const { record, log } = await playEpisode(spent, spentContract, scriptedDriver(['go']), 1, 10, 0, undefined, { stopAtGameOver: true }) + assert.equal(record.turns, 0) + assert.equal(record.stoppedBy, 'gameOver') + assert.equal(record.gameOver, true) + assert.equal(record.spentUsd, 0) + assert.equal(log.inputs().length, 0) + // A zero-decision run is still a complete, replay-checked record. + assert.deepEqual(record.verified, ['booted']) + replayVerifies(spent, spentContract, 0, record, log) +} + +// --------------------------------------------------------------------------- +// An `over()` that does not answer with a boolean is an adapter bug, named as +// one rather than silently read as false. +{ + const broken = { ...sprint, id: 'game-over-broken', over: (() => 1) as unknown as (s: SprintState) => boolean } + await assert.rejects( + playEpisode(broken, { ...sprintContract, gameId: 'game-over-broken' }, scriptedDriver(['go']), 1, 4, 0, undefined, { stopAtGameOver: true }), + /game-over-broken over\(\) returned number, expected a boolean/u, + ) +} + +// --------------------------------------------------------------------------- +// A campaign ends on the same signal, and its ledger says so. +{ + const saved: CampaignLedger[] = [] + const { record, ledger } = await runCampaign(screenPuzzle, puzzle, drive(), { + budgetUsd: 1, + maxTurns: 20, + segmentTurns: 3, + stopAtGameOver: true, + onLedger: (next) => { saved.push(structuredClone(next)) }, + }) + assert.equal(record.turns, GATE_TURN) + assert.equal(record.stoppedBy, 'gameOver') + assert.equal(ledger.inputs.length, GATE_TURN) + assert.equal(ledger.segments.at(-1)?.stoppedBy, 'gameOver') + assert.equal(saved.length > 0, true) + + // Resuming a finished campaign plays nothing and records no empty segment. + const segmentsBefore = ledger.segments.length + const resumed = await runCampaign(screenPuzzle, puzzle, drive(), { + budgetUsd: 1, + maxTurns: 20, + segmentTurns: 3, + stopAtGameOver: true, + ledger, + }) + assert.equal(resumed.record.turns, GATE_TURN) + assert.equal(resumed.record.stoppedBy, 'gameOver') + assert.equal(resumed.ledger.segments.length, segmentsBefore) + assert.deepEqual(resumed.ledger.inputs, ledger.inputs) + + // The same campaign without the stop keeps playing to the turn limit. + const full = await runCampaign(screenPuzzle, puzzle, drive(), { + budgetUsd: 1, + maxTurns: 20, + segmentTurns: 3, + }) + assert.equal(full.record.turns, 20) + assert.equal(full.record.stoppedBy, 'maxTurns') + assert.equal(full.record.gameOver, true) + assert.deepEqual(full.record.verified, record.verified) +} + +console.log(`playproof game-over stop: ${PUZZLE_WIDTH - 1} moves to the gate, three distinct stop reasons, compatibility and replay OK`) diff --git a/gymnasium.test.mts b/gymnasium.test.mts index faf6dd2..973427f 100644 --- a/gymnasium.test.mts +++ b/gymnasium.test.mts @@ -15,6 +15,7 @@ import { strict as assert } from 'node:assert' import { spawnSync } from 'node:child_process' import { tmpdir } from 'node:os' import { attestRun } from './attestation' +import { playEpisode, scriptedDriver } from './episode' import { logFrom } from './runtime' import { validateContract } from './schema' import { GymRpc } from './adapters/gym-rpc' @@ -176,6 +177,30 @@ if (!pythonHasGymnasium()) { for (const input of ['a0', 'a1', 'a2']) ended = lake.game.step(ended, input) assert.deepEqual(lake.game.evidence(ended), atEnd, 'a terminated episode kept stepping') + // The game-over stop on a second real substrate. FrozenLake terminates + // when the walk reaches the goal, and the worker freezes the environment + // there, so every later decision is inert exactly as ALE's is. + // Bound once: the driver factory is a closure, and `lake` is the mutable + // handle the teardown clears. + const frozenLake = lake + const LAKE_TURNS = frozenLake.reference.length + 20 + const walk = () => scriptedDriver([...frozenLake.reference]) + const lakeFull = await playEpisode(frozenLake.game, frozenLake.contract, walk(), 1, LAKE_TURNS, frozenLake.seed) + const lakeStopped = await playEpisode( + frozenLake.game, frozenLake.contract, walk(), 1, LAKE_TURNS, frozenLake.seed, undefined, { stopAtGameOver: true }, + ) + assert.equal(lakeFull.record.turns, LAKE_TURNS) + assert.equal(lakeFull.record.stoppedBy, 'maxTurns') + assert.equal(lakeFull.record.gameOver, true) + assert.equal(lakeStopped.record.turns, frozenLake.reference.length) + assert.equal(lakeStopped.record.stoppedBy, 'gameOver') + assert.deepEqual(lakeStopped.record.verified, lakeAll) + assert.deepEqual(lakeStopped.record.verified, lakeFull.record.verified) + assert.equal(lakeStopped.record.verdict, 'clean') + assert.equal(lakeStopped.record.replayDivergence, false) + const lakeRecomputed = attestRun(frozenLake.game, frozenLake.contract, frozenLake.seed, lakeStopped.log, [...lakeStopped.record.verified]) + assert.equal(lakeRecomputed.verdict, 'clean', lakeRecomputed.reasons.join('; ')) + // Determinism across processes on the second environment too. const lakeFirst = trace(lake, lake.reference) second = makeGymnasium({ envId: FROZENLAKE }) @@ -202,7 +227,9 @@ if (!pythonHasGymnasium()) { `gymnasium: ${cartpole.identity.envId} on ${cartpole.identity.actionSpace} and ${FROZENLAKE} — ` + `derivation, ${cartpole.contract.milestones.length}+${lakeAll.length}-milestone contracts, known-good, ` + `graded partial, false-claim, cross-process determinism over ${first.length} snapshots, ` + - `engine and replay checkpoints, unknown-input no-op, teardown OK ` + + `engine and replay checkpoints, unknown-input no-op, ` + + `game-over stop at ${lakeStopped.record.turns} of ${LAKE_TURNS} FrozenLake decisions ` + + `(${lakeFull.record.turns - lakeStopped.record.turns} dropped, milestones unchanged), teardown OK ` + `(reference balances ${finalState.steps} steps for reward ${finalState.cumulativeReward / 1000})`, ) } finally { diff --git a/package.json b/package.json index f531db1..e495ab2 100644 --- a/package.json +++ b/package.json @@ -130,7 +130,7 @@ "build": "pnpm clean && tsup && node scripts/copy-assets.mjs", "check:boundary": "node scripts/check-boundary.mjs", "typecheck": "tsc -p tsconfig.json --noEmit", - "test": "tsx playproof.test.mts && tsx calibration.test.mts && tsx episode.test.mts && tsx observation.test.mts && tsx platform.test.mts && tsx desktop-platforms.test.mts && tsx drivers.test.mts && tsx campaign.test.mts", + "test": "tsx playproof.test.mts && tsx calibration.test.mts && tsx episode.test.mts && tsx game-over.test.mts && tsx observation.test.mts && tsx platform.test.mts && tsx desktop-platforms.test.mts && tsx drivers.test.mts && tsx campaign.test.mts", "test:pyboy": "tsx pyboy-tetris.test.mts", "test:retro": "tsx stable-retro.test.mts", "test:ale": "tsx ale.test.mts", diff --git a/runtime.ts b/runtime.ts index 309cd15..b5aa771 100644 --- a/runtime.ts +++ b/runtime.ts @@ -71,6 +71,27 @@ export interface Game { observe?(state: S): Observation /** Privileged progression channel — harness-side only. */ evidence(state: S): Evidence + /** + * Whether the game is finished, so that no further input can change it. + * + * The member is optional and a game that omits it is never over. That is + * what a game with no terminal state means, and it is what every adapter + * written before this member already did. + * + * It must be PURE, like `step`. A verifier replays the input log, recomputes + * the final state, and asks again; an answer that reads wall time or the + * live process would make "this episode stopped at game over" a claim the + * verifier cannot reproduce. + * + * Derive it from state the adapter already holds. An emulator adapter reads + * the terminal flag its worker publishes in `evidence().engineState`. There + * is no shared spelling of that flag across substrates — ALE writes + * `terminal`, Gymnasium writes `terminated` and `truncated`, stable-retro + * writes `episodeDone`, the 2048 core writes `gameOver` — so the mapping + * belongs to the adapter that knows its own engine, not to a guess the + * harness makes over field names. + */ + over?(state: S): boolean } /** @@ -159,6 +180,22 @@ export function observationTextOf(game: Game, state: S): string { return text } +/** + * Whether a game declares this state finished, with the never-over default. + * + * Every path that asks the question goes through this function, so a game that + * publishes no terminal state answers the same way everywhere: the episode + * loop, the campaign loop, and the record all read one definition. + */ +export function isGameOver(game: Game, state: S): boolean { + if (game.over === undefined) return false + const over = game.over(state) + if (typeof over !== 'boolean') { + throw new Error(`game ${game.id} over() returned ${typeof over}, expected a boolean`) + } + return over +} + /** Validate one image against the bounds and return its decoded byte count. */ function checkObservationImage(gameId: string, index: number, image: ObservationImage): number { const where = `game ${gameId} observation image ${index}`