From 81fc64c762be3ab5ecef21aaaf44170ef30c925c Mon Sep 17 00:00:00 2001
From: Drew Stone <drewstone329@gmail.com>
Date: Sat, 22 Aug 2026 11:07:30 -0700
Subject: [PATCH 1/6] feat(schema): split milestones into achievement and
 replay-identity

A hash check pins the exact bytes one recorded run produced. It proves that
a replay reproduced that run. It is not a progression an independent policy
can reach by playing, so it must not count as one.

The role is derived from the check kind, not declared on the milestone, so
every existing contract keeps its bytes and its hash, and no author can
forget to set it. `contractEarnability` also walks `requires`: an
achievement gated behind a hash is as unreachable as the hash, because
`MilestoneTracker` admits a milestone only after every prerequisite passed.

`MilestoneScore` reports earned over earnable next to verified over total,
so a score reads "3 of 4 earnable" instead of "3 of 6".
---
 schema.ts | 144 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 144 insertions(+)

diff --git a/schema.ts b/schema.ts
index ec1e07b..dc95b5c 100644
--- a/schema.ts
+++ b/schema.ts
@@ -107,3 +107,147 @@ export function canonicalContractJson(c: MilestoneContract): string {
 export function contractHash(c: MilestoneContract): string {
   return createHash('sha256').update(canonicalContractJson(c)).digest('hex')
 }
+
+/**
+ * What a milestone's check can prove.
+ *
+ * `achievement` — a threshold, a normalized field, or an event that an
+ * independent policy reaches by playing. Two different valid trajectories can
+ * both satisfy it.
+ *
+ * `identity` — a hash over the exact bytes one recorded run produced. It proves
+ * that a replay reproduced that run, which is what replay attestation is for.
+ * It is not a progression: earning it means reproducing the reference's frame
+ * or save, not playing well.
+ */
+export type MilestoneRole = 'achievement' | 'identity'
+
+const ROLE_FOR_CHECK: Record<MilestoneCheck['kind'], MilestoneRole> = {
+  'state-path': 'achievement',
+  'save-path': 'achievement',
+  'save-hash': 'identity',
+  'log-contains': 'achievement',
+  'frame-path': 'achievement',
+  'frame-hash': 'identity',
+}
+
+/**
+ * The role a check carries, derived from its kind.
+ *
+ * The role is derived rather than declared so that an existing contract keeps
+ * its bytes and its hash, and so that an author cannot forget to set it. A
+ * hash check is an identity check whatever the author intended.
+ */
+export function checkRole(check: MilestoneCheck): MilestoneRole {
+  return ROLE_FOR_CHECK[check.kind]
+}
+
+/** Which milestones of a contract an independent policy can earn. */
+export interface ContractEarnability {
+  /** Milestone ids an independent policy can earn by playing. */
+  earnable: string[]
+  /** Milestone ids only a replay of the reference run earns. */
+  unearnable: string[]
+  /** Why each unearnable milestone is out of reach, keyed by milestone id. */
+  reasons: Record<string, string>
+}
+
+/**
+ * Split a contract into the milestones a policy can earn and the ones it cannot.
+ *
+ * A milestone is unearnable when its own check pins exact bytes, and also when
+ * it depends on one that does: `MilestoneTracker` admits a milestone only after
+ * every prerequisite has passed, so an achievement gated behind a hash is as
+ * unreachable as the hash. A milestone with a missing or cyclic requirement is
+ * unearnable for the same reason — no run ever satisfies it.
+ */
+export function contractEarnability(contract: MilestoneContract): ContractEarnability {
+  const byId = new Map(contract.milestones.map((m) => [m.id, m]))
+  const decided = new Map<string, string | null>()
+  const visiting = new Set<string>()
+  const reasonFor = (m: Milestone): string | null => {
+    const cached = decided.get(m.id)
+    if (cached !== undefined) return cached
+    if (visiting.has(m.id)) return `sits on a dependency cycle, so no run satisfies it`
+    visiting.add(m.id)
+    let reason: string | null = null
+    if (checkRole(m.check) === 'identity') {
+      reason = `its ${m.check.kind} check pins the reference run's exact bytes`
+    } else {
+      for (const required of m.requires) {
+        const prerequisite = byId.get(required)
+        if (prerequisite === undefined) {
+          reason = `requires ${required}, which the contract does not declare`
+          break
+        }
+        if (reasonFor(prerequisite) !== null) {
+          reason = `requires ${required}, which no independent policy can earn`
+          break
+        }
+      }
+    }
+    visiting.delete(m.id)
+    decided.set(m.id, reason)
+    return reason
+  }
+
+  const earnable: string[] = []
+  const unearnable: string[] = []
+  const reasons: Record<string, string> = {}
+  for (const m of contract.milestones) {
+    const reason = reasonFor(m)
+    if (reason === null) earnable.push(m.id)
+    else {
+      unearnable.push(m.id)
+      reasons[m.id] = reason
+    }
+  }
+  return { earnable, unearnable, reasons }
+}
+
+/**
+ * A run's progress against a contract, with the earnable denominator separated
+ * from the total.
+ *
+ * `earned` over `earnable` is the score a run may be compared on. `verified`
+ * over `total` includes the replay-identity checks, which only a replay of the
+ * reference reproduces.
+ */
+export interface MilestoneScore {
+  /** Milestones the run verified, replay-identity checks included. */
+  verified: number
+  /** Of those, the ones an independent policy can earn. */
+  earned: number
+  /** Milestones of the contract an independent policy can earn. */
+  earnable: number
+  /** Milestones of the contract. */
+  total: number
+}
+
+/** The verified milestones an independent policy can earn, in verified order. */
+export function earnedMilestones(contract: MilestoneContract, verified: readonly string[]): string[] {
+  const earnable = new Set(contractEarnability(contract).earnable)
+  return verified.filter((id) => earnable.has(id))
+}
+
+/** Score a verified milestone set against its contract. */
+export function scoreMilestones(contract: MilestoneContract, verified: readonly string[]): MilestoneScore {
+  const { earnable } = contractEarnability(contract)
+  const set = new Set(earnable)
+  return {
+    verified: verified.length,
+    earned: verified.filter((id) => set.has(id)).length,
+    earnable: earnable.length,
+    total: contract.milestones.length,
+  }
+}
+
+/** One line for a report or a log. Never states an earned count alone. */
+export function formatMilestoneScore(score: MilestoneScore): string {
+  const identity = score.total - score.earnable
+  if (identity === 0) return `${score.earned} of ${score.earnable} earnable`
+  return (
+    `${score.earned} of ${score.earnable} earnable ` +
+    `(${score.verified} of ${score.total} verified, ${identity} replay-identity)`
+  )
+}

From 96144ed0a335e477be9c908d1ecf0f3a077916eb Mon Sep 17 00:00:00 2001
From: Drew Stone <drewstone329@gmail.com>
Date: Sat, 22 Aug 2026 11:07:30 -0700
Subject: [PATCH 2/6] feat(attestation): report earned milestones next to
 verified

`Attestation` and `EpisodeRecord` carry `earned` and `score`. `verified`
keeps its meaning, so replay attestation and every existing consumer are
unchanged. The campaign segment report carries `scoreSoFar`, because an
analyst who reads progress mid-run must see the earnable denominator.

The campaign ledger is unchanged: its `verified` list plus the contract it
pins by hash reproduce the score through `scoreMilestones`, so an old
ledger still loads.
---
 attestation.ts  | 16 ++++++++++++++--
 campaign.ts     |  8 +++++++-
 episode-loop.ts |  2 ++
 episode.ts      | 11 ++++++++++-
 4 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/attestation.ts b/attestation.ts
index dad58ed..a40e86a 100644
--- a/attestation.ts
+++ b/attestation.ts
@@ -8,14 +8,24 @@
  */
 import { logFrom } from './runtime'
 import type { Evidence, Game, Input, InputLog } from './runtime'
-import { contractHash } from './schema'
-import type { Milestone, MilestoneContract, NumericOperator } from './schema'
+import { contractHash, earnedMilestones, scoreMilestones } from './schema'
+import type { Milestone, MilestoneContract, MilestoneScore, NumericOperator } from './schema'
 
 export interface Attestation {
   gameId: string
   verdict: 'clean' | 'rejected'
   reasons: string[]
+  /** Every milestone the replay reproduced, replay-identity checks included. */
   verified: string[]
+  /**
+   * The subset of `verified` an independent policy can earn by playing.
+   *
+   * A hash check proves a replay reproduced the recorded run. Counting it as
+   * progress inflates the denominator of every reported score, so the two sets
+   * are reported apart. `verified` stays the attestation statement.
+   */
+  earned: string[]
+  score: MilestoneScore
   checks: { name: string; passed: boolean }[]
 }
 
@@ -109,6 +119,8 @@ export function attestRun<S>(
     verdict: reasons.length === 0 ? 'clean' : 'rejected',
     reasons,
     verified,
+    earned: earnedMilestones(contract, verified),
+    score: scoreMilestones(contract, verified),
     checks: [
       { name: 'input-log-chain', passed: chainOk },
       { name: 'claimed-milestones-reproduced', passed: notReproduced.length === 0 && unknownClaims.length === 0 },
diff --git a/campaign.ts b/campaign.ts
index 73f19ad..d372339 100644
--- a/campaign.ts
+++ b/campaign.ts
@@ -29,7 +29,7 @@ import type {
   MilestoneCostRow,
 } from './episode'
 import { observationTextOf, type Game, type InputLog } from './runtime'
-import { contractHash, type MilestoneContract } from './schema'
+import { contractHash, scoreMilestones, type MilestoneContract, type MilestoneScore } from './schema'
 
 /**
  * Trajectory entries kept in memory during a campaign.
@@ -121,6 +121,11 @@ export interface SegmentReport {
   newMilestones: MilestoneCostRow[]
   /** Milestones the live tracker observed so far, in observation order. */
   verifiedSoFar: string[]
+  /**
+   * Progress so far with the earnable denominator separated from the total, so
+   * an analyst reads "3 of 4 earnable" instead of "3 of 6".
+   */
+  scoreSoFar: MilestoneScore
   lastFrame: string
   recentHistory: AgentHistoryEntry[]
   /** Decision latencies of this segment only. */
@@ -263,6 +268,7 @@ export async function runCampaign<S>(
           remainingBudgetUsd: Math.max(0, budgetUsd - rollout.spent),
           newMilestones: rollout.milestones.slice(milestonesBefore),
           verifiedSoFar: rollout.tracker.verified(),
+          scoreSoFar: scoreMilestones(contract, rollout.tracker.verified()),
           // The segment report is a text ledger read by an analyst and written
           // to disk, so it carries the observation text and never its pixels.
           lastFrame: observationTextOf(game, rollout.state),
diff --git a/episode-loop.ts b/episode-loop.ts
index 898c1e8..23f7708 100644
--- a/episode-loop.ts
+++ b/episode-loop.ts
@@ -183,6 +183,8 @@ export function finalizeRecord<S>(
     budgetUsd,
     budgetExhausted: rollout.spent >= budgetUsd,
     verified: attestation.verified,
+    earned: attestation.earned,
+    score: attestation.score,
     milestones: [...rollout.milestones],
     replayDivergence: JSON.stringify(rollout.observed) !== JSON.stringify(attestation.verified),
     latencyMs: [...rollout.latencyMs],
diff --git a/episode.ts b/episode.ts
index 57fa9dd..34de041 100644
--- a/episode.ts
+++ b/episode.ts
@@ -1,7 +1,7 @@
 import type { InputStats } from './attestation'
 import { advanceRollout, finalizeRecord, startRollout } from './episode-loop'
 import type { InputLog, Game, Observation } from './runtime'
-import type { MilestoneContract } from './schema'
+import type { MilestoneContract, MilestoneScore } from './schema'
 
 /**
  * One prior action and the observation text produced by that action.
@@ -97,7 +97,16 @@ export interface EpisodeRecord {
   spentUsd: number
   budgetUsd: number
   budgetExhausted: boolean
+  /** Every milestone the replay reproduced, replay-identity checks included. */
   verified: string[]
+  /** The subset of `verified` an independent policy can earn by playing. */
+  earned: string[]
+  /**
+   * The run's progress with the earnable denominator separated from the total.
+   * Report `earned` of `earnable`; `verified` of `total` counts hash checks
+   * that only a replay of the reference reproduces.
+   */
+  score: MilestoneScore
   milestones: MilestoneCostRow[]
   replayDivergence: boolean
   latencyMs: number[]

From c24995877dcf4e1c105218a8da4063193de4cb6b Mon Sep 17 00:00:00 2001
From: Drew Stone <drewstone329@gmail.com>
Date: Sat, 22 Aug 2026 11:07:30 -0700
Subject: [PATCH 3/6] feat(calibration): refuse a contract with undeclared
 unearnable milestones

The separation test alone reads a replay-identity milestone as the
contract's strongest evidence: no trivial baseline earns it, so it lands in
`separating`. An all-hash contract therefore reported `separates: true`
while nothing but a replay of the reference could score on it.

`separating` now holds earnable milestones only, and `separates` compares
earnable counts, so a baseline that beats the reference on real progress
cannot be outvoted by hashes the reference reproduces by construction.

`assertContractSeparates` gains an exact declaration of the identity checks
an author accepts. An undeclared one, a stale declaration, or an identity
hash a trivial baseline reproduced all fail the gate.
`assertMilestonesEarnable` is the same check for a target that is not meant
to separate.
---
 calibration.ts | 201 ++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 181 insertions(+), 20 deletions(-)

diff --git a/calibration.ts b/calibration.ts
index 4f7d603..d0693c8 100644
--- a/calibration.ts
+++ b/calibration.ts
@@ -12,13 +12,23 @@
  * comparison. `assertContractSeparates` is the fail-closed gate a target author
  * calls before publishing a contract.
  *
+ * A trivial baseline is not the only way a contract fails to measure. A
+ * milestone can also be out of reach of EVERY policy, because a hash check
+ * pins the reference run's exact bytes. Such a milestone is never earned by a
+ * baseline, so the separation test alone reads it as the contract's strongest
+ * evidence, when in fact nothing but a replay of the reference reproduces it.
+ * The report therefore splits the contract into earnable and unearnable
+ * milestones, and the gate refuses a contract whose unearnable milestones the
+ * author has not declared.
+ *
  * Everything here is pure and synchronous, and depends only on the runtime,
  * schema, and attestation planes. It imports no adapter and no model provider.
  */
 import { attestRun } from './attestation'
 import { logFrom } from './runtime'
 import type { Game } from './runtime'
-import type { MilestoneContract } from './schema'
+import { contractEarnability, formatMilestoneScore, scoreMilestones } from './schema'
+import type { MilestoneContract, MilestoneScore } from './schema'
 
 /**
  * A deterministic input policy that needs no observation of the game.
@@ -101,12 +111,43 @@ export interface CalibrationReport {
   vocabulary: string[]
   reference: BaselineOutcome
   baselines: BaselineOutcome[]
-  /** milestones no baseline earned */
+  /**
+   * Earnable milestones the reference reached and no baseline did — the whole
+   * of the contract's discriminating power. A replay-identity milestone is
+   * excluded even though no baseline earned it, because being out of reach of
+   * every policy is not separation.
+   */
   separating: string[]
   /** milestones at least one baseline earned */
   trivial: string[]
+  /** contract milestones an independent policy can earn by playing */
+  earnable: string[]
+  /**
+   * Contract milestones only a replay of the reference earns: a hash check on
+   * the reference run's exact bytes, or a milestone gated behind one.
+   */
+  unearnable: string[]
+  /** why each unearnable milestone is out of reach, keyed by milestone id */
+  unearnableReasons: Record<string, string>
+  /**
+   * Unearnable milestones a trivial baseline reproduced anyway.
+   *
+   * A non-empty set is a measured finding about the game, not about the
+   * policy: the hash covers so little entropy that another trajectory collides
+   * with it, so it identifies nothing. Such a milestone is also listed in
+   * `trivial`.
+   */
+  unearnableReproduced: string[]
   /** the strongest baseline's verified count */
   bestBaselineCount: number
+  /**
+   * The strongest baseline's EARNABLE count. This is the number a reference
+   * must beat: a raw verified count flatters the reference, which reproduces
+   * every replay-identity check by construction.
+   */
+  bestBaselineEarnedCount: number
+  /** the reference's own progress, with the earnable denominator separated */
+  referenceScore: MilestoneScore
   separates: boolean
 }
 
@@ -164,9 +205,14 @@ export function calibrateContract<S>(
 
   const earnedByBaseline = new Set(baselines.flatMap((b) => b.verified))
   const order = contract.milestones.map((m) => m.id)
-  const separating = reference.verified.filter((id) => !earnedByBaseline.has(id))
+  const { earnable, unearnable, reasons } = contractEarnability(contract)
+  const earnableSet = new Set(earnable)
+  const separating = reference.verified.filter((id) => !earnedByBaseline.has(id) && earnableSet.has(id))
   const trivial = order.filter((id) => earnedByBaseline.has(id))
   const bestBaselineCount = baselines.reduce((best, b) => Math.max(best, b.verified.length), 0)
+  const earnedCount = (outcome: BaselineOutcome): number => outcome.verified.filter((id) => earnableSet.has(id)).length
+  const bestBaselineEarnedCount = baselines.reduce((best, b) => Math.max(best, earnedCount(b)), 0)
+  const referenceScore = scoreMilestones(contract, reference.verified)
 
   return {
     turns,
@@ -176,30 +222,145 @@ export function calibrateContract<S>(
     baselines,
     separating,
     trivial,
+    earnable,
+    unearnable,
+    unearnableReasons: reasons,
+    unearnableReproduced: unearnable.filter((id) => earnedByBaseline.has(id)),
     bestBaselineCount,
-    separates: separating.length > 0 && reference.verified.length > bestBaselineCount,
+    bestBaselineEarnedCount,
+    referenceScore,
+    separates: separating.length > 0 && referenceScore.earned > bestBaselineEarnedCount,
+  }
+}
+
+/**
+ * The replay-identity checks a contract is allowed to carry.
+ *
+ * The declaration is an exact set, not a switch. An author who pins a hash
+ * writes the id down, so a hash milestone that a later derivation adds cannot
+ * enter a published contract unnoticed.
+ */
+export interface EarnabilityDeclaration {
+  /** Milestone ids the author accepts as replay-identity pins. */
+  identityChecks?: readonly string[]
+}
+
+function earners(report: CalibrationReport, milestone: string): string {
+  return report.baselines.filter((b) => b.verified.includes(milestone)).map((b) => b.id).join(', ')
+}
+
+/**
+ * Every way a contract's earnable/unearnable split can be wrong, as message
+ * blocks. An empty array means the split is sound and declared.
+ */
+function earnabilityProblems(report: CalibrationReport, declaration: EarnabilityDeclaration): string[] {
+  const problems: string[] = []
+  const { total } = report.referenceScore
+  const declared = declaration.identityChecks
+  const share = total === 0 ? 0 : Math.round((report.unearnable.length / total) * 100)
+  const detail = report.unearnable.map((id) => `  unearnable: ${id} — ${report.unearnableReasons[id]}`)
+
+  if (declared === undefined) {
+    if (report.unearnable.length > 0) {
+      problems.push(
+        [
+          `contract has ${report.unearnable.length} of ${total} milestone(s) no policy can earn ` +
+            `(${report.earnable.length} earnable, ${share}% unearnable)`,
+          ...detail,
+          `  the reference itself scored ${formatMilestoneScore(report.referenceScore)}, so a score out of ` +
+            `${total} is not reachable by an independent policy`,
+          'A replay-identity check proves that a replay reproduced the recorded run. It is not progress.',
+          `Keep it and declare it — assertContractSeparates(report, { identityChecks: ` +
+            `[${report.unearnable.map((id) => `'${id}'`).join(', ')}] }) — or state the progression with a ` +
+            'state-path, save-path, frame-path, or log-contains check.',
+        ].join('\n'),
+      )
+    }
+  } else {
+    const accepted = new Set(declared)
+    const pinned = new Set(report.unearnable)
+    const undeclared = report.unearnable.filter((id) => !accepted.has(id))
+    const stale = [...accepted].filter((id) => !pinned.has(id))
+    if (undeclared.length > 0) {
+      problems.push(
+        [
+          `contract has ${undeclared.length} undeclared milestone(s) no policy can earn: ${undeclared.join(', ')}`,
+          ...detail,
+          'Declare every replay-identity check, or state the progression with a semantic check.',
+        ].join('\n'),
+      )
+    }
+    if (stale.length > 0) {
+      problems.push(
+        `identityChecks names ${stale.length} milestone(s) the contract does not pin: ${stale.join(', ')} — ` +
+          'the declaration is stale, and it would hide a hash milestone added later',
+      )
+    }
   }
+
+  if (report.unearnableReproduced.length > 0) {
+    problems.push(
+      [
+        `${report.unearnableReproduced.length} replay-identity milestone(s) were reproduced by a trivial baseline:`,
+        ...report.unearnableReproduced.map((id) => `  ${id} — reproduced by ${earners(report, id)}`),
+        'A hash that another trajectory reproduces identifies no run. Remove it or hash more state.',
+      ].join('\n'),
+    )
+  }
+  return problems
+}
+
+/**
+ * Fail closed on a contract whose milestones no policy can earn.
+ *
+ * Identity checks are correct and useful: replay attestation is exactly the
+ * claim that one run reproduced another's bytes. They must not be counted as
+ * achievements, and an undeclared one must not reach a published contract,
+ * because it puts points in the denominator that only the reference can score.
+ *
+ * Call this for a contract that is not meant to separate — a demonstration
+ * target — where `assertContractSeparates` would fail for the other reason.
+ */
+export function assertMilestonesEarnable(
+  report: CalibrationReport,
+  declaration: EarnabilityDeclaration = {},
+): void {
+  const problems = earnabilityProblems(report, declaration)
+  if (problems.length > 0) throw new Error(problems.join('\n'))
 }
 
 /**
- * Fail closed on a contract that a trivial policy satisfies.
+ * Fail closed on a contract that a trivial policy satisfies, and on one that
+ * carries milestones no policy can earn.
  *
  * Call this wherever a target is published. A contract that does not separate
  * still produces scores; those scores report how many frames elapsed, and
- * comparing two agents on them compares nothing.
+ * comparing two agents on them compares nothing. A contract whose points are
+ * partly unearnable still produces scores too, and every one of them is quoted
+ * against a denominator no agent can reach.
+ *
+ * Both failures are reported together, so one run of the gate names everything
+ * an author must fix.
  */
-export function assertContractSeparates(report: CalibrationReport): void {
-  if (report.separates) return
-  const best = report.baselines.filter((b) => b.verified.length === report.bestBaselineCount).map((b) => b.id)
-  const earners = (milestone: string): string =>
-    report.baselines.filter((b) => b.verified.includes(milestone)).map((b) => b.id).join(', ')
-  const lines = [
-    `contract does not separate: the reference verified ${report.reference.verified.length} milestone(s) ` +
-      `and the best trivial baseline verified ${report.bestBaselineCount} over ${report.turns} turns ` +
-      `(seed ${report.seed}, strongest: ${best.join(', ') || 'none'})`,
-    ...report.trivial.map((id) => `  trivial: ${id} — earned by ${earners(id)}`),
-    `  out of reach of every baseline: ${report.separating.join(', ') || 'nothing'}`,
-    'A derived contract is a hypothesis until it separates. Pin a progression a trivial policy cannot reach.',
-  ]
-  throw new Error(lines.join('\n'))
+export function assertContractSeparates(
+  report: CalibrationReport,
+  declaration: EarnabilityDeclaration = {},
+): void {
+  const problems = earnabilityProblems(report, declaration)
+  if (!report.separates) {
+    const best = report.baselines.filter((b) => b.verified.length === report.bestBaselineCount).map((b) => b.id)
+    problems.push(
+      [
+        `contract does not separate: the reference verified ${report.reference.verified.length} milestone(s) ` +
+          `and the best trivial baseline verified ${report.bestBaselineCount} over ${report.turns} turns ` +
+          `(seed ${report.seed}, strongest: ${best.join(', ') || 'none'})`,
+        `  on earnable milestones alone: reference ${report.referenceScore.earned}, ` +
+          `best baseline ${report.bestBaselineEarnedCount}, of ${report.referenceScore.earnable} earnable`,
+        ...report.trivial.map((id) => `  trivial: ${id} — earned by ${earners(report, id)}`),
+        `  earnable and out of reach of every baseline: ${report.separating.join(', ') || 'nothing'}`,
+        'A derived contract is a hypothesis until it separates. Pin a progression a trivial policy cannot reach.',
+      ].join('\n'),
+    )
+  }
+  if (problems.length > 0) throw new Error(problems.join('\n'))
 }

From bb7345e62db1524378931431e3172574c4ad7c74 Mon Sep 17 00:00:00 2001
From: Drew Stone <drewstone329@gmail.com>
Date: Sat, 22 Aug 2026 11:14:32 -0700
Subject: [PATCH 4/6] test(calibration): pin the earnable split on toys and
 both emulators

The regression measured on origin/main: a contract whose two milestones are
hashes over the whole input chain reports `separates: true` with every
baseline earning nothing. Unreachable read as hard. It now reports
`separates: false` and the gate names both milestones.

Also pinned: a mixed contract reports 2 of 3 earnable and refuses to ship
until the hash is declared; the packaged save-levels contract is 0 of 2
earnable through the dependency rule; screen-puzzle's two frame hashes are
reproduced by `constant:r`, so they identify nothing; and the three toy
contract hashes are unchanged, byte for byte, because the split is derived.

Real emulators, measured this run:
  ALE Breakout    4 of 6 earnable; reference 4 of 4, best baseline 0
  Libbet PyBoy    4 of 6 earnable; reference 3 of 4, best baseline 3
---
 ale.test.mts          |  71 +++++++++-
 calibration.test.mts  | 300 +++++++++++++++++++++++++++++++++++++++++-
 campaign.test.mts     |   7 +
 episode.test.mts      |  20 +++
 pyboy-libbet.test.mts |  34 ++++-
 5 files changed, 419 insertions(+), 13 deletions(-)

diff --git a/ale.test.mts b/ale.test.mts
index e4c0ef9..dc167fe 100644
--- a/ale.test.mts
+++ b/ale.test.mts
@@ -7,18 +7,19 @@
  * into a loud failure (that is how CI proves the job really executed).
  *
  * Battery: contract derivation across three evidence tiers, known-good
- * attestation, garbage rejection, graded partial credit, cross-process
- * determinism including the save-state hash, checkpoint round-trip,
- * unknown-input no-op, the observation image channel, and worker teardown.
- * Zero model spend.
+ * attestation, garbage rejection, graded partial credit, calibration with the
+ * earnable split, cross-process determinism including the save-state hash,
+ * checkpoint round-trip, unknown-input no-op, the observation image channel,
+ * and worker teardown. Zero model spend.
  */
 import { strict as assert } from 'node:assert'
 import { spawnSync } from 'node:child_process'
 import { createHash } from 'node:crypto'
 import { attestRun } from './attestation'
+import { assertContractSeparates, assertMilestonesEarnable, calibrateContract } from './calibration'
 import { logFrom, observationOf } from './runtime'
 import { decodePng, unscale } from './test-png.mts'
-import { validateContract } from './schema'
+import { contractEarnability, formatMilestoneScore, scoreMilestones, validateContract } from './schema'
 import { AleRpc } from './adapters/ale-rpc'
 import { bundledReference, makeAle, type Ale, type AleState } from './adapters/ale'
 
@@ -109,6 +110,66 @@ if (!pythonHasAle()) {
     assert.deepEqual(partial.verified, ['score-opened', 'frame-at-first-score', 'save-at-first-score', 'score-tier-2'])
     assert.deepEqual(partial.reasons, ['claimed-not-reproduced:score-tier-4,life-lost'])
 
+    // The reference verified all six milestones, and only four of them are
+    // progress. `frame-at-first-score` and `save-at-first-score` hash the exact
+    // screen and save state this trajectory produced, so reaching them means
+    // reproducing this run rather than playing Breakout.
+    assert.deepEqual(contractEarnability(adapter.contract), {
+      earnable: ['score-opened', 'score-tier-2', 'score-tier-4', 'life-lost'],
+      unearnable: ['frame-at-first-score', 'save-at-first-score'],
+      reasons: {
+        'frame-at-first-score': "its frame-hash check pins the reference run's exact bytes",
+        'save-at-first-score': "its save-hash check pins the reference run's exact bytes",
+      },
+    })
+    assert.deepEqual(good.earned, ['score-opened', 'score-tier-2', 'score-tier-4', 'life-lost'])
+    assert.deepEqual(good.score, { verified: 6, earned: 4, earnable: 4, total: 6 })
+    assert.equal(formatMilestoneScore(partial.score), '2 of 4 earnable (4 of 6 verified, 2 replay-identity)')
+
+    // The measured defect this split exists for. An authored policy and a
+    // hand-written ball tracker each verified score-opened, score-tier-2 and
+    // life-lost on this contract, and neither reproduced either hash. Both
+    // read as three of six. Three of four is the honest statement, and it is
+    // the whole of what an independent policy can score.
+    const played = ['score-opened', 'score-tier-2', 'life-lost']
+    assert.deepEqual(scoreMilestones(adapter.contract, played), { verified: 3, earned: 3, earnable: 4, total: 6 })
+    assert.equal(formatMilestoneScore(scoreMilestones(adapter.contract, played)),
+      '3 of 4 earnable (3 of 6 verified, 2 replay-identity)')
+
+    // Calibration on the real emulator. Every baseline plays the reference's
+    // 210 turns, so the comparison is length-matched.
+    const calibration = calibrateContract(adapter.game, adapter.contract, {
+      reference: adapter.reference,
+      vocabulary: adapter.inputs,
+      seed: adapter.seed,
+    })
+    for (const outcome of [calibration.reference, ...calibration.baselines]) {
+      console.log(`  ${outcome.id.padEnd(32)} ${String(outcome.verified.length).padStart(2)}  ${outcome.verdict}  ${outcome.verified.join(',') || '-'}`)
+    }
+    assert.deepEqual(calibration.earnable, ['score-opened', 'score-tier-2', 'score-tier-4', 'life-lost'])
+    assert.deepEqual(calibration.unearnable, ['frame-at-first-score', 'save-at-first-score'])
+    assert.deepEqual(calibration.referenceScore, { verified: 6, earned: 4, earnable: 4, total: 6 })
+    // No trivial policy reproduced either hash, so both really are identity
+    // checks on this substrate and not low-entropy channels.
+    assert.deepEqual(calibration.unearnableReproduced, [])
+    // Undeclared, the contract cannot ship, whatever the separation verdict.
+    assert.throws(() => assertMilestonesEarnable(calibration), /2 of 6 milestone\(s\) no policy can earn/u)
+    const declared = { identityChecks: ['frame-at-first-score', 'save-at-first-score'] }
+    assertMilestonesEarnable(calibration, declared)
+    // Whether Breakout separates is a fact about the ROM, not about this
+    // change: assert only that the earnable comparison is the one being made.
+    assert.equal(
+      calibration.separates,
+      calibration.separating.length > 0 && calibration.referenceScore.earned > calibration.bestBaselineEarnedCount,
+    )
+    if (calibration.separates) assertContractSeparates(calibration, declared)
+    else assert.throws(() => assertContractSeparates(calibration, declared), /does not separate/u)
+    console.log(
+      `ale: calibration — reference ${formatMilestoneScore(calibration.referenceScore)}, ` +
+      `best trivial baseline ${calibration.bestBaselineEarnedCount} earnable over ${calibration.turns} turns, ` +
+      `separating=${calibration.separating.join(',') || 'nothing'}, separates=${calibration.separates}`,
+    )
+
     // Determinism: two replays in this worker and one in a freshly spawned
     // worker must agree on every frame hash, every save-state hash, and every
     // privileged variable. Cross-process is the load-bearing case, because a
diff --git a/calibration.test.mts b/calibration.test.mts
index 97ae1d0..43f05c8 100644
--- a/calibration.test.mts
+++ b/calibration.test.mts
@@ -7,17 +7,35 @@
  * pins a channel that moves whenever one button is pressed — the Libbet shape —
  * so it must not. The native-2048 adapter then runs the same gate on a real
  * out-of-process game.
+ *
+ * The second half covers the other way a contract fails to measure: a milestone
+ * NO policy can earn, because a hash check pins the reference run's exact
+ * bytes. The trace game makes that case unambiguous, because its hash covers
+ * the whole input sequence.
  */
 import { strict as assert } from 'node:assert'
+import { createHash } from 'node:crypto'
 import { deriveContract } from './authoring'
 import {
   assertContractSeparates,
+  assertMilestonesEarnable,
   calibrateContract,
   trivialBaselines,
   UNKNOWN_BASELINE_WORD,
 } from './calibration'
 import type { Game } from './runtime'
+import {
+  canonicalContractJson,
+  contractEarnability,
+  contractHash,
+  earnedMilestones,
+  formatMilestoneScore,
+  scoreMilestones,
+} from './schema'
 import { makeNative2048, NATIVE_2048_INPUTS, NATIVE_2048_REFERENCE } from './adapters/native-2048'
+import { engineCrawlerContract } from './adapters/engine-crawler'
+import { saveLevels, saveLevelsContract, SAVE_LEVELS_REFERENCE } from './adapters/save-levels'
+import { screenPuzzle, screenPuzzleContract, SCREEN_PUZZLE_REFERENCE } from './adapters/screen-puzzle'
 
 // --- a game that needs an exact sequence ------------------------------------
 
@@ -25,25 +43,34 @@ const LOCK_VOCABULARY = ['a', 'b', 'c', 'd', 'e', 'f']
 const LOCK_CODE = ['c', 'a', 'f', 'b', 'e', 'd']
 const LOCK_REFERENCE = ['b', 'a', ...LOCK_CODE]
 
+const chain = (previous: string, input: string): string =>
+  createHash('sha256').update(`${previous}:${input}`).digest('hex')
+
 interface LockState {
   progress: number
   opened: number
   steps: number
+  /** Hash chain over the inputs, so a frame hash pins one trajectory. */
+  trace: string
 }
 
 const comboLock: Game<LockState> = {
   id: 'combo-lock',
-  init: () => ({ progress: 0, opened: 0, steps: 0 }),
+  init: () => ({ progress: 0, opened: 0, steps: 0, trace: chain('boot', '') }),
   step: (s, input) => {
     const steps = s.steps + 1
-    if (s.opened === 1) return { ...s, steps }
+    const trace = chain(s.trace, input)
+    if (s.opened === 1) return { ...s, steps, trace }
     const advances = input === LOCK_CODE[s.progress]
     const restarts = !advances && input === LOCK_CODE[0]
     const progress = advances ? s.progress + 1 : restarts ? 1 : 0
-    return { progress, opened: progress === LOCK_CODE.length ? 1 : 0, steps }
+    return { progress, opened: progress === LOCK_CODE.length ? 1 : 0, steps, trace }
   },
   frame: (s) => `steps ${s.steps} · the lock is ${s.opened === 1 ? 'open' : 'shut'}`,
-  evidence: (s) => ({ engineState: { progress: s.progress, opened: s.opened, steps: s.steps } }),
+  evidence: (s) => ({
+    engineState: { progress: s.progress, opened: s.opened, steps: s.steps },
+    frameHash: s.trace,
+  }),
 }
 
 const lockContract = deriveContract(comboLock, 0, [...LOCK_REFERENCE], [
@@ -259,4 +286,267 @@ try {
   adapter.dispose()
 }
 
-console.log('playproof calibration: separating and non-separating contracts, policy determinism, edge cases OK')
+// --- earnable milestones vs replay-identity checks ---------------------------
+
+// (f) an all-achievement contract is unchanged: every milestone is earnable,
+// nothing is declared, and the gate passes exactly as it did before.
+{
+  const report = calibrateContract(comboLock, lockContract, {
+    reference: LOCK_REFERENCE,
+    vocabulary: LOCK_VOCABULARY,
+  })
+  assert.deepEqual(report.earnable, ['moved', 'lock-opened'])
+  assert.deepEqual(report.unearnable, [])
+  assert.deepEqual(report.unearnableReasons, {})
+  assert.deepEqual(report.unearnableReproduced, [])
+  assert.deepEqual(report.referenceScore, { verified: 2, earned: 2, earnable: 2, total: 2 })
+  assert.equal(formatMilestoneScore(report.referenceScore), '2 of 2 earnable')
+  assert.equal(report.bestBaselineEarnedCount, 1)
+  assert.equal(report.separates, true)
+  assertContractSeparates(report)
+  assertMilestonesEarnable(report)
+}
+
+// (g) a contract mixing both kinds reports the right earnable count, and the
+// identity check must be declared before the contract ships.
+//
+// `frame-at-open` is a hash over the whole input chain, so only a replay of the
+// reference reproduces it. It is a correct attestation check and it stays in
+// the contract; it is not a third point an agent can score.
+const lockIdentityContract = deriveContract(comboLock, 0, [...LOCK_REFERENCE], [
+  {
+    id: 'moved',
+    tier: 'engine-state',
+    glitchClass: 'legal',
+    when: (e) => (e.engineState?.steps ?? 0) >= 1,
+    sample: (e) => ({ kind: 'state-path', path: 'steps', op: '>=', value: e.engineState?.steps ?? 1 }),
+  },
+  {
+    id: 'lock-opened',
+    tier: 'engine-state',
+    glitchClass: 'legal',
+    requires: ['moved'],
+    when: (e) => (e.engineState?.opened ?? 0) >= 1,
+    sample: (e) => ({ kind: 'state-path', path: 'opened', op: '>=', value: e.engineState?.opened ?? 1 }),
+  },
+  {
+    id: 'frame-at-open',
+    tier: 'screen-frame',
+    glitchClass: 'legal',
+    requires: ['lock-opened'],
+    when: (e) => (e.engineState?.opened ?? 0) >= 1,
+    sample: (e) => ({ kind: 'frame-hash', hash: e.frameHash ?? '' }),
+  },
+])
+
+{
+  const report = calibrateContract(comboLock, lockIdentityContract, {
+    reference: LOCK_REFERENCE,
+    vocabulary: LOCK_VOCABULARY,
+  })
+  assert.deepEqual(report.reference.verified, ['moved', 'lock-opened', 'frame-at-open'])
+  assert.deepEqual(report.earnable, ['moved', 'lock-opened'])
+  assert.deepEqual(report.unearnable, ['frame-at-open'])
+  assert.match(report.unearnableReasons['frame-at-open'] ?? '', /frame-hash check pins the reference run's exact bytes/u)
+  assert.deepEqual(report.unearnableReproduced, [])
+  // The reference scored three of three, and only two of them were earnable.
+  assert.deepEqual(report.referenceScore, { verified: 3, earned: 2, earnable: 2, total: 3 })
+  assert.equal(formatMilestoneScore(report.referenceScore), '2 of 2 earnable (3 of 3 verified, 1 replay-identity)')
+  // The identity check is out of reach of every baseline, and that is not
+  // separation: `separating` names the earnable milestone only.
+  assert.deepEqual(report.separating, ['lock-opened'])
+  assert.equal(report.separates, true)
+
+  // Shipping it silently is impossible: the gate refuses until the author
+  // writes the id down.
+  assert.throws(
+    () => assertContractSeparates(report),
+    (error: unknown) => {
+      const message = (error as Error).message
+      assert.match(message, /1 of 3 milestone\(s\) no policy can earn/u)
+      assert.match(message, /2 earnable, 33% unearnable/u)
+      assert.match(message, /frame-at-open/u)
+      assert.match(message, /identityChecks: \['frame-at-open'\]/u)
+      assert.doesNotMatch(message, /does not separate/u)
+      return true
+    },
+  )
+  assert.throws(() => assertMilestonesEarnable(report), /no policy can earn/u)
+
+  // Declared, the same contract passes both gates.
+  assertContractSeparates(report, { identityChecks: ['frame-at-open'] })
+  assertMilestonesEarnable(report, { identityChecks: ['frame-at-open'] })
+
+  // A declaration is an exact set. A missing id and a stale id both fail, so a
+  // hash milestone added by a later derivation cannot hide behind it.
+  assert.throws(() => assertContractSeparates(report, { identityChecks: [] }), /1 undeclared milestone\(s\)/u)
+  assert.throws(
+    () => assertContractSeparates(report, { identityChecks: ['frame-at-open', 'lock-opened'] }),
+    /the contract does not pin: lock-opened/u,
+  )
+
+  // Attestation reports the same split for one run.
+  const score = scoreMilestones(lockIdentityContract, report.reference.verified)
+  assert.deepEqual(score, report.referenceScore)
+  assert.deepEqual(earnedMilestones(lockIdentityContract, report.reference.verified), ['moved', 'lock-opened'])
+}
+
+// (h) an all-identity contract reports 0 earnable, and calibration refuses it.
+//
+// This is the regression. Every baseline earns nothing, so before earnability
+// existed both milestones landed in `separating` and the report called the
+// contract a benchmark — one on which no policy but a replay of the reference
+// can ever score a point.
+const TRACE_VOCABULARY = ['a', 'b', 'c']
+const TRACE_REFERENCE = ['a', 'b', 'b', 'c', 'a']
+
+interface TraceState {
+  steps: number
+  trace: string
+}
+
+const traceGame: Game<TraceState> = {
+  id: 'trace-hash',
+  init: () => ({ steps: 0, trace: chain('boot', '') }),
+  step: (s, input) => ({ steps: s.steps + 1, trace: chain(s.trace, input) }),
+  frame: (s) => `steps ${s.steps}`,
+  evidence: (s) => ({
+    engineState: { steps: s.steps },
+    frameHash: s.trace,
+    saveBlobHash: chain(s.trace, 'save'),
+  }),
+}
+
+const traceContract = deriveContract(traceGame, 0, [...TRACE_REFERENCE], [
+  {
+    afterInputs: 3,
+    id: 'frame-at-three',
+    tier: 'screen-frame',
+    glitchClass: 'legal',
+    sample: (e) => ({ kind: 'frame-hash', hash: e.frameHash ?? '' }),
+  },
+  {
+    afterInputs: 5,
+    id: 'save-at-five',
+    tier: 'save-file',
+    glitchClass: 'legal',
+    requires: ['frame-at-three'],
+    sample: (e) => ({ kind: 'save-hash', hash: e.saveBlobHash ?? '' }),
+  },
+])
+
+{
+  const report = calibrateContract(traceGame, traceContract, {
+    reference: TRACE_REFERENCE,
+    vocabulary: TRACE_VOCABULARY,
+  })
+  assert.deepEqual(report.reference.verified, ['frame-at-three', 'save-at-five'])
+  assert.deepEqual(report.earnable, [])
+  assert.deepEqual(report.unearnable, ['frame-at-three', 'save-at-five'])
+  assert.deepEqual(report.referenceScore, { verified: 2, earned: 0, earnable: 0, total: 2 })
+  assert.equal(formatMilestoneScore(report.referenceScore), '0 of 0 earnable (2 of 2 verified, 2 replay-identity)')
+
+  // No baseline earned anything at all, and the contract still does not
+  // separate. That combination is the whole point: unreachable is not hard.
+  assert.deepEqual(report.trivial, [])
+  assert.ok(report.baselines.every((b) => b.verified.length === 0))
+  assert.deepEqual(report.separating, [])
+  assert.equal(report.bestBaselineEarnedCount, 0)
+  assert.equal(report.separates, false)
+  assert.throws(
+    () => assertContractSeparates(report),
+    (error: unknown) => {
+      const message = (error as Error).message
+      assert.match(message, /2 of 2 milestone\(s\) no policy can earn/u)
+      assert.match(message, /0 earnable, 100% unearnable/u)
+      assert.match(message, /the reference itself scored 0 of 0 earnable \(2 of 2 verified, 2 replay-identity\)/u)
+      assert.match(message, /does not separate/u)
+      return true
+    },
+  )
+  // Declaring both is honest about the checks and still not a benchmark.
+  assert.throws(
+    () => assertContractSeparates(report, { identityChecks: ['frame-at-three', 'save-at-five'] }),
+    (error: unknown) => {
+      const message = (error as Error).message
+      assert.match(message, /does not separate/u)
+      assert.doesNotMatch(message, /no policy can earn/u)
+      return true
+    },
+  )
+  assertMilestonesEarnable(report, { identityChecks: ['frame-at-three', 'save-at-five'] })
+
+  // A milestone gated behind an identity check is unearnable too, whatever its
+  // own check kind: the tracker admits it only after its prerequisite passed.
+  const gated = contractEarnability({
+    ...traceContract,
+    milestones: [
+      ...traceContract.milestones,
+      {
+        id: 'steps-after-save',
+        tier: 'engine-state',
+        requires: ['save-at-five'],
+        glitchClass: 'legal',
+        check: { kind: 'state-path', path: 'steps', op: '>=', value: 5 },
+      },
+    ],
+  })
+  assert.deepEqual(gated.earnable, [])
+  assert.equal(gated.reasons['steps-after-save'], 'requires save-at-five, which no independent policy can earn')
+}
+
+// (i) the same split on the two packaged toy adapters, measured rather than
+// assumed. Both are demonstration targets for the evidence tiers, and both
+// turn out to carry no earnable milestone at all.
+{
+  const levels = contractEarnability(saveLevelsContract())
+  assert.deepEqual(levels.earnable, [])
+  assert.deepEqual(levels.unearnable, ['level-2-saved', 'level-2-logged'])
+  // The log-event check is semantic; it is unearnable only through its
+  // prerequisite, which is the dependency rule doing real work.
+  assert.equal(levels.reasons['level-2-logged'], 'requires level-2-saved, which no independent policy can earn')
+  const report = calibrateContract(saveLevels, saveLevelsContract(), {
+    reference: SAVE_LEVELS_REFERENCE,
+    vocabulary: ['clear', 'grind'],
+  })
+  assert.equal(report.separates, false)
+  assert.deepEqual(report.referenceScore, { verified: 2, earned: 0, earnable: 0, total: 2 })
+
+  // screen-puzzle renders from one coordinate, so `constant:r` walks to the
+  // same square and reproduces both pinned frames. A hash another trajectory
+  // reproduces identifies no run, and the gate says so even when the author
+  // declares it.
+  const puzzle = calibrateContract(screenPuzzle, screenPuzzleContract(), {
+    reference: SCREEN_PUZZLE_REFERENCE,
+    vocabulary: ['l', 'r'],
+  })
+  assert.deepEqual(puzzle.earnable, [])
+  assert.deepEqual(puzzle.unearnableReproduced, ['midway-frame', 'east-gate-frame'])
+  assert.throws(
+    () => assertMilestonesEarnable(puzzle, { identityChecks: ['midway-frame', 'east-gate-frame'] }),
+    (error: unknown) => {
+      const message = (error as Error).message
+      assert.match(message, /2 replay-identity milestone\(s\) were reproduced by a trivial baseline/u)
+      assert.match(message, /midway-frame — reproduced by constant:r/u)
+      return true
+    },
+  )
+}
+
+// (j) the split is derived, so every existing contract keeps its bytes. These
+// hashes were recorded before earnability existed; a contract whose hash moves
+// invalidates every artifact that pinned it.
+{
+  assert.equal(contractHash(engineCrawlerContract()), '29a9ff9f3bb296a898493589d6bcdb539b5be15f7f53931de7f3725916274426')
+  assert.equal(contractHash(saveLevelsContract()), '6c330131dd004db07d4ee3948af75bdf4d128569c734df6d2d95e6c2f7684498')
+  assert.equal(contractHash(screenPuzzleContract()), 'e9b818f754bd5de89277df0797bf212dad3be079453cf47065ee2be8e89da37f')
+  for (const contract of [engineCrawlerContract(), saveLevelsContract(), screenPuzzleContract()]) {
+    const canonical = JSON.parse(canonicalContractJson(contract)) as { milestones: Record<string, unknown>[] }
+    for (const milestone of canonical.milestones) {
+      assert.deepEqual(Object.keys(milestone), ['id', 'tier', 'requires', 'check', 'glitchClass'],
+        'a milestone gained a serialized field; every published contract hash would move')
+    }
+  }
+}
+
+console.log('playproof calibration: separating and non-separating contracts, earnable/identity split, policy determinism, edge cases OK')
diff --git a/campaign.test.mts b/campaign.test.mts
index d038dc9..b651c7a 100644
--- a/campaign.test.mts
+++ b/campaign.test.mts
@@ -167,6 +167,13 @@ try {
     assert.ok(first.recentHistory.length <= 8)
     assert.ok(first.lastFrame.length > 0)
     assert.ok(first.verifiedSoFar.includes('first-legal-move'))
+    // An analyst reads progress against the earnable denominator, not the raw
+    // milestone count. Every 2048 milestone is a semantic check, so the two
+    // agree here, and the field says so instead of leaving it to be assumed.
+    assert.equal(first.scoreSoFar.verified, first.verifiedSoFar.length)
+    assert.equal(first.scoreSoFar.earned, first.verifiedSoFar.length)
+    assert.equal(first.scoreSoFar.earnable, contract.milestones.length)
+    assert.equal(first.scoreSoFar.total, contract.milestones.length)
     assert.equal(first.ledger.decisions.length, 2)
   }
 
diff --git a/episode.test.mts b/episode.test.mts
index 6b985f8..78844ed 100644
--- a/episode.test.mts
+++ b/episode.test.mts
@@ -4,7 +4,9 @@
  */
 import { strict as assert } from 'node:assert'
 import { playEpisode, scriptedDriver } from './episode'
+import { formatMilestoneScore } from './schema'
 import { engineCrawler, engineCrawlerContract, ENGINE_CRAWLER_REFERENCE } from './adapters/engine-crawler'
+import { saveLevels, saveLevelsContract, SAVE_LEVELS_REFERENCE } from './adapters/save-levels'
 
 {
   // scripted agent solves the crawler; all milestones verify
@@ -12,6 +14,11 @@ import { engineCrawler, engineCrawlerContract, ENGINE_CRAWLER_REFERENCE } from '
   assert.equal(record.verdict, 'clean')
   assert.deepEqual(record.verified, ['hp-untouched', 'room-1', 'room-2-plus', 'room-3'])
   assert.equal(record.spentUsd, 0)
+  // Every crawler milestone is an engine-state threshold, so the earned set is
+  // the verified set and the score has no hidden denominator.
+  assert.deepEqual(record.earned, record.verified)
+  assert.deepEqual(record.score, { verified: 4, earned: 4, earnable: 4, total: 4 })
+  assert.equal(formatMilestoneScore(record.score), '4 of 4 earnable')
 }
 
 {
@@ -28,6 +35,19 @@ import { engineCrawler, engineCrawlerContract, ENGINE_CRAWLER_REFERENCE } from '
   const { record } = await playEpisode(engineCrawler, engineCrawlerContract(), scriptedDriver(ENGINE_CRAWLER_REFERENCE), 1, 2)
   assert.equal(record.turns, 2)
   assert.deepEqual(record.verified, ['hp-untouched', 'room-1'])
+  assert.deepEqual(record.score, { verified: 2, earned: 2, earnable: 4, total: 4 })
+}
+
+// A run on a contract whose milestones are all hash checks verifies them and
+// earns none of them. The record reports both, so nobody quotes the run as two
+// out of two.
+{
+  const { record } = await playEpisode(saveLevels, saveLevelsContract(), scriptedDriver(SAVE_LEVELS_REFERENCE), 1, 8)
+  assert.equal(record.verdict, 'clean')
+  assert.deepEqual(record.verified, ['level-2-saved', 'level-2-logged'])
+  assert.deepEqual(record.earned, [])
+  assert.deepEqual(record.score, { verified: 2, earned: 0, earnable: 0, total: 2 })
+  assert.equal(formatMilestoneScore(record.score), '0 of 0 earnable (2 of 2 verified, 2 replay-identity)')
 }
 
 // A scripted driver is positioned by the harness turn, so a driver created in a
diff --git a/pyboy-libbet.test.mts b/pyboy-libbet.test.mts
index bb4e24b..fee494a 100644
--- a/pyboy-libbet.test.mts
+++ b/pyboy-libbet.test.mts
@@ -20,9 +20,9 @@ import { existsSync, readFileSync } from 'node:fs'
 import { fileURLToPath } from 'node:url'
 import { autoMarks, loadDiscovery, makePyBoyGeneric } from './adapters/pyboy-generic'
 import { attestRun } from './attestation'
-import { assertContractSeparates, calibrateContract, UNKNOWN_BASELINE_WORD } from './calibration'
+import { assertContractSeparates, assertMilestonesEarnable, calibrateContract, UNKNOWN_BASELINE_WORD } from './calibration'
 import { logFrom, observationOf } from './runtime'
-import { validateContract } from './schema'
+import { contractEarnability, formatMilestoneScore, validateContract } from './schema'
 import { decodePng, unscale } from './test-png.mts'
 
 /** The Game Boy pad, matching `BUTTONS` in pyboy/tetris.py. */
@@ -172,7 +172,35 @@ try {
   // word earns nothing, so this is not a pure function of elapsed frames.
   assert.deepEqual(report.baselines.find((b) => b.id === `constant:${UNKNOWN_BASELINE_WORD}`)?.verified, [])
 
-  console.log(`pyboy-libbet: calibration regression — reference ${report.reference.verified.length} milestones, best trivial baseline ${report.bestBaselineCount} over ${report.turns} turns, separates=${report.separates} OK`)
+  // (e2) The earnable split on the derived contract. `autoMarks` anchors one
+  // save-hash and one frame-hash milestone at the confirmed channel's first
+  // progression, so two of the six points can only be scored by a replay of
+  // this exploration trajectory.
+  const earnability = contractEarnability(adapter.contract)
+  assert.deepEqual(earnability.unearnable, ['state-at-first-progression', 'frame-at-first-progression'])
+  assert.equal(earnability.earnable.length, 4)
+  assert.ok(earnability.earnable.every((id) => id.endsWith('-progressed')))
+  assert.deepEqual(report.earnable, earnability.earnable)
+  assert.deepEqual(report.referenceScore, { verified: 3, earned: 3, earnable: 4, total: 6 })
+  assert.equal(formatMilestoneScore(report.referenceScore), '3 of 4 earnable (3 of 6 verified, 2 replay-identity)')
+  // No baseline reproduced either hash, so both are real identity checks here.
+  assert.deepEqual(report.unearnableReproduced, [])
+  // Undeclared, the contract fails the earnability gate on its own, which is a
+  // second, independent reason this target must not be published as a score.
+  assert.throws(
+    () => assertMilestonesEarnable(report),
+    (error: unknown) => {
+      const message = (error as Error).message
+      assert.match(message, /2 of 6 milestone\(s\) no policy can earn/u)
+      assert.match(message, /4 earnable, 33% unearnable/u)
+      return true
+    },
+  )
+  const declared = { identityChecks: ['state-at-first-progression', 'frame-at-first-progression'] }
+  assertMilestonesEarnable(report, declared)
+  assert.throws(() => assertContractSeparates(report, declared), /does not separate/u)
+
+  console.log(`pyboy-libbet: calibration regression — reference ${formatMilestoneScore(report.referenceScore)}, best trivial baseline ${report.bestBaselineCount} verified / ${report.bestBaselineEarnedCount} earnable over ${report.turns} turns, separates=${report.separates} OK`)
 
   // (f) The observation image channel on the real emulator. PyBoy's own
   // screen.image needs Pillow, which is absent here — the boot logs say so —

From ca599f9e19896e474b59d40db296e064854a5671 Mon Sep 17 00:00:00 2001
From: Drew Stone <drewstone329@gmail.com>
Date: Sat, 22 Aug 2026 11:16:43 -0700
Subject: [PATCH 5/6] docs(calibration): state the earnable/identity split and
 its measurements

README and docs/adapters.md name the two kinds of statement a contract
holds, show `contractEarnability` and the exact `identityChecks`
declaration, and carry the measured Breakout and Libbet splits. CHANGELOG
0.6.0 records the defect, the gate defect behind it, and what replay
attestation keeps.
---
 CHANGELOG.md     | 35 ++++++++++++++++++++++++++++
 README.md        | 59 ++++++++++++++++++++++++++++++++++++++++++++++--
 docs/adapters.md | 30 ++++++++++++++++++++++++
 3 files changed, 122 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cd65248..4da5232 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,41 @@
 
 All notable changes to Playproof are documented here.
 
+## 0.6.0
+
+### Earnable milestones are separated from replay-identity checks
+
+- **The defect.** `deriveContract` samples frame hashes and save hashes from the reference trajectory and emits them as milestones. On ALE Breakout the derived contract has six milestones, and two of them — `frame-at-first-score` and `save-at-first-score` — pin the exact bytes the reference produced. No independent policy earns them. An authored policy that verified `score-opened`, `score-tier-2`, and `life-lost` read as 3 of 6; a hand-written ball tracker reached the same three. A third of that contract's points were reachable only by a replay of the reference, and every score quoted from it carried that denominator.
+- A milestone's role is now derived from its check kind. `save-hash` and `frame-hash` are replay-identity; `state-path`, `save-path`, `frame-path`, and `log-contains` are achievements. Nothing is stored on the milestone, so every existing contract keeps its bytes and its hash, and no author can forget to set it.
+- `contractEarnability(contract)` follows `requires` as well. An achievement gated behind a hash is unreachable too, because `MilestoneTracker` admits a milestone only after every prerequisite has passed. The packaged `save-levels` contract is the case in the repo: its `log-contains` milestone is semantic and still unearnable, through its save-hash prerequisite.
+- `Attestation` and `EpisodeRecord` keep `verified` unchanged and gain `earned` and `score`. `MilestoneScore` is `{ verified, earned, earnable, total }`, and `formatMilestoneScore` writes it as `3 of 4 earnable (3 of 6 verified, 2 replay-identity)`. A campaign segment report gains `scoreSoFar`, so an analyst reads progress against the earnable denominator mid-run.
+- The campaign ledger is unchanged. Its `verified` list and the contract it pins by hash reproduce the score through `scoreMilestones`, so a ledger written by 0.5.0 still loads.
+
+### Calibration refuses a contract with points no policy can score
+
+- **A second defect, in the gate itself.** A replay-identity milestone is never earned by a trivial baseline, so it landed in `separating` and the separation test read it as the contract's strongest evidence. Measured on 0.5.0: a contract whose two milestones are hashes over the whole input chain reports `separates: true` with every baseline earning nothing. Unreachable read as hard.
+- `CalibrationReport.separating` now holds earnable milestones only, and `separates` compares earnable counts through the new `bestBaselineEarnedCount`. A baseline that beats the reference on real progress can no longer be outvoted by hashes the reference reproduces by construction.
+- The report gains `earnable`, `unearnable`, `unearnableReasons`, `unearnableReproduced`, and `referenceScore`.
+- `assertContractSeparates(report, { identityChecks })` takes an exact declaration of the identity checks the author accepts. An undeclared one, a stale id, and an identity hash that a trivial baseline reproduced all fail the gate, so a hash milestone added by a later derivation cannot enter a published contract unnoticed. `assertMilestonesEarnable` runs the same check alone, for a target that is not meant to separate.
+- Identity checks are not removed and not discouraged. Replay attestation is exactly the claim that one run reproduced another's bytes. The fix is to stop counting them as achievements.
+
+### Measured
+
+| Contract | Milestones | Earnable | Reference | Best trivial baseline |
+|---|---|---|---|---|
+| ALE Breakout, 210 turns, seed 0 | 6 | 4 | 4 of 4 earnable (6 of 6 verified) | 0 earnable |
+| Libbet through `pyboy-generic`, 70 turns, seed 0 | 6 | 4 | 3 of 4 earnable (3 of 6 verified) | 3 earnable |
+| `save-levels` toy | 2 | 0 | 0 of 0 earnable (2 of 2 verified) | 0 earnable |
+| `screen-puzzle` toy | 2 | 0 | 0 of 0 earnable (2 of 2 verified) | 2 verified |
+
+- Breakout separates on its earnable milestones and its scores were quoted out of the wrong denominator. Libbet still does not separate, and its four earnable milestones are exactly the ones a constant `a` press already earns.
+- `screen-puzzle` renders from one coordinate, so `constant:r` walks to the same square and reproduces both pinned frames. A hash another trajectory reproduces identifies no run, and the gate now says so.
+
+### Replay attestation is unaffected
+
+- `verified` keeps its meaning and its contents. The three packaged toy contract hashes are byte-identical across the change, and `calibration.test.mts` pins them, together with the serialized milestone key set, so a contract that gains a field fails the build.
+- `ale.test.mts` and `pyboy-libbet.test.mts` verify the same milestone ids on the same runs as before, and now also report the split.
+
 ## 0.5.0
 
 ### The observation channel
diff --git a/README.md b/README.md
index caefafb..6b7f563 100644
--- a/README.md
+++ b/README.md
@@ -270,6 +270,33 @@ Use semantic checks such as `score >= 10` for progression. Exact hashes identify
 
 Dependencies between milestones form a declared partial order. A later achievement cannot verify before its prerequisites, even when its raw condition already holds.
 
+### Earnable milestones and replay-identity checks
+
+A contract holds two kinds of statement, and one score cannot carry both.
+
+- An **achievement** is a threshold, a normalized field, or an event. Two different valid trajectories both satisfy it, so an independent policy earns it by playing.
+- A **replay-identity check** is a hash over the exact bytes one recorded run produced. It proves that a replay reproduced that run, and no independent policy earns it.
+
+The role is derived from the check kind, so no contract changes and no author has to remember to set it.
+`save-hash` and `frame-hash` are identity; `state-path`, `save-path`, `frame-path`, and `log-contains` are achievements.
+`requires` is followed: an achievement gated behind a hash is unreachable too, because a milestone verifies only after every prerequisite has.
+
+```ts
+import { contractEarnability, formatMilestoneScore } from '@tangle-network/playproof'
+
+contractEarnability(contract)
+// { earnable: ['score-opened', 'score-tier-2', 'score-tier-4', 'life-lost'],
+//   unearnable: ['frame-at-first-score', 'save-at-first-score'],
+//   reasons: { 'frame-at-first-score': "its frame-hash check pins the reference run's exact bytes", … } }
+
+formatMilestoneScore(record.score) // '3 of 4 earnable (3 of 6 verified, 2 replay-identity)'
+```
+
+`Attestation` and `EpisodeRecord` carry `verified` unchanged, plus `earned` and `score`.
+A campaign segment report carries `scoreSoFar`.
+Report a run as earned over earnable.
+Identity checks stay in the contract and stay in `verified`; they are what replay attestation proves.
+
 ## Calibration: does the contract separate?
 
 A milestone contract says which progressions count.
@@ -292,10 +319,38 @@ assertContractSeparates(report)
 `calibrateContract` replays the reference and a suite of trivial policies through the same attestation path: one constant policy per input word, a word the game cannot interpret, a round-robin cycle over the vocabulary, and a seeded pseudo-random walk over it.
 Every policy is deterministic in the seed, so a report reproduces from one number.
 
-The report names `separating` (milestones no baseline earned), `trivial` (milestones at least one baseline earned), and `bestBaselineCount`.
-`separates` is true only when something is out of reach of every baseline **and** the reference verifies strictly more milestones than the strongest baseline.
+The report names `separating` (earnable milestones no baseline earned), `trivial` (milestones at least one baseline earned), `earnable` and `unearnable`, and both baseline counts.
+`separates` is true only when an **earnable** milestone is out of reach of every baseline **and** the reference earns strictly more earnable milestones than the strongest baseline.
 `assertContractSeparates` throws otherwise, and the message names every trivial milestone with the baseline that earned it.
 
+### The gate also refuses points no policy can score
+
+A replay-identity check is never earned by a baseline, so the separation test alone reads it as the contract's strongest evidence.
+It is the opposite: a milestone out of reach of every policy measures nothing, and it inflates the denominator of every score quoted from the contract.
+
+Measured on ALE Breakout: the derived contract has six milestones and two of them are hashes of the screen and the save state at the first point.
+An authored policy that verified `score-opened`, `score-tier-2`, and `life-lost` read as 3 of 6.
+It could never have reached 6.
+The honest number is 3 of 4.
+
+```ts
+assertContractSeparates(report, {
+  identityChecks: ['frame-at-first-score', 'save-at-first-score'],
+})
+```
+
+The declaration is an exact set, not a switch: an undeclared hash milestone, a stale id, and an identity hash that a trivial baseline reproduced all fail the gate.
+A hash milestone that a later derivation adds therefore cannot enter a published contract unnoticed.
+`assertMilestonesEarnable` runs the same check alone, for a demonstration target that is not meant to separate.
+
+| Contract | Milestones | Earnable | Reference score | Best trivial baseline |
+|---|---|---|---|---|
+| ALE Breakout | 6 | 4 | 4 of 4 earnable | 0 earnable |
+| Libbet through `pyboy-generic` | 6 | 4 | 3 of 4 earnable | 3 earnable |
+
+Breakout separates and its score was quoted out of the wrong denominator.
+Libbet does not separate, and its earnable milestones are exactly the ones a constant button press already earns.
+
 ### The measurement that made this exist
 
 A live agent campaign ran 70 turns on Libbet and the Magic Floor through `adapters/pyboy-generic` and the packaged `pyboy/discovery-libbet.json` blind-discovery document.
diff --git a/docs/adapters.md b/docs/adapters.md
index 5de4def..5da81b2 100644
--- a/docs/adapters.md
+++ b/docs/adapters.md
@@ -118,6 +118,36 @@ Over the same 70 turns, on the same ROM and the same contract:
 The finding is not a Libbet defect and not a PyBoy defect.
 It is what a derived contract is worth before somebody measures it.
 
+### Earnable milestones and replay-identity checks
+
+A derived contract fails in a second way, and the separation test alone cannot see it.
+`deriveContract` samples whatever the mark asks for, so a mark that asks for a frame hash or a save hash pins the exact bytes the reference run produced.
+No independent policy earns such a milestone. Reaching it means reproducing the reference, not playing.
+Because no trivial baseline earns it either, the separation test used to read it as the contract's strongest evidence.
+
+`contractEarnability(contract)` splits the milestones by check kind, and follows `requires`: a threshold gated behind a hash is unreachable too.
+`assertContractSeparates` and `assertMilestonesEarnable` refuse a contract whose unearnable milestones the author has not declared by id.
+The declaration is an exact set, so a hash milestone that a later derivation adds cannot enter a published contract unnoticed.
+
+Measured on the packaged references:
+
+| Contract | Milestones | Earnable | Replay-identity |
+|---|---|---|---|
+| ALE Breakout | 6 | 4 | 2 — `frame-at-first-score`, `save-at-first-score` |
+| Libbet through `pyboy-generic` | 6 | 4 | 2 — `state-at-first-progression`, `frame-at-first-progression` |
+| `save-levels` toy | 2 | 0 | 2 — the save hash, and the event gated behind it |
+| `screen-puzzle` toy | 2 | 0 | 2 — both frame hashes |
+
+Keep the identity checks. They are the whole of what replay attestation proves.
+Declare them, and report the score as earned over earnable:
+
+```ts
+const report = calibrateContract(game, contract, { reference, vocabulary })
+assertContractSeparates(report, {
+  identityChecks: ['frame-at-first-score', 'save-at-first-score'],
+})
+```
+
 ## Libretro consoles through stable-retro
 
 ```ts

From 953366b241dc863632e956db4237bcc3143196e9 Mon Sep 17 00:00:00 2001
From: Drew Stone <drewstone329@gmail.com>
Date: Sat, 22 Aug 2026 11:28:33 -0700
Subject: [PATCH 6/6] docs(adapters): list the earnable split of every packaged
 contract

Six of the eleven packaged contracts carry at least one replay-identity
check. PyBoy Tetris is 3 of 5 earnable, Gymnasium FrozenLake 2 of 3.
The Breakout and Libbet rows are asserted against a running emulator; the
rest follow from the check kinds their references declare.
---
 docs/adapters.md | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/docs/adapters.md b/docs/adapters.md
index 5da81b2..7ffd921 100644
--- a/docs/adapters.md
+++ b/docs/adapters.md
@@ -129,15 +129,25 @@ Because no trivial baseline earns it either, the separation test used to read it
 `assertContractSeparates` and `assertMilestonesEarnable` refuse a contract whose unearnable milestones the author has not declared by id.
 The declaration is an exact set, so a hash milestone that a later derivation adds cannot enter a published contract unnoticed.
 
-Measured on the packaged references:
+Every packaged contract, by check kind and `requires` graph:
 
 | Contract | Milestones | Earnable | Replay-identity |
 |---|---|---|---|
 | ALE Breakout | 6 | 4 | 2 — `frame-at-first-score`, `save-at-first-score` |
 | Libbet through `pyboy-generic` | 6 | 4 | 2 — `state-at-first-progression`, `frame-at-first-progression` |
+| PyBoy Tetris | 5 | 3 | 2 — `game-started`, `state-at-line-1` |
+| stable-retro Airstriker | 5 | 4 | 1 — `frame-at-first-score` |
+| Gymnasium CartPole | 5 | 4 | 1 — `frame-at-25-steps` |
+| Gymnasium FrozenLake | 3 | 2 | 1 — `goal-frame` |
+| RetroArch, `n` channels with screen milestones | n + 2 | n + 1 | 1 — `frame-at-first-progression` |
+| `native-2048` | 7 | 7 | 0 |
+| `engine-crawler` toy | 4 | 4 | 0 |
 | `save-levels` toy | 2 | 0 | 2 — the save hash, and the event gated behind it |
 | `screen-puzzle` toy | 2 | 0 | 2 — both frame hashes |
 
+The Breakout and Libbet rows are asserted against the running emulator in `ale.test.mts` and `pyboy-libbet.test.mts`.
+The rest follow from the check kinds their references declare, which is the whole of the rule.
+
 Keep the identity checks. They are the whole of what replay attestation proves.
 Declare them, and report the score as earned over earnable: