From dd6fcc6b69950f1024dc8e295c6e5814fc283fe5 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 21 Aug 2026 04:11:02 -0700 Subject: [PATCH] fix(surface-identity): mint a component surface identity that does not depend on the host collation --- CHANGELOG.md | 10 ++ clients/python/pyproject.toml | 2 +- clients/python/src/agent_eval_rpc/__init__.py | 2 +- clients/python/uv.lock | 2 +- package.json | 2 +- src/analyst/benchmark-implementation.ts | 2 +- src/campaign/presets/run-optimization.ts | 6 +- src/campaign/provenance.ts | 9 +- src/campaign/surface-identity.test.ts | 101 +++++++++++++----- src/campaign/surface-identity.ts | 57 +++++++++- 10 files changed, 157 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bdacf3c..1a8662ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.168.0] — 2026-08-21 + +### Fixed + +- A component surface's stored identity no longer depends on the host's collation. `componentSurfaceIdentityMaterial` ordered component names with `localeCompare`, which reads the runtime's default collation rather than the value, and that material feeds `surfaceContentHash` and `surfaceHash` — identities that are **written down** and later recomputed and compared. Three sites do exactly that (`src/campaign/provenance.ts:351`, `src/campaign/presets/run-optimization.ts:658` and `:693`), each of them a fail-closed refusal, so under a different ICU build or locale a legitimate resume or parent selection could be rejected as a surface that does not match its own identity. The material now comes from `canonicalString`, which orders keys by UTF-16 code unit (RFC 8785) — a property of the value alone. +- `surfaceHashMatches(surface, storedHash)` is the verify path for the retention window. A stored loop key is 16 hex characters with no room for a scheme tag, so unlike an `agent-profile-cell` id it cannot name the scheme that minted it; the matcher tries the current material and then the retired one, which gives the same property — a key minted by an earlier release still matches its own surface, and an edited surface matches neither. The retired builder is private and reachable only from that matcher; nothing mints from it. The three comparison sites now use the matcher instead of `!==`. +- Every stored component-surface identity moves, not only one whose names sort differently under a collation: RFC 8785 also orders the two top-level keys, so `components` now precedes `schema`. Prompt and code surfaces are unaffected — neither builds material from an ordered key list. + +--- + ## [0.167.1] — 2026-08-21 ### Changed diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index fd686781..e12b4277 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.167.1" +version = "0.168.0" description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index 0e5d021b..1dcec90d 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -53,7 +53,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.167.1" + __version__ = "0.168.0" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index ac6e1522..fa1b40f6 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -34,7 +34,7 @@ conflicts = [[ [[package]] name = "agent-eval-rpc" -version = "0.167.1" +version = "0.168.0" source = { editable = "." } dependencies = [ { name = "filelock" }, diff --git a/package.json b/package.json index b5a1665f..c545ddf1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.167.1", + "version": "0.168.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index 9ae9cba3..4b0f4673 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = - 'a372aba7edb147c768813488fa7c69395f093f4c2b9bb9862aff2d3b468be47a' + '4985691f6640e6088c25c3cdafc5c06135a9b68273430f0b65bdb48ff4aded5a' /** The published benchmark evidence was produced at this package version, by * the retired one-shot direct runner, before trace analysts moved to the diff --git a/src/campaign/presets/run-optimization.ts b/src/campaign/presets/run-optimization.ts index c038df70..f6997cf7 100644 --- a/src/campaign/presets/run-optimization.ts +++ b/src/campaign/presets/run-optimization.ts @@ -36,7 +36,7 @@ import { import type { SearchHistoryReceipt } from '../search-history-receipt' import { type SearchLedgerBinding, SearchRecorder } from '../search-ledger-recording' import { createRunCostLedger, fsCampaignStorage } from '../storage' -import { surfaceHash } from '../surface-identity' +import { surfaceHash, surfaceHashMatches } from '../surface-identity' import { type CampaignResult, type GenerationRecord, @@ -655,7 +655,7 @@ function resolveSelectedParent( `runOptimization: selectParent returned surface "${parent.surfaceHash}" in generation ${generation}, which this run has not measured to completion; a parent must be a scored surface from the frontier`, ) } - if (surfaceHash(parent.surface) !== parent.surfaceHash) { + if (!surfaceHashMatches(parent.surface, parent.surfaceHash)) { throw new Error( `runOptimization: selectParent returned a parent whose surface does not match its surfaceHash "${parent.surfaceHash}" (generation ${generation})`, ) @@ -690,7 +690,7 @@ function validatedPremeasuredBaseline(arg seed: number }): CampaignResult { const { input } = args - if (input.surfaceHash !== surfaceHash(args.baselineSurface)) { + if (!surfaceHashMatches(args.baselineSurface, input.surfaceHash)) { throw new Error( 'runOptimization: premeasured baseline surface hash does not match baselineSurface', ) diff --git a/src/campaign/provenance.ts b/src/campaign/provenance.ts index ce762e19..fe8361de 100644 --- a/src/campaign/provenance.ts +++ b/src/campaign/provenance.ts @@ -45,7 +45,12 @@ import type { RunImprovementLoopResult } from './presets/run-improvement-loop' import { campaignCellExecutionEvidence, projectCampaignCellQuality } from './run-record' import { campaignMeanComposite, campaignMeanCompositeOrNull } from './score-utils' import type { CampaignStorage } from './storage' -import { renderSurfaceDiff, surfaceContentHash, surfaceHash } from './surface-identity' +import { + renderSurfaceDiff, + surfaceContentHash, + surfaceHash, + surfaceHashMatches, +} from './surface-identity' import type { CampaignResult, GateContribution, @@ -348,7 +353,7 @@ export function buildLoopProvenanceRecord throw new Error('buildLoopProvenanceRecord: measured candidate is missing its surface') } const { surface, campaign } = measured - if (surfaceHash(surface) !== c.surfaceHash) { + if (!surfaceHashMatches(surface, c.surfaceHash)) { throw new Error( 'buildLoopProvenanceRecord: candidate surface hash does not match its surface bytes', ) diff --git a/src/campaign/surface-identity.test.ts b/src/campaign/surface-identity.test.ts index 76767d46..f3bb28fd 100644 --- a/src/campaign/surface-identity.test.ts +++ b/src/campaign/surface-identity.test.ts @@ -1,32 +1,83 @@ +import { createHash } from 'node:crypto' import { describe, expect, it } from 'vitest' +import { canonicalString } from '../ledger-core/canonical' +import { + componentSurfaceIdentityMaterial, + surfaceHash, + surfaceHashMatches, +} from './surface-identity' +import type { ComponentSurface } from './types' -import { renderSurfaceDiff } from './surface-identity' -import type { CodeSurface } from './types' +/** Component names that sort differently by collation than by code unit. */ +const mixedCase: ComponentSurface = { + kind: 'components', + components: { Accuracy: 'a', brevity: 'b', Clarity: 'c' }, +} -function codeSurface(digit: string): CodeSurface { - return { - kind: 'code', - worktreeRef: `/tmp/${digit}`, - baseRef: 'main', - baseCommit: 'a'.repeat(40), - baseTree: 'b'.repeat(40), - candidateCommit: digit.repeat(40), - candidateTree: digit.repeat(40), - patch: { - format: 'git-diff-binary', - sha256: `sha256:${digit.repeat(64)}`, - byteLength: 42, - }, - summary: 'Measured implementation change', - } +/** The material this package minted before the identity moved to RFC 8785. */ +function retiredMaterial(surface: ComponentSurface): string { + return JSON.stringify({ + schema: 'tangle.component-surface', + components: Object.fromEntries( + Object.entries(surface.components).sort(([left], [right]) => left.localeCompare(right)), + ), + }) } -describe('renderSurfaceDiff', () => { - it('identifies an exact code patch instead of a mutable worktree path', () => { - const diff = renderSurfaceDiff(codeSurface('c'), codeSurface('d')) - expect(diff).toContain(`patch=sha256:${'c'.repeat(64)}`) - expect(diff).toContain(`patch=sha256:${'d'.repeat(64)}`) - expect(diff).toContain('patchBytes=42') - expect(diff).not.toContain('/tmp/') +const retiredHash = (surface: ComponentSurface): string => + createHash('sha256').update(retiredMaterial(surface)).digest('hex').slice(0, 16) + +describe('component surface identity', () => { + /** + * The defect this closes: the identity is stored and later recomputed, so an + * order that comes from the host's collation rather than from the value means + * the same surface can fail to match its own recorded identity on another + * machine. + */ + it('orders component names by code unit, not by the host collation', () => { + expect(componentSurfaceIdentityMaterial(mixedCase)).toBe( + canonicalString({ schema: 'tangle.component-surface', components: mixedCase.components }), + ) + const names = Object.keys(mixedCase.components) + expect([...names].sort()).toEqual(['Accuracy', 'Clarity', 'brevity']) + expect([...names].sort((a, b) => a.localeCompare(b))).toEqual([ + 'Accuracy', + 'brevity', + 'Clarity', + ]) + // The orders disagree here; the identity moves for every component surface + // regardless, which is why the retired material still has to verify. + expect(retiredMaterial(mixedCase)).not.toBe(componentSurfaceIdentityMaterial(mixedCase)) + }) + + it('still matches an identity minted by the retired material', () => { + const stored = retiredHash(mixedCase) + expect(stored).not.toBe(surfaceHash(mixedCase)) + expect(surfaceHashMatches(mixedCase, stored)).toBe(true) + expect(surfaceHashMatches(mixedCase, surfaceHash(mixedCase))).toBe(true) + }) + + it('refuses an identity that belongs to a different surface', () => { + const edited: ComponentSurface = { + kind: 'components', + components: { ...mixedCase.components, brevity: 'edited' }, + } + expect(surfaceHashMatches(edited, retiredHash(mixedCase))).toBe(false) + expect(surfaceHashMatches(edited, surfaceHash(mixedCase))).toBe(false) + }) + + /** + * The retention window covers every stored component-surface identity, not + * just a mixed-case one: RFC 8785 orders the two top-level keys as well, so + * `components` precedes `schema` where the retired builder emitted them in + * literal order. + */ + it('moves the identity even for names that sort the same either way', () => { + const ascii: ComponentSurface = { + kind: 'components', + components: { alpha: 'a', beta: 'b' }, + } + expect(surfaceHash(ascii)).not.toBe(retiredHash(ascii)) + expect(surfaceHashMatches(ascii, retiredHash(ascii))).toBe(true) }) }) diff --git a/src/campaign/surface-identity.ts b/src/campaign/surface-identity.ts index 708daa80..98674e07 100644 --- a/src/campaign/surface-identity.ts +++ b/src/campaign/surface-identity.ts @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto' +import { canonicalString } from '../ledger-core/canonical' import type { CodeSurface, ComponentSurface, MutableSurface } from './types' const GIT_OBJECT_ID = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/ @@ -69,8 +70,39 @@ function assertComponentSurface(surface: unknown): asserts surface is ComponentS } } -/** Return deterministic identity material independent of component key order. */ +/** + * Deterministic identity material for a component surface. + * + * `canonicalString` orders keys by UTF-16 code unit (RFC 8785), which is a + * property of the value alone. The previous material ordered them with + * `localeCompare`, which reads the host's collation — so the same surface + * could produce two different identities on two machines, and the stored + * identity would stop matching a recomputation of the identical surface. + */ export function componentSurfaceIdentityMaterial(surface: ComponentSurface): string { + assertComponentSurface(surface) + return canonicalString({ + schema: 'tangle.component-surface', + components: surface.components, + }) +} + +/** + * The retired material builder, kept PRIVATE and reachable only from + * {@link surfaceHashMatches}. + * + * A surface identity recorded before this release was minted from these bytes. + * The verify path tries the current material first and falls back to this one, + * so a stored identity still matches its own surface; nothing mints from it. + * + * Every component surface's identity moves, not only one whose names sort + * differently under the host's collation: RFC 8785 also orders the two + * top-level keys, so `components` precedes `schema` where this builder emitted + * them in literal order. The retention window therefore covers every stored + * component-surface identity, which is why this builder is kept rather than + * scoped to the mixed-case case. + */ +function retiredComponentSurfaceIdentityMaterial(surface: ComponentSurface): string { assertComponentSurface(surface) return JSON.stringify({ schema: 'tangle.component-surface', @@ -114,6 +146,29 @@ export function surfaceHash(surface: MutableSurface): string { return surfaceContentHash(surface).slice('sha256:'.length, 'sha256:'.length + 16) } +/** + * Whether `storedHash` is the loop key of `surface`, under the current identity + * material or the retired one. + * + * A stored key is 16 hex characters with no room for a scheme tag, so the + * scheme cannot be read off the value the way an `agent-profile-cell` id names + * its own. The verify path therefore tries both, which gives the same property: + * a key minted by an earlier release still matches its own surface, and a + * surface that was actually edited matches neither. + * + * Only a component surface can differ between the two; a prompt or code surface + * produces identical material under both, so the second comparison is a no-op + * for them. + */ +export function surfaceHashMatches(surface: MutableSurface, storedHash: string): boolean { + if (surfaceHash(surface) === storedHash) return true + if (typeof surface === 'string' || surface.kind !== 'components') return false + const retired = createHash('sha256') + .update(retiredComponentSurfaceIdentityMaterial(surface)) + .digest('hex') + return retired.slice(0, 16) === storedHash +} + /** Canonical customer-visible description of the exact before/after surfaces. */ export function renderSurfaceDiff( winnerSurface: MutableSurface,