From 4e6df63a8d5a0b8c9c0cb29d5900e495995be565 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 21 Aug 2026 04:40:00 -0700 Subject: [PATCH] chore(gate): fail an ordering that reads the host collation before a digest --- CHANGELOG.md | 9 + clients/python/pyproject.toml | 2 +- clients/python/src/agent_eval_rpc/__init__.py | 2 +- clients/python/uv.lock | 2 +- package.json | 5 +- scripts/check-collation-ordering.mjs | 162 ++++++++++++++++++ scripts/check-collation-ordering.test.mjs | 103 +++++++++++ scripts/source-scan.mjs | 65 +++++++ src/analyst/benchmark-implementation.ts | 2 +- 9 files changed, 346 insertions(+), 6 deletions(-) create mode 100644 scripts/check-collation-ordering.mjs create mode 100644 scripts/check-collation-ordering.test.mjs create mode 100644 scripts/source-scan.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index eb1387d2..1338055c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.170.0] — 2026-08-21 + +### Added + +- `pnpm check:collation-ordering`, wired into `verify:package`. It reports a `localeCompare` comparator inside a function that also canonicalizes or hashes — the shape that let eleven orderings decide a digest from the host's collation rather than from the value. The rule is narrow on purpose: ordering a table a human reads is not this gate's business, and a Markdown renderer that sorts rows does not canonicalize, so it is not reported. An allowlist entry matching nothing fails the gate, so a stale waiver cannot hide a new one. +- `scripts/source-scan.mjs` holds the reading primitives both source gates share — walking `src/`, naming a function by its binding, locating an offset, visiting a tree. A second gate with its own copies would have re-created the duplication these releases removed, and sharing them means the naming fix in 0.168.1 reaches both gates rather than one. + +--- + ## [0.169.0] — 2026-08-21 ### Fixed diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 5daa8814..bbf51d42 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.169.0" +version = "0.170.0" description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index 9cc523d5..7f01de0f 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -53,7 +53,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.169.0" + __version__ = "0.170.0" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index 3744e6f9..2b27891a 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -34,7 +34,7 @@ conflicts = [[ [[package]] name = "agent-eval-rpc" -version = "0.169.0" +version = "0.170.0" source = { editable = "." } dependencies = [ { name = "filelock" }, diff --git a/package.json b/package.json index dce6b702..55d43a06 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.169.0", + "version": "0.170.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { @@ -184,11 +184,12 @@ "check:skill": "node scripts/check-skill.mjs", "check:model-ids": "node scripts/check-model-id-requests.mjs", "check:canonical-json": "node scripts/check-canonical-json.mjs", + "check:collation-ordering": "node scripts/check-collation-ordering.mjs", "api:census": "node scripts/export-census.mjs", "check:analyst-benchmark": "node scripts/check-analyst-benchmark-implementation.mjs", "analyst:pin": "node scripts/check-analyst-benchmark-implementation.mjs --write", "openapi": "node dist/cli.js openapi --out dist/openapi.json", - "verify:package": "pnpm check:analyst-benchmark && pnpm run check:skill && pnpm run check:model-ids && pnpm run check:canonical-json && publint && attw --pack --profile esm-only . && node scripts/verify-package-exports.mjs && pnpm run contract:finding:check && pnpm run evidence:check", + "verify:package": "pnpm check:analyst-benchmark && pnpm run check:skill && pnpm run check:model-ids && pnpm run check:canonical-json && pnpm run check:collation-ordering && publint && attw --pack --profile esm-only . && node scripts/verify-package-exports.mjs && pnpm run contract:finding:check && pnpm run evidence:check", "contract:finding": "tsx scripts/emit-finding-contract.ts", "contract:finding:check": "tsx scripts/emit-finding-contract.ts --check", "evidence:render": "tsx scripts/render-evidence-index.ts", diff --git a/scripts/check-collation-ordering.mjs b/scripts/check-collation-ordering.mjs new file mode 100644 index 00000000..386da6f9 --- /dev/null +++ b/scripts/check-collation-ordering.mjs @@ -0,0 +1,162 @@ +/** + * Collation-ordering gate. + * + * RFC 8785 canonicalizes an array BY POSITION. A sort in front of a canonical + * serialization therefore decides the digest bytes, and a comparator built on + * `String.prototype.localeCompare` reads the host's collation rather than the + * value: the ids `Accuracy, brevity, Clarity` order as `Accuracy,brevity,Clarity` + * under an en-US collation and as `Accuracy,Clarity,brevity` by code unit, and + * the two produce different digests for the same data. A digest that moves with + * the machine is not an identity. + * + * The rule is narrow on purpose: a `localeCompare` comparator is reported only + * inside a function that ALSO canonicalizes or hashes. Ordering a table a human + * reads is not this gate's business, and a Markdown renderer that sorts rows + * does not canonicalize, so it is not reported. + * + * The type system cannot express this: `a.localeCompare(b)` is an ordinary, + * well-typed comparator. `compareCodeUnits` in `src/ledger-core/canonical.ts` + * is the sanctioned alternative. + */ + +import { readFileSync, statSync } from 'node:fs' +import { dirname, relative, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { parseSync } from 'oxc-parser' +import { functionName, lineOf, sourceFiles, visitNodes } from './source-scan.mjs' + +const REPOSITORY_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..') + +/** Calls that turn a value into canonical bytes or a digest. */ +const CANONICALIZERS = new Set([ + 'canonicalString', + 'hashCanonical', + 'canonicalDigest', + 'createHash', + 'contentHash', + 'sha256Digest', +]) + +/** A member call whose name ends here also counts: `x.someDigest(...)`. */ +const CANONICALIZER_SUFFIX = /Digest$/ + +/** + * Comparators that must keep `localeCompare` even though their function also + * hashes. `file` and `fn` must both match; an entry matching nothing fails the + * gate, so a stale waiver cannot hide a new one. + */ +const ALLOWLIST = [] + +const FUNCTION_NODES = new Set([ + 'FunctionDeclaration', + 'FunctionExpression', + 'ArrowFunctionExpression', +]) + +export function checkCollationOrdering({ root = REPOSITORY_ROOT, allowlist = ALLOWLIST } = {}) { + const offences = [] + const usedWaivers = new Set() + const sourceRoot = resolve(root, 'src') + if (statSync(sourceRoot, { throwIfNoEntry: false })?.isDirectory()) { + for (const file of sourceFiles(sourceRoot)) { + const relativePath = relative(root, file).replaceAll('\\', '/') + if (/\.(test|spec)\.ts$/.test(relativePath)) continue + collect({ file, relativePath, allowlist, usedWaivers, offences }) + } + } + return { + offences, + unusedWaivers: allowlist.filter((entry) => !usedWaivers.has(`${entry.file} ${entry.fn}`)), + } +} + +function collect({ file, relativePath, allowlist, usedWaivers, offences }) { + const source = readFileSync(file, 'utf8') + const { program, errors } = parseSync(file, source) + if (errors.length > 0) throw new Error(`${relativePath}: parse failed — ${errors[0].message}`) + + eachTopLevelFunction(program, (fn) => { + if (!canonicalizes(fn)) return + const line = collatingSortLine(fn, source) + if (line === undefined) return + const name = functionName(fn, source) + const waiver = allowlist.find((e) => e.file === relativePath && e.fn === name) + if (waiver !== undefined) usedWaivers.add(`${waiver.file} ${waiver.fn}`) + else offences.push({ file: relativePath, line, fn: name }) + }) +} + +/** Visit every function, innermost scopes included, exactly once. */ +function eachTopLevelFunction(program, visitor) { + visitNodes(program, (node) => { + if (FUNCTION_NODES.has(node.type)) visitor(node) + }) +} + +/** Whether the body turns a value into canonical bytes or a digest. */ +function canonicalizes(fn) { + let found = false + visitNodes(fn.body, (node) => { + if (node.type !== 'CallExpression') return + const callee = node.callee + const name = + callee?.type === 'Identifier' + ? callee.name + : callee?.type === 'MemberExpression' + ? callee.property?.name + : undefined + if (typeof name !== 'string') return + if (CANONICALIZERS.has(name) || CANONICALIZER_SUFFIX.test(name)) found = true + }) + return found +} + +/** The line of a `.sort(...)` whose comparator calls `localeCompare`. */ +function collatingSortLine(fn, source) { + let line + visitNodes(fn.body, (node) => { + if (line !== undefined) return + if (node.type !== 'CallExpression') return + if (node.callee?.type !== 'MemberExpression') return + if (node.callee.property?.name !== 'sort') return + for (const argument of node.arguments ?? []) { + if (!callsLocaleCompare(argument)) continue + line = lineOf(source, argument.start) + return + } + }) + return line +} + +function callsLocaleCompare(node) { + let found = false + visitNodes(node, (child) => { + if (child.type !== 'CallExpression') return + if (child.callee?.type !== 'MemberExpression') return + if (child.callee.property?.name === 'localeCompare') found = true + }) + return found +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const { offences, unusedWaivers } = checkCollationOrdering() + for (const offence of offences) { + console.error( + `${offence.file}:${offence.line}: ${offence.fn}() orders with localeCompare and then canonicalizes — ` + + 'the host collation would decide the bytes. Sort with compareCodeUnits from src/ledger-core/canonical.ts.', + ) + } + for (const waiver of unusedWaivers) { + console.error( + `scripts/check-collation-ordering.mjs: the allowlist entry for ${waiver.fn}() in ${waiver.file} matches nothing; remove it.`, + ) + } + if (offences.length > 0 || unusedWaivers.length > 0) { + console.error( + '\nRFC 8785 canonicalizes an array by position, so an ordering that reads the host collation makes the digest a property of the machine. compareCodeUnits is the one comparator for an ordering that reaches a digest.', + ) + process.exitCode = 1 + } else { + console.log('collation ordering gate valid: no canonicalizing function orders with localeCompare') + } +} diff --git a/scripts/check-collation-ordering.test.mjs b/scripts/check-collation-ordering.test.mjs new file mode 100644 index 00000000..fb3fc7aa --- /dev/null +++ b/scripts/check-collation-ordering.test.mjs @@ -0,0 +1,103 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, test } from 'vitest' +import { checkCollationOrdering } from './check-collation-ordering.mjs' + +const tempRoots = [] + +afterEach(() => { + for (const root of tempRoots.splice(0)) rmSync(root, { force: true, recursive: true }) +}) + +function run(files, allowlist = []) { + const root = mkdtempSync(join(tmpdir(), 'collation-gate-')) + tempRoots.push(root) + for (const [name, content] of Object.entries(files)) { + const path = join(root, 'src', name) + mkdirSync(join(path, '..'), { recursive: true }) + writeFileSync(path, content) + } + return checkCollationOrdering({ root, allowlist }) +} + +describe('an ordering that reaches a digest', () => { + test('is reported with its file, line, and function name', () => { + const { offences } = run({ + 'digest.ts': [ + 'export function suiteDigest(files: { path: string }[]): string {', + ' const sorted = [...files].sort((a, b) => a.path.localeCompare(b.path))', + ' return hashCanonical(sorted)', + '}', + ].join('\n'), + }) + + expect(offences).toEqual([{ file: 'src/digest.ts', line: 2, fn: 'suiteDigest' }]) + }) + + test('is reported when the digest comes from createHash rather than the canonical helper', () => { + const { offences } = run({ + 'suite.ts': [ + 'export function fold(files: { path: string }[]): string {', + ' const hash = createHash("sha256")', + ' for (const f of [...files].sort((a, b) => a.path.localeCompare(b.path))) hash.update(f.path)', + ' return hash.digest("hex")', + '}', + ].join('\n'), + }) + + expect(offences.map((o) => o.fn)).toEqual(['fold']) + }) + + test('is not reported once the comparator orders by code unit', () => { + const { offences } = run({ + 'digest.ts': [ + 'export function suiteDigest(files: { path: string }[]): string {', + ' const sorted = [...files].sort((a, b) => compareCodeUnits(a.path, b.path))', + ' return hashCanonical(sorted)', + '}', + ].join('\n'), + }) + + expect(offences).toEqual([]) + }) +}) + +/** + * The predicate keys on "canonicalizes AND orders by collation", not on "sorts + * and serializes somewhere in the same body". A report a human reads is free to + * order by collation, and this is the distinction a broader predicate could not + * make. + */ +describe('an ordering that reaches a report', () => { + test('is not reported when the function never canonicalizes', () => { + const { offences } = run({ + 'render.ts': [ + 'export function renderTable(rows: Record): string {', + ' return Object.entries(rows)', + ' .sort(([a], [b]) => a.localeCompare(b))', + ' .map(([k, v]) => `| ${k} | ${v} |`)', + ' .join("\\n")', + '}', + ].join('\n'), + }) + + expect(offences).toEqual([]) + }) +}) + +describe('the allowlist', () => { + test('fails the gate when an entry matches nothing, so a stale waiver cannot hide a new one', () => { + const { unusedWaivers } = run({ 'clean.ts': 'export const x = 1\n' }, [ + { file: 'src/gone.ts', fn: 'removed', reason: 'stale' }, + ]) + + expect(unusedWaivers).toHaveLength(1) + }) +}) + +describe('the shipped repository', () => { + test('passes its own gate', () => { + expect(checkCollationOrdering()).toEqual({ offences: [], unusedWaivers: [] }) + }) +}) diff --git a/scripts/source-scan.mjs b/scripts/source-scan.mjs new file mode 100644 index 00000000..c448595a --- /dev/null +++ b/scripts/source-scan.mjs @@ -0,0 +1,65 @@ +/** + * Shared reading primitives for the source gates. + * + * Both gates walk `src/`, locate a function, and name it. Keeping one copy of + * each means a fix to how a function is named — such as no longer naming an + * anonymous arrow after the function it is passed to — reaches every gate at + * once rather than one of them. + */ + +import { readdirSync, statSync } from 'node:fs' +import { join } from 'node:path' + +/** Every `.ts` file under `directory`, in a stable order. */ +export function* sourceFiles(directory) { + for (const name of readdirSync(directory).sort()) { + const path = join(directory, name) + if (statSync(path).isDirectory()) yield* sourceFiles(path) + else if (path.endsWith('.ts')) yield path + } +} + +/** + * The name a function is BOUND to, or undefined when it has none. + * + * A declaration id, a `const`/`let`/`var` binding, or an object-property key. + * Deliberately NOT the identifier in front of an open paren: an arrow passed as + * an argument — `sumOver(() => …)` — sits behind the text `sumOver(`, and + * reading that as its name gives a function the name of the thing it is passed + * to. Only a bound function can be called by name. + */ +export function boundName(fn, source) { + if (fn.id?.name) return fn.id.name + const before = source.slice(Math.max(0, fn.start - 200), fn.start) + const declared = before.match(/(?:const|let|var|function)\s+([A-Za-z0-9_$]+)\s*(?::[^=]*)?=?\s*$/) + if (declared) return declared[1] + const property = before.match(/([A-Za-z0-9_$]+)\s*:\s*$/) + return property ? property[1] : undefined +} + +/** The name to print for a function. An unbound one is located by its line. */ +export function functionName(fn, source) { + return boundName(fn, source) ?? '(anonymous)' +} + +/** 1-based line of a byte offset. */ +export function lineOf(source, offset) { + let line = 1 + for (let index = 0; index < offset; index++) if (source.charCodeAt(index) === 10) line++ + return line +} + +/** Call `onNode` for every AST node under `node`. */ +export function visitNodes(node, onNode) { + if (node === null || typeof node !== 'object') return + if (Array.isArray(node)) { + for (const child of node) visitNodes(child, onNode) + return + } + if (typeof node.type !== 'string') return + onNode(node) + for (const key of Object.keys(node)) { + if (key === 'type' || key === 'start' || key === 'end') continue + visitNodes(node[key], onNode) + } +} diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index a7c13f6e..10362dd3 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = - 'fbebf7931e6cd5d7b34c05effa56c2783b07dc06b9b173af790042dcd0341b65' + '66c9e3c60b694c8a06926381cd4eebc413826faf06b6f397b521d28bdea28e22' /** The published benchmark evidence was produced at this package version, by * the retired one-shot direct runner, before trace analysts moved to the