chore(gate): fail an ordering that reads the host collation before a digest

CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 
 ---
 
+## [0.170.0] — 2026-08-21
+
+### Added
+
+- `pnpm check:collation-ordering`, wired into `verify:package`. It reports a `localeCompare` comparator inside a function that also canonicalizes or hashes — the shape that let eleven orderings decide a digest from the host's collation rather than from the value. The rule is narrow on purpose: ordering a table a human reads is not this gate's business, and a Markdown renderer that sorts rows does not canonicalize, so it is not reported. An allowlist entry matching nothing fails the gate, so a stale waiver cannot hide a new one.
+- `scripts/source-scan.mjs` holds the reading primitives both source gates share — walking `src/`, naming a function by its binding, locating an offset, visiting a tree. A second gate with its own copies would have re-created the duplication these releases removed, and sharing them means the naming fix in 0.168.1 reaches both gates rather than one.
+
+---
+
 ## [0.169.0] — 2026-08-21
 
 ### Fixed

clients/python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "agent-eval-rpc"
-version = "0.169.0"
+version = "0.170.0"
 description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval."
 readme = "README.md"
 requires-python = ">=3.10"

clients/python/src/agent_eval_rpc/__init__.py

@@ -53,7 +53,7 @@
 try:
     __version__ = version("agent-eval-rpc")
 except PackageNotFoundError:
-    __version__ = "0.169.0"
+    __version__ = "0.170.0"
 
 __all__ = [
     "Client",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.169.0",
+  "version": "0.170.0",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -184,11 +184,12 @@
     "check:skill": "node scripts/check-skill.mjs",
     "check:model-ids": "node scripts/check-model-id-requests.mjs",
     "check:canonical-json": "node scripts/check-canonical-json.mjs",
+    "check:collation-ordering": "node scripts/check-collation-ordering.mjs",
     "api:census": "node scripts/export-census.mjs",
     "check:analyst-benchmark": "node scripts/check-analyst-benchmark-implementation.mjs",
     "analyst:pin": "node scripts/check-analyst-benchmark-implementation.mjs --write",
     "openapi": "node dist/cli.js openapi --out dist/openapi.json",
-    "verify:package": "pnpm check:analyst-benchmark && pnpm run check:skill && pnpm run check:model-ids && pnpm run check:canonical-json && publint && attw --pack --profile esm-only . && node scripts/verify-package-exports.mjs && pnpm run contract:finding:check && pnpm run evidence:check",
+    "verify:package": "pnpm check:analyst-benchmark && pnpm run check:skill && pnpm run check:model-ids && pnpm run check:canonical-json && pnpm run check:collation-ordering && publint && attw --pack --profile esm-only . && node scripts/verify-package-exports.mjs && pnpm run contract:finding:check && pnpm run evidence:check",
     "contract:finding": "tsx scripts/emit-finding-contract.ts",
     "contract:finding:check": "tsx scripts/emit-finding-contract.ts --check",
     "evidence:render": "tsx scripts/render-evidence-index.ts",

scripts/check-collation-ordering.mjs

@@ -0,0 +1,162 @@
+/**
+ * Collation-ordering gate.
+ *
+ * RFC 8785 canonicalizes an array BY POSITION. A sort in front of a canonical
+ * serialization therefore decides the digest bytes, and a comparator built on
+ * `String.prototype.localeCompare` reads the host's collation rather than the
+ * value: the ids `Accuracy, brevity, Clarity` order as `Accuracy,brevity,Clarity`
+ * under an en-US collation and as `Accuracy,Clarity,brevity` by code unit, and
+ * the two produce different digests for the same data. A digest that moves with
+ * the machine is not an identity.
+ *
+ * The rule is narrow on purpose: a `localeCompare` comparator is reported only
+ * inside a function that ALSO canonicalizes or hashes. Ordering a table a human
+ * reads is not this gate's business, and a Markdown renderer that sorts rows
+ * does not canonicalize, so it is not reported.
+ *
+ * The type system cannot express this: `a.localeCompare(b)` is an ordinary,
+ * well-typed comparator. `compareCodeUnits` in `src/ledger-core/canonical.ts`
+ * is the sanctioned alternative.
+ */
+
+import { readFileSync, statSync } from 'node:fs'
+import { dirname, relative, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { parseSync } from 'oxc-parser'
+import { functionName, lineOf, sourceFiles, visitNodes } from './source-scan.mjs'
+
+const REPOSITORY_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
+
+/** Calls that turn a value into canonical bytes or a digest. */
+const CANONICALIZERS = new Set([
+  'canonicalString',
+  'hashCanonical',
+  'canonicalDigest',
+  'createHash',
+  'contentHash',
+  'sha256Digest',
+])
+
+/** A member call whose name ends here also counts: `x.someDigest(...)`. */
+const CANONICALIZER_SUFFIX = /Digest$/
+
+/**
+ * Comparators that must keep `localeCompare` even though their function also
+ * hashes. `file` and `fn` must both match; an entry matching nothing fails the
+ * gate, so a stale waiver cannot hide a new one.
+ */
+const ALLOWLIST = []
+
+const FUNCTION_NODES = new Set([
+  'FunctionDeclaration',
+  'FunctionExpression',
+  'ArrowFunctionExpression',
+])
+
+export function checkCollationOrdering({ root = REPOSITORY_ROOT, allowlist = ALLOWLIST } = {}) {
+  const offences = []
+  const usedWaivers = new Set()
+  const sourceRoot = resolve(root, 'src')
+  if (statSync(sourceRoot, { throwIfNoEntry: false })?.isDirectory()) {
+    for (const file of sourceFiles(sourceRoot)) {
+      const relativePath = relative(root, file).replaceAll('\\', '/')
+      if (/\.(test|spec)\.ts$/.test(relativePath)) continue
+      collect({ file, relativePath, allowlist, usedWaivers, offences })
+    }
+  }
+  return {
+    offences,
+    unusedWaivers: allowlist.filter((entry) => !usedWaivers.has(`${entry.file} ${entry.fn}`)),
+  }
+}
+
+function collect({ file, relativePath, allowlist, usedWaivers, offences }) {
+  const source = readFileSync(file, 'utf8')
+  const { program, errors } = parseSync(file, source)
+  if (errors.length > 0) throw new Error(`${relativePath}: parse failed — ${errors[0].message}`)
+
+  eachTopLevelFunction(program, (fn) => {
+    if (!canonicalizes(fn)) return
+    const line = collatingSortLine(fn, source)
+    if (line === undefined) return
+    const name = functionName(fn, source)
+    const waiver = allowlist.find((e) => e.file === relativePath && e.fn === name)
+    if (waiver !== undefined) usedWaivers.add(`${waiver.file} ${waiver.fn}`)
+    else offences.push({ file: relativePath, line, fn: name })
+  })
+}
+
+/** Visit every function, innermost scopes included, exactly once. */
+function eachTopLevelFunction(program, visitor) {
+  visitNodes(program, (node) => {
+    if (FUNCTION_NODES.has(node.type)) visitor(node)
+  })
+}
+
+/** Whether the body turns a value into canonical bytes or a digest. */
+function canonicalizes(fn) {
+  let found = false
+  visitNodes(fn.body, (node) => {
+    if (node.type !== 'CallExpression') return
+    const callee = node.callee
+    const name =
+      callee?.type === 'Identifier'
+        ? callee.name
+        : callee?.type === 'MemberExpression'
+          ? callee.property?.name
+          : undefined
+    if (typeof name !== 'string') return
+    if (CANONICALIZERS.has(name) || CANONICALIZER_SUFFIX.test(name)) found = true
+  })
+  return found
+}
+
+/** The line of a `.sort(...)` whose comparator calls `localeCompare`. */
+function collatingSortLine(fn, source) {
+  let line
+  visitNodes(fn.body, (node) => {
+    if (line !== undefined) return
+    if (node.type !== 'CallExpression') return
+    if (node.callee?.type !== 'MemberExpression') return
+    if (node.callee.property?.name !== 'sort') return
+    for (const argument of node.arguments ?? []) {
+      if (!callsLocaleCompare(argument)) continue
+      line = lineOf(source, argument.start)
+      return
+    }
+  })
+  return line
+}
+
+function callsLocaleCompare(node) {
+  let found = false
+  visitNodes(node, (child) => {
+    if (child.type !== 'CallExpression') return
+    if (child.callee?.type !== 'MemberExpression') return
+    if (child.callee.property?.name === 'localeCompare') found = true
+  })
+  return found
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const { offences, unusedWaivers } = checkCollationOrdering()
+  for (const offence of offences) {
+    console.error(
+      `${offence.file}:${offence.line}: ${offence.fn}() orders with localeCompare and then canonicalizes — ` +
+        'the host collation would decide the bytes. Sort with compareCodeUnits from src/ledger-core/canonical.ts.',
+    )
+  }
+  for (const waiver of unusedWaivers) {
+    console.error(
+      `scripts/check-collation-ordering.mjs: the allowlist entry for ${waiver.fn}() in ${waiver.file} matches nothing; remove it.`,
+    )
+  }
+  if (offences.length > 0 || unusedWaivers.length > 0) {
+    console.error(
+      '\nRFC 8785 canonicalizes an array by position, so an ordering that reads the host collation makes the digest a property of the machine. compareCodeUnits is the one comparator for an ordering that reaches a digest.',
+    )
+    process.exitCode = 1
+  } else {
+    console.log('collation ordering gate valid: no canonicalizing function orders with localeCompare')
+  }
+}

scripts/check-collation-ordering.test.mjs

@@ -0,0 +1,103 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, test } from 'vitest'
+import { checkCollationOrdering } from './check-collation-ordering.mjs'
+
+const tempRoots = []
+
+afterEach(() => {
+  for (const root of tempRoots.splice(0)) rmSync(root, { force: true, recursive: true })
+})
+
+function run(files, allowlist = []) {
+  const root = mkdtempSync(join(tmpdir(), 'collation-gate-'))
+  tempRoots.push(root)
+  for (const [name, content] of Object.entries(files)) {
+    const path = join(root, 'src', name)
+    mkdirSync(join(path, '..'), { recursive: true })
+    writeFileSync(path, content)
+  }
+  return checkCollationOrdering({ root, allowlist })
+}
+
+describe('an ordering that reaches a digest', () => {
+  test('is reported with its file, line, and function name', () => {
+    const { offences } = run({
+      'digest.ts': [
+        'export function suiteDigest(files: { path: string }[]): string {',
+        '  const sorted = [...files].sort((a, b) => a.path.localeCompare(b.path))',
+        '  return hashCanonical(sorted)',
+        '}',
+      ].join('\n'),
+    })
+
+    expect(offences).toEqual([{ file: 'src/digest.ts', line: 2, fn: 'suiteDigest' }])
+  })
+
+  test('is reported when the digest comes from createHash rather than the canonical helper', () => {
+    const { offences } = run({
+      'suite.ts': [
+        'export function fold(files: { path: string }[]): string {',
+        '  const hash = createHash("sha256")',
+        '  for (const f of [...files].sort((a, b) => a.path.localeCompare(b.path))) hash.update(f.path)',
+        '  return hash.digest("hex")',
+        '}',
+      ].join('\n'),
+    })
+
+    expect(offences.map((o) => o.fn)).toEqual(['fold'])
+  })
+
+  test('is not reported once the comparator orders by code unit', () => {
+    const { offences } = run({
+      'digest.ts': [
+        'export function suiteDigest(files: { path: string }[]): string {',
+        '  const sorted = [...files].sort((a, b) => compareCodeUnits(a.path, b.path))',
+        '  return hashCanonical(sorted)',
+        '}',
+      ].join('\n'),
+    })
+
+    expect(offences).toEqual([])
+  })
+})
+
+/**
+ * The predicate keys on "canonicalizes AND orders by collation", not on "sorts
+ * and serializes somewhere in the same body". A report a human reads is free to
+ * order by collation, and this is the distinction a broader predicate could not
+ * make.
+ */
+describe('an ordering that reaches a report', () => {
+  test('is not reported when the function never canonicalizes', () => {
+    const { offences } = run({
+      'render.ts': [
+        'export function renderTable(rows: Record<string, number>): string {',
+        '  return Object.entries(rows)',
+        '    .sort(([a], [b]) => a.localeCompare(b))',
+        '    .map(([k, v]) => `| ${k} | ${v} |`)',
+        '    .join("\\n")',
+        '}',
+      ].join('\n'),
+    })
+
+    expect(offences).toEqual([])
+  })
+})
+
+describe('the allowlist', () => {
+  test('fails the gate when an entry matches nothing, so a stale waiver cannot hide a new one', () => {
+    const { unusedWaivers } = run({ 'clean.ts': 'export const x = 1\n' }, [
+      { file: 'src/gone.ts', fn: 'removed', reason: 'stale' },
+    ])
+
+    expect(unusedWaivers).toHaveLength(1)
+  })
+})
+
+describe('the shipped repository', () => {
+  test('passes its own gate', () => {
+    expect(checkCollationOrdering()).toEqual({ offences: [], unusedWaivers: [] })
+  })
+})

scripts/source-scan.mjs

@@ -0,0 +1,65 @@
+/**
+ * Shared reading primitives for the source gates.
+ *
+ * Both gates walk `src/`, locate a function, and name it. Keeping one copy of
+ * each means a fix to how a function is named — such as no longer naming an
+ * anonymous arrow after the function it is passed to — reaches every gate at
+ * once rather than one of them.
+ */
+
+import { readdirSync, statSync } from 'node:fs'
+import { join } from 'node:path'
+
+/** Every `.ts` file under `directory`, in a stable order. */
+export function* sourceFiles(directory) {
+  for (const name of readdirSync(directory).sort()) {
+    const path = join(directory, name)
+    if (statSync(path).isDirectory()) yield* sourceFiles(path)
+    else if (path.endsWith('.ts')) yield path
+  }
+}
+
+/**
+ * The name a function is BOUND to, or undefined when it has none.
+ *
+ * A declaration id, a `const`/`let`/`var` binding, or an object-property key.
+ * Deliberately NOT the identifier in front of an open paren: an arrow passed as
+ * an argument — `sumOver(() => …)` — sits behind the text `sumOver(`, and
+ * reading that as its name gives a function the name of the thing it is passed
+ * to. Only a bound function can be called by name.
+ */
+export function boundName(fn, source) {
+  if (fn.id?.name) return fn.id.name
+  const before = source.slice(Math.max(0, fn.start - 200), fn.start)
+  const declared = before.match(/(?:const|let|var|function)\s+([A-Za-z0-9_$]+)\s*(?::[^=]*)?=?\s*$/)
+  if (declared) return declared[1]
+  const property = before.match(/([A-Za-z0-9_$]+)\s*:\s*$/)
+  return property ? property[1] : undefined
+}
+
+/** The name to print for a function. An unbound one is located by its line. */
+export function functionName(fn, source) {
+  return boundName(fn, source) ?? '(anonymous)'
+}
+
+/** 1-based line of a byte offset. */
+export function lineOf(source, offset) {
+  let line = 1
+  for (let index = 0; index < offset; index++) if (source.charCodeAt(index) === 10) line++
+  return line
+}
+
+/** Call `onNode` for every AST node under `node`. */
+export function visitNodes(node, onNode) {
+  if (node === null || typeof node !== 'object') return
+  if (Array.isArray(node)) {
+    for (const child of node) visitNodes(child, onNode)
+    return
+  }
+  if (typeof node.type !== 'string') return
+  onNode(node)
+  for (const key of Object.keys(node)) {
+    if (key === 'type' || key === 'start' || key === 'end') continue
+    visitNodes(node[key], onNode)
+  }
+}