From d1595a85cdc4f5915c609b655640c8527ba03ede Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 18 Aug 2026 12:14:19 -0400 Subject: [PATCH 1/2] publish Flow 1.31 governed evidence --- README.md | 15 +- docs/eval_results/PRODUCTION_READINESS.md | 66 + docs/eval_results/PUBLISHED_EVIDENCE.json | 21 +- .../EVIDENCE_MANIFEST.json | 487 +++++++ .../REPORT.md | 70 + .../REPRODUCE.md | 47 + .../dependency-freeze.txt | 37 + .../remote_lease_safety/REPORT.md | 53 + .../remote_lease_safety/dependency-freeze.txt | 37 + .../remote_lease_safety/results.json | 590 +++++++++ .../replication/REPORT.md | 70 + .../replication/dependency-freeze.txt | 37 + .../replication/results.json | 1118 ++++++++++++++++ .../theme_postcondition_over_halt.json | 30 + .../results.json | 1118 ++++++++++++++++ .../theme_postcondition_over_halt.json | 30 + .../transaction_probe/REPORT.md | 60 + .../transaction_probe/dependency-freeze.txt | 37 + .../transaction_probe/results.json | 1178 +++++++++++++++++ scripts/build_evidence_manifest.py | 22 +- scripts/check_published_evidence_freshness.py | 147 +- tests/test_published_evidence_freshness.py | 111 +- 22 files changed, 5366 insertions(+), 15 deletions(-) create mode 100644 docs/eval_results/PRODUCTION_READINESS.md create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/EVIDENCE_MANIFEST.json create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/REPORT.md create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/REPRODUCE.md create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/dependency-freeze.txt create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/REPORT.md create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/dependency-freeze.txt create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/results.json create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/replication/REPORT.md create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/replication/dependency-freeze.txt create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/replication/results.json create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/replication/theme_postcondition_over_halt.json create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/results.json create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/theme_postcondition_over_halt.json create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/REPORT.md create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/dependency-freeze.txt create mode 100644 docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/results.json diff --git a/README.md b/README.md index 2418b09..0527818 100644 --- a/README.md +++ b/README.md @@ -246,16 +246,23 @@ matches the current published release. The current set also has an `EVIDENCE_MANIFEST.json` that inventories the verifier scripts, retained artifacts, public reports, campaign environments, and task/oracle contracts. The 1.28 set remains stale because its run did not retain an exact browser -revision or installed dependency snapshot. The current 1.30 set records those +revision or installed dependency snapshot. The current 1.31 set records those facts for every campaign. The check runs offline on every pull request and against PyPI on a daily schedule (`.github/workflows/evidence-freshness.yml`). Re-run the comparison and publish a new evidence set rather than editing an old one: superseded reports stay reproducible against the wheel they were measured on. -See `docs/eval_results/current_flow_v1_30_0_local_20260806/` for the current -comparison and its exact retained campaign reports. The older 1.28 set remains -available as stale historical evidence; it was not relabeled. +The word `current` means release-fresh. It does not mean production-accepted. +Each campaign records its evidence class, whether it is production acceptance, +and whether it counts silent incorrect success and over-halt. The present set +contains local synthetic and contract-fixture evidence only. See the +[production-readiness evidence boundary](docs/eval_results/PRODUCTION_READINESS.md) +for the missing acceptance tracks and their exit conditions. + +See `docs/eval_results/current_flow_v1_31_0_local_20260818/` for the current +comparison and its exact retained campaign reports. The older sets remain +available as stale historical evidence; they were not relabeled. ## More workflows diff --git a/docs/eval_results/PRODUCTION_READINESS.md b/docs/eval_results/PRODUCTION_READINESS.md new file mode 100644 index 0000000..288ff5e --- /dev/null +++ b/docs/eval_results/PRODUCTION_READINESS.md @@ -0,0 +1,66 @@ +# Production-readiness evidence boundary + +OpenAdapt has production-capable mechanisms and bounded accepted workflows. +It does not have one global production-readiness state. Each workflow, +application, version, environment, identity rule, effect oracle, and deployment +boundary needs its own qualification. + +The `current` label in `PUBLISHED_EVIDENCE.json` has one narrow meaning: the +evidence set matches the current published `openadapt-flow` release. It does +not mean the campaign is production acceptance. The evidence manifest records +that distinction in `campaigns[].evidence_scope.production_acceptance`. + +## What the current public Evals set establishes + +The Flow 1.31.0 set contains four campaigns. Every condition has three trials. + +| Campaign | Environment | What it measures | Production acceptance | +| --- | --- | --- | --- | +| Comparison | Synthetic MockMed, local macOS host, headless Chromium | Compiled replay and two Playwright controls under clean, theme, and bounded label drift | No | +| Independent replication | The same bounded environment in a second complete run | Repeatability of the comparison result | No | +| Transaction probe | Synthetic MockMed fault server and its independent persistence store | Outcome taxonomy under normal, timeout, optimistic-screen, session, and duplicate faults | No | +| Remote lease safety | An instrumented fake remote backend | The input-edge refusal contract when a frame lease is absent or changes | No | + +The comparison and replication count silent incorrect success and over-halt. +The transaction and remote-lease probes do not count both metrics as production +reliability measures. Their manifest entries say `not_counted`. They therefore +cannot satisfy a production-acceptance gate. + +No campaign in this set represents hosted execution, a customer workflow, +Windows UIA, native macOS Accessibility, Linux AT-SPI, a real RDP session, or a +real Citrix ICA/HDX session. Other repositories retain bounded substrate +acceptance records. Those records remain specific to their exact tasks and +environments. + +## Acceptance exit conditions + +A production claim for one workflow needs one immutable acceptance record that +contains all of these facts: + +1. The exact task, application version, operating system, display/session + properties, OpenAdapt release, runner, and deployment revisions. +2. At least three trials for each healthy and fault condition in the accepted + matrix, with no hidden retry or excluded run. +3. An independent oracle for the intended effect, wrong-record effects, + duplicates, collateral changes, and effect absence or uncertainty. +4. A complete failure taxonomy that counts silent incorrect success, + over-halt, safe halt, uncertain delivery, platform failure, and operator + intervention. +5. The full delivery path for that operating model. A managed-browser result + includes authenticated submission, idempotency, polling, one-use delivery, + execution, independent verification, a signed receipt, and a signed webhook. +6. Real configured production dependencies. A mock or simulated success cannot + satisfy the gate. +7. A clean-machine install and first-run check on every operating system that + the claim names. + +For a real RDP or Citrix claim, the record also includes reconnect, session +change, stale-frame, resolution/DPI, and compression fault conditions. A fake +remote backend proves a runtime contract only. It does not replace a real +session result. + +For a generally available Desktop release, distribution evidence is separate +from workflow qualification. The release needs the intended signing, +notarization or Authenticode, installer, update, rollback, and key-lifecycle +evidence. An unsigned or ad-hoc-signed Beta installer can still provide useful +evaluation evidence, but it is not a generally available signed release. diff --git a/docs/eval_results/PUBLISHED_EVIDENCE.json b/docs/eval_results/PUBLISHED_EVIDENCE.json index 1760a84..0c3f660 100644 --- a/docs/eval_results/PUBLISHED_EVIDENCE.json +++ b/docs/eval_results/PUBLISHED_EVIDENCE.json @@ -6,6 +6,8 @@ "was measured against. scripts/check_published_evidence_freshness.py fails", "when a set marked current no longer pins the current published release, or", "when a pinned wheel digest does not match that release on PyPI.", + "The current label means release-fresh, not production-accepted. Every", + "current entry and campaign states its production_acceptance value.", "Mark a set superseded (never delete it) when a newer set replaces it: the", "old numbers stay reproducible against the wheel they were measured on." ], @@ -49,12 +51,27 @@ }, { "path": "docs/eval_results/current_flow_v1_30_0_local_20260806", - "status": "current", + "status": "superseded", + "superseded_by": "docs/eval_results/current_flow_v1_31_0_local_20260818", "flow_version": "1.30.0", "wheel_sha256": "7bf1a7b00388172a79bda666def182688c296ffb5bc9be2fd3281169fc36ae63", "sdist_sha256": "3a402610e35f47fd54daaf066b8c3a6006c13483cb4c778740014abea1854ea4", "evidence_manifest": "docs/eval_results/current_flow_v1_30_0_local_20260806/EVIDENCE_MANIFEST.json", - "measured_on": "2026-08-06" + "measured_on": "2026-08-06", + "production_acceptance": false, + "stale_reason": "These retained results measured Flow 1.30.0. They were not relabeled. A complete Flow 1.31.0 rerun replaced them after the published release changed." + }, + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818", + "status": "current", + "flow_version": "1.31.0", + "flow_source_commit": "2d225dea9a0ad29ca84ce1b037cc0ac671367e28", + "flow_release_tag": "v1.31.0", + "wheel_sha256": "81133db1528ad1bb1f26e3fcb6aea61b0651db6d905cf2e4943e8383c1f3d29c", + "sdist_sha256": "cf1fc356d14d267df82be188de3e9a3575734f18f46ef91ac8075438cc731540", + "evidence_manifest": "docs/eval_results/current_flow_v1_31_0_local_20260818/EVIDENCE_MANIFEST.json", + "measured_on": "2026-08-18", + "production_acceptance": false } ] } diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/EVIDENCE_MANIFEST.json b/docs/eval_results/current_flow_v1_31_0_local_20260818/EVIDENCE_MANIFEST.json new file mode 100644 index 0000000..f1d50b7 --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/EVIDENCE_MANIFEST.json @@ -0,0 +1,487 @@ +{ + "artifacts": [ + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818/dependency-freeze.txt", + "sha256": "215571cdf516e7806338348ad72b387730c629e071abb78817d70b147f554b9b" + }, + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/dependency-freeze.txt", + "sha256": "215571cdf516e7806338348ad72b387730c629e071abb78817d70b147f554b9b" + }, + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/results.json", + "sha256": "2fe5250f4d2be832d9e2872ecf9fcb0a2909ea35d9ee64cd423fabd998b28a7c" + }, + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818/replication/dependency-freeze.txt", + "sha256": "215571cdf516e7806338348ad72b387730c629e071abb78817d70b147f554b9b" + }, + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818/replication/results.json", + "sha256": "c4490e2849609c41d608f78aea76ae4d5417d30063cccd5d8797cc6a9bc3e0e8" + }, + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818/replication/theme_postcondition_over_halt.json", + "sha256": "ae239cfb7b96ed64e576177a020e17d57e0b22d1ff3914b430e478263391e6fe" + }, + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818/results.json", + "sha256": "266706f2a0b4993bb49d0f33567c2a9b53266847b29fc02d4a3fc2b83fbf9306" + }, + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818/theme_postcondition_over_halt.json", + "sha256": "ae239cfb7b96ed64e576177a020e17d57e0b22d1ff3914b430e478263391e6fe" + }, + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/dependency-freeze.txt", + "sha256": "215571cdf516e7806338348ad72b387730c629e071abb78817d70b147f554b9b" + }, + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/results.json", + "sha256": "361e3be35f7959ab88024d25cbce558b31ac62768f29141efabf0e8531bcb896" + } + ], + "campaigns": [ + { + "environment": { + "platform": "macOS-15.7.3-arm64-arm-64bit", + "python": "3.12.13" + }, + "evidence_scope": { + "class": "contract_fixture", + "customer_workflow": false, + "hosted_execution": false, + "production_acceptance": false, + "real_remote_session": false + }, + "metric_coverage": { + "over_halt": "not_counted", + "silent_incorrect_success": "not_counted" + }, + "name": "remote_lease_safety", + "results_path": "docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/results.json", + "runtime": { + "dependency_snapshot_filename": "dependency-freeze.txt", + "dependency_snapshot_sha256": "215571cdf516e7806338348ad72b387730c629e071abb78817d70b147f554b9b", + "openadapt_types_version": "0.9.0", + "python_version": "3.12.13" + }, + "verifier_path": "scripts/probe_remote_lease_safety.py" + }, + { + "environment": { + "chromium": "Playwright-managed headless Chromium", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "playwright": "1.61.0", + "python": "3.12.13" + }, + "evidence_scope": { + "class": "local_synthetic", + "customer_workflow": false, + "hosted_execution": false, + "production_acceptance": false, + "real_remote_session": false + }, + "metric_coverage": { + "over_halt": "counted", + "silent_incorrect_success": "counted" + }, + "name": "replication", + "results_path": "docs/eval_results/current_flow_v1_31_0_local_20260818/replication/results.json", + "runtime": { + "browser_revision": "1228", + "browser_version": "149.0.7827.55", + "dependency_snapshot_filename": "dependency-freeze.txt", + "dependency_snapshot_sha256": "215571cdf516e7806338348ad72b387730c629e071abb78817d70b147f554b9b", + "openadapt_types_version": "0.9.0", + "python_version": "3.12.13" + }, + "verifier_path": "scripts/run_current_flow_local_benchmark.py" + }, + { + "environment": { + "chromium": "Playwright-managed headless Chromium", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "playwright": "1.61.0", + "python": "3.12.13" + }, + "evidence_scope": { + "class": "local_synthetic", + "customer_workflow": false, + "hosted_execution": false, + "production_acceptance": false, + "real_remote_session": false + }, + "metric_coverage": { + "over_halt": "counted", + "silent_incorrect_success": "counted" + }, + "name": "comparison", + "results_path": "docs/eval_results/current_flow_v1_31_0_local_20260818/results.json", + "runtime": { + "browser_revision": "1228", + "browser_version": "149.0.7827.55", + "dependency_snapshot_filename": "dependency-freeze.txt", + "dependency_snapshot_sha256": "215571cdf516e7806338348ad72b387730c629e071abb78817d70b147f554b9b", + "openadapt_types_version": "0.9.0", + "python_version": "3.12.13" + }, + "verifier_path": "scripts/run_current_flow_local_benchmark.py" + }, + { + "environment": { + "chromium": "Playwright-managed headless Chromium", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "playwright": "1.61.0", + "python": "3.12.13" + }, + "evidence_scope": { + "class": "local_synthetic", + "customer_workflow": false, + "hosted_execution": false, + "production_acceptance": false, + "real_remote_session": false + }, + "metric_coverage": { + "over_halt": "not_counted", + "silent_incorrect_success": "not_counted" + }, + "name": "transaction_probe", + "results_path": "docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/results.json", + "runtime": { + "browser_revision": "1228", + "browser_version": "149.0.7827.55", + "dependency_snapshot_filename": "dependency-freeze.txt", + "dependency_snapshot_sha256": "215571cdf516e7806338348ad72b387730c629e071abb78817d70b147f554b9b", + "openadapt_types_version": "0.9.0", + "python_version": "3.12.13" + }, + "verifier_path": "scripts/run_flow_transaction_probe.py" + } + ], + "evals_commit": "1082192e9c2ec299d31330608c1016a939d3b88d", + "flow": { + "release_tag": "v1.31.0", + "sdist_sha256": "cf1fc356d14d267df82be188de3e9a3575734f18f46ef91ac8075438cc731540", + "source_commit": "2d225dea9a0ad29ca84ce1b037cc0ac671367e28", + "version": "1.31.0", + "wheel_sha256": "81133db1528ad1bb1f26e3fcb6aea61b0651db6d905cf2e4943e8383c1f3d29c" + }, + "public_reports": [ + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818/REPORT.md", + "sha256": "7f8e74dc795f71f3dd242e4f6252c1e6cd6fd1dce43708b80a6f4fb39ffefa06" + }, + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818/REPRODUCE.md", + "sha256": "4af74592a4697d2cb4817833d769efc5b3c798d6776c9b0c8b26abe9b03ec652" + }, + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/REPORT.md", + "sha256": "d68bd07b9fe2a9a6584ea82be28811a6a737675053bff2c9898bca1ab4441f47" + }, + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818/replication/REPORT.md", + "sha256": "907ecd345302b4ec0baaa35cc8295b149ef8cb86958f780a946fa868fef61c48" + }, + { + "path": "docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/REPORT.md", + "sha256": "5eb33943d36a261853fb5b328bbd255a61d9c25ee82aea3349e381e08960a168" + } + ], + "schema_version": 3, + "task_contracts": [ + { + "results_path": "docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/results.json", + "sha256": "108c3c9b003db7febc9bd3d0e0f4b6b3948d2eac75344cd41d8ddea893df21a2", + "value": { + "caveats": [ + "The backend is a fake implementing only the two-phase remote actuation lease; this measures a runtime contract, not a real remote session.", + "One synthetic single-step workflow; resolution is scripted so the delivery decision is the only variable.", + "No wrong-target immunity, identity coverage, or hosted lifecycle claim is made here." + ], + "cells": [ + "ungoverned_lease", + "governed_lease", + "lease_frame_changed" + ], + "invariants": [ + { + "applicable_count": 3, + "holds": true, + "id": "governed_run_refuses_receiptless_remote_write", + "statement": "A governed run must refuse a consequential remote click on a backend that cannot bind its exact fresh frame and target to delivery, before the first input edge.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 9, + "holds": true, + "id": "lease_delivery_is_never_a_production_success", + "statement": "A remote write delivered through the frame lease alone carries no typed receipt, so no execution profile may classify it VERIFIED or production-eligible.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 3, + "holds": true, + "id": "lease_delivery_carries_no_receipt_or_actuation_tier", + "statement": "A lease-only delivery must leave the result unlabeled: no typed delivery receipt and no recorded actuation tier.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 3, + "holds": true, + "id": "changed_frame_aborts_delivery", + "statement": "The lease is the safety property: a remote frame that changed between the lease and the input edge must stop delivery.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 9, + "holds": true, + "id": "zero_model_calls_and_cost", + "statement": "A compiled remote replay makes no model calls and costs $0.", + "vacuous": false, + "violation_count": 0, + "violations": [] + } + ], + "paid_or_remote_mutations": [], + "profiles": [ + "demo", + "standard", + "regulated" + ], + "scope": "remote frame-lease delivery safety properties (runtime contract)", + "trials_per_cell": 3 + } + }, + { + "results_path": "docs/eval_results/current_flow_v1_31_0_local_20260818/replication/results.json", + "sha256": "f76e277a4550d273f903fe478ac872f7d79d28e08c4a95d128a34e652aab220d", + "value": { + "arms": [ + "compiled", + "dom", + "dom_named" + ], + "caveats": [ + "Synthetic MockMed, one workflow, one macOS host, headless Chromium.", + "Published Flow wheel and release-tagged source are exact-bound.", + "Warm OS/browser caches remain after the first fresh-browser launch.", + "Final screenshot bytes were discarded after their hashes were retained.", + "No hosted, Windows, RDP, Citrix, or customer workflow is represented." + ], + "conditions": [ + "clean", + "theme", + "rename" + ], + "identity_coverage": { + "applicable_steps": 8, + "armed_steps": 5, + "unarmed": [ + { + "reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "step_id": "step_000" + }, + { + "reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "step_id": "step_002" + }, + { + "reason": "the only readable row text is the target's own label (mutable evidence, excluded from identity)", + "step_id": "step_004" + } + ] + }, + "oracle": "arm-independent screenshot/OCR final-state check: exact note and saved banner, Triage row, intended patient, no wrong-type write", + "outcome_definitions": { + "correct": "arm reported completion and independent oracle confirmed effect", + "halt_or_error": "arm reported halt/incomplete and independent oracle found effect absent", + "over_halt": "arm reported halt/incomplete but independent oracle confirmed effect", + "silent_incorrect_success": "arm reported completion but independent oracle did not confirm effect", + "wrong_action": "independent oracle observed a wrong-patient or wrong-type write" + }, + "paid_or_remote_mutations": [], + "scope": "exact-current local deterministic runtime overhead and robustness", + "task": "MockMed triage: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "trials_per_arm_condition": 3 + } + }, + { + "results_path": "docs/eval_results/current_flow_v1_31_0_local_20260818/results.json", + "sha256": "f76e277a4550d273f903fe478ac872f7d79d28e08c4a95d128a34e652aab220d", + "value": { + "arms": [ + "compiled", + "dom", + "dom_named" + ], + "caveats": [ + "Synthetic MockMed, one workflow, one macOS host, headless Chromium.", + "Published Flow wheel and release-tagged source are exact-bound.", + "Warm OS/browser caches remain after the first fresh-browser launch.", + "Final screenshot bytes were discarded after their hashes were retained.", + "No hosted, Windows, RDP, Citrix, or customer workflow is represented." + ], + "conditions": [ + "clean", + "theme", + "rename" + ], + "identity_coverage": { + "applicable_steps": 8, + "armed_steps": 5, + "unarmed": [ + { + "reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "step_id": "step_000" + }, + { + "reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "step_id": "step_002" + }, + { + "reason": "the only readable row text is the target's own label (mutable evidence, excluded from identity)", + "step_id": "step_004" + } + ] + }, + "oracle": "arm-independent screenshot/OCR final-state check: exact note and saved banner, Triage row, intended patient, no wrong-type write", + "outcome_definitions": { + "correct": "arm reported completion and independent oracle confirmed effect", + "halt_or_error": "arm reported halt/incomplete and independent oracle found effect absent", + "over_halt": "arm reported halt/incomplete but independent oracle confirmed effect", + "silent_incorrect_success": "arm reported completion but independent oracle did not confirm effect", + "wrong_action": "independent oracle observed a wrong-patient or wrong-type write" + }, + "paid_or_remote_mutations": [], + "scope": "exact-current local deterministic runtime overhead and robustness", + "task": "MockMed triage: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "trials_per_arm_condition": 3 + } + }, + { + "results_path": "docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/results.json", + "sha256": "79dd9acdaab30a6cb0c6717f6d390796db4570725901160c7db5a3d2c8b01102", + "value": { + "caveats": [ + "Synthetic MockMed fault server, one workflow, one macOS host, headless Chromium.", + "Published Flow wheel and release-tagged source are exact-bound.", + "The effect contract in the effect_verified cells is authored by this script.", + "No hosted, Windows, RDP, Citrix, or customer workflow is represented." + ], + "fault_modes": [ + "ok", + "timeout", + "optimistic", + "session", + "duplicate" + ], + "invariants": [ + { + "applicable_count": 12, + "holds": true, + "id": "completed_unverified_is_never_success", + "statement": "COMPLETED_UNVERIFIED must never be reported as a production success and must never be billable.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 30, + "holds": true, + "id": "only_verified_is_billable", + "statement": "Only VERIFIED may be billable.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 0, + "holds": true, + "id": "no_false_absence_claim", + "statement": "An outcome that asserts no business effect occurred (HALTED_BEFORE_EFFECT / FAILED_PLATFORM / CANCELED / REJECTED_POLICY) must not be reported when the system of record shows the write landed.", + "vacuous": true, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 21, + "holds": true, + "id": "unverified_delivered_write_needs_reconciliation", + "statement": "A consequential step that reached actuation but whose effect was never verified cannot be classified as a proven absence; it must be RECONCILIATION_REQUIRED.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 24, + "holds": true, + "id": "no_blind_retry_of_consequential_write", + "statement": "A single run must never write the intended record more than once. A duplicate row from a run whose backend is NOT the double-accepting 'duplicate' mode would be a blind retry.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 0, + "holds": true, + "id": "verified_requires_a_real_effect", + "statement": "VERIFIED must never be reported when nothing landed.", + "vacuous": true, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 30, + "holds": true, + "id": "zero_model_calls_and_cost", + "statement": "A healthy compiled run makes no model calls and costs $0.", + "vacuous": false, + "violation_count": 0, + "violations": [] + } + ], + "oracle": "system-of-record snapshot delta at GET /api/db: exactly one new Triage encounter for the intended patient carrying this run's note, with no pre-existing row destroyed", + "paid_or_remote_mutations": [], + "scope": "transaction outcome taxonomy against a real persistence boundary", + "task": "MockMed triage against the fault-injection persistence boundary: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "trials_per_cell": 3, + "verification_modes": [ + "unverified", + "effect_verified" + ] + } + } + ], + "verifiers": [ + { + "path": "scripts/run_current_flow_local_benchmark.py", + "sha256": "fe49e0d9fc9ebf6d66120fc8f16683390071b96af6d252b82afbb94bb7449119" + }, + { + "path": "scripts/run_flow_transaction_probe.py", + "sha256": "e27ecc7fcda287c9f946bf002f02443c8628c3a2cbc50f88ed03b1ad3abf6a42" + }, + { + "path": "scripts/probe_remote_lease_safety.py", + "sha256": "d75c1c62248818d937705e3a7b3170b09ea211b54591f37a57520018b25a7388" + }, + { + "path": "scripts/extract_over_halt_regression.py", + "sha256": "7712798260fbf8ff50b96b00b71c7dbd5a509f7dd7553a63a0e6c791635b2554" + }, + { + "path": "scripts/evidence_runtime.py", + "sha256": "334839a541c46a52892724d9285b295e503a52667899295f831dc154a887dcdd" + } + ] +} diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/REPORT.md b/docs/eval_results/current_flow_v1_31_0_local_20260818/REPORT.md new file mode 100644 index 0000000..156967a --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/REPORT.md @@ -0,0 +1,70 @@ +# Exact-current local Flow performance + +This is a deterministic-runtime overhead and bounded robustness comparison. It is **not** a zero-shot computer-use comparison. + +## Source and environment + +- Flow commit: `2d225dea9a0ad29ca84ce1b037cc0ac671367e28` (version `1.31.0`; tracked-clean source) +- Release tag: `v1.31.0` +- Wheel SHA-256: `81133db1528ad1bb1f26e3fcb6aea61b0651db6d905cf2e4943e8383c1f3d29c` +- Evals base commit: `1082192e9c2ec299d31330608c1016a939d3b88d` +- Runner SHA-256: `fe49e0d9fc9ebf6d66120fc8f16683390071b96af6d252b82afbb94bb7449119` +- Platform: `macOS-15.7.3-arm64-arm-64bit` +- Python: `3.12.13` +- Playwright: `1.61.0` +- Chromium: `Playwright-managed headless Chromium` +- Network/provider use: loopback bundled MockMed only; no cloud VM, hosted runner, or model API + +## Counted result + +One synthetic MockMed workflow: sign in, open the intended referral, create a Triage encounter, enter a trial-unique note, and save. Each arm used a fresh browser. The arm-independent screenshot/OCR oracle required the exact saved note, Triage row, and intended patient, and separately flagged wrong-target writes. + +| Condition | Arm | Runs | Task success | Silent incorrect | Wrong action | Over-halt | Halt/error | Steady median | Steady p95 | End-to-end median | End-to-end p95 | Model calls | Cost | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `clean` | compiled replay | 3 | 3/3 | 0 | 0 | 0 | 0 | 6.751s | 6.819s | 7.406s | 7.569s | 0 | $0.00 | +| `clean` | DOM positional | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.217s | 0.237s | 0.924s | 0.933s | 0 | $0.00 | +| `clean` | DOM name-scoped | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.217s | 0.218s | 0.927s | 0.965s | 0 | $0.00 | +| `theme` | compiled replay | 3 | 3/3 | 0 | 0 | 0 | 0 | 6.902s | 7.336s | 7.573s | 7.961s | 0 | $0.00 | +| `theme` | DOM positional | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.215s | 0.216s | 0.932s | 1.138s | 0 | $0.00 | +| `theme` | DOM name-scoped | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.221s | 0.221s | 0.981s | 1.319s | 0 | $0.00 | +| `rename` | compiled replay | 3 | 3/3 | 0 | 0 | 0 | 0 | 6.891s | 7.116s | 7.703s | 7.947s | 0 | $0.00 | +| `rename` | DOM positional | 3 | 0/3 | 0 | 0 | 0 | 3 | 30.082s | 30.085s | 30.791s | 30.893s | 0 | $0.00 | +| `rename` | DOM name-scoped | 3 | 0/3 | 0 | 0 | 0 | 3 | 30.080s | 30.084s | 30.770s | 30.790s | 0 | $0.00 | + +Steady time wraps only the replay/script action loop. End-to-end time additionally includes fresh browser launch, the independent final screenshot/OCR oracle, and browser teardown. Local server startup and one-time record/compile are excluded from both and reported separately below. p95 is nearest-rank; with n=3 it is the slowest counted trial. + +Outcome definitions: **silent incorrect success** means the arm reported completion but the independent oracle did not confirm the intended effect; **wrong action** means the oracle observed a write to the wrong patient or encounter type; **over-halt** means the arm reported halt/incomplete while the independent oracle confirmed the intended effect; **halt/error** means the arm stopped and the intended effect was absent. The same definitions apply to every arm. The selector controls therefore can count as over-halts when alternate final-state evidence contradicts them; their rename failures did not, because the oracle confirmed no write. + +## One-time setup + +- Bundled app server startup: 0.003s (n=1 diagnostic) +- Setup trial 1: record 2.614s; compile 7.564s; bundle `205a91df82fd3c2b7d2b29553f9fbac1fd577407b591ef55556126e1646e5d8b` +- Setup trial 2: record 2.505s; compile 7.096s; bundle `4e010fd848d8e8b4737251f41d430133f6c77b239b51151c2869b79cd6e7305a` +- Setup trial 3: record 2.633s; compile 7.819s; bundle `e62dc18a45c067235acc5a331a13004df133adce5c61e2faa57a44c7b9a06490` + +## What this establishes + +- The independent final-state oracle confirmed the intended effect for every compiled clean, theme-drift, and label-drift trial. Compiled theme runs reported 0/3 over-halts; these are counted rather than relabelled as clean completions. +- The selector controls are steelmanned Playwright scripts. Their clean/theme speed is the correct reminder that API/structural actuation should remain the preferred tier where available. +- The `rename` surface changes `Open` to `View` and `Save Encounter` to `Submit Encounter`. Both selector controls failed loudly at the first renamed locator before mutation. These are unsupported-drift halts, not silent wrong actions. +- Label drift is an intentionally bounded robustness probe, not evidence for arbitrary drift or arbitrary applications. +- The compiled bundle had 5/8 identity-applicable clicks armed. Task completion is not a universal wrong-target-immunity claim for unarmed steps. +- The exact theme postcondition failures are retained in `theme_postcondition_over_halt.json` as a compact regression artifact. + +## Exact-current Flow versus zero-shot: not run + +No current Flow-versus-zero-shot result is claimed. The Azure WAA VM was not started and no model was called. The existing `scripts/eval_flow_on_waa.py` live replay path currently leaves the WAA evaluator unwired (so it cannot independently score success), while its hybrid live path explicitly returns before execution because the adapter is not wired. A valid future run therefore requires, in order: + +1. Wire `WAALiveAdapter.evaluate` into the Flow replay path and wire the same model/adapter into the zero-shot arm. +2. Prepare one exact compiled bundle per retained task and bind Flow/evals/model/environment revisions in the run manifest. +3. Obtain explicit approval for Azure VM start and a hard model spend cap; then confirm VM snapshot/readiness without changing the task set. +4. Run at least three trials per task per condition for both arms, using WAA's evaluator as the oracle and recording correct, silent incorrect, over-halt, timeout/error, latency, tokens, and cost. +5. Deallocate the VM, verify no orphaned resources, and publish the immutable raw summaries plus a normalized report. + +## Caveats + +- Synthetic MockMed, one workflow, one macOS host, headless Chromium. +- This report runs the exact published wheel named above, extracted locally, and binds it to the release-tagged tracked-clean source. +- Browser startup is fresh per run but OS/browser caches are warm after the first launch; the Latin-rotated arm order reduces but does not eliminate host-order effects. +- Final screenshots were inspected and hashed during the run; only their SHA-256 hashes are retained in this committed report. +- No hosted lifecycle, Windows UIA, RDP, Citrix, or real customer application is represented. diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/REPRODUCE.md b/docs/eval_results/current_flow_v1_31_0_local_20260818/REPRODUCE.md new file mode 100644 index 0000000..572f66d --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/REPRODUCE.md @@ -0,0 +1,47 @@ +# Reproduce the Flow 1.31.0 local evidence + +This set uses the exact published `openadapt-flow` 1.31.0 wheel and the exact +release-tagged source at `2d225dea9a0ad29ca84ce1b037cc0ac671367e28`. +It uses the synthetic MockMed fixture, loopback network traffic, and a local +headless Chromium browser. It uses no paid service and no customer data. + +Create a clean Python 3.12 environment. Install +`openadapt-flow[browser]==1.31.0`, `openadapt-types==0.9.0`, +`playwright==1.61.0`, and `requests`. Install Playwright Chromium revision +`1228`. Download the Flow wheel and source archive without changing them, and +check out the `v1.31.0` source tag. + +Run these commands from Evals commit +`1082192e9c2ec299d31330608c1016a939d3b88d`: + +```bash +python scripts/run_current_flow_local_benchmark.py \ + --flow-source /path/to/openadapt-flow-v1.31.0 \ + --flow-wheel /path/to/openadapt_flow-1.31.0-py3-none-any.whl \ + --out out/current + +python scripts/run_current_flow_local_benchmark.py \ + --flow-source /path/to/openadapt-flow-v1.31.0 \ + --flow-wheel /path/to/openadapt_flow-1.31.0-py3-none-any.whl \ + --out out/current/replication + +python scripts/run_flow_transaction_probe.py \ + --flow-source /path/to/openadapt-flow-v1.31.0 \ + --flow-wheel /path/to/openadapt_flow-1.31.0-py3-none-any.whl \ + --out out/current/transaction_probe + +python scripts/probe_remote_lease_safety.py \ + --flow-source /path/to/openadapt-flow-v1.31.0 \ + --flow-wheel /path/to/openadapt_flow-1.31.0-py3-none-any.whl \ + --out out/current/remote_lease_safety +``` + +Each campaign retains its installed-package freeze, Python version, +`openadapt-types` version, browser version when applicable, and exact +Playwright Chromium revision. `EVIDENCE_MANIFEST.json` binds every verifier, +result, replication artifact, report, task/oracle contract, environment, +dependency snapshot, reliability-metric coverage, and maturity boundary. + +The manifest marks every campaign as `production_acceptance: false`. These are +local synthetic or contract-fixture results. They do not establish hosted, +customer-workflow, RDP-session, or Citrix-session production acceptance. diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/dependency-freeze.txt b/docs/eval_results/current_flow_v1_31_0_local_20260818/dependency-freeze.txt new file mode 100644 index 0000000..163a80e --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/dependency-freeze.txt @@ -0,0 +1,37 @@ +annotated-types==0.8.0 +anyio==4.14.2 +certifi==2026.7.22 +cffi==2.1.1 +charset-normalizer==3.5.1 +cryptography==50.0.0 +flatbuffers==25.12.19 +greenlet==3.5.5 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.19 +numpy==2.5.2 +onnxruntime==1.29.0 +openadapt-flow @ file:///private/tmp/openadapt_flow-1.31.0-py3-none-any.whl +openadapt-types==0.9.0 +opencv-python==5.0.0.93 +opencv-python-headless==5.0.0.93 +packaging==26.3 +pillow==12.3.0 +pip==26.2.1 +playwright==1.61.0 +protobuf==7.35.1 +pyclipper==1.4.0 +pycparser==3.0 +pydantic==2.13.4 +pydantic_core==2.46.4 +pyee==13.0.1 +PyYAML==6.0.3 +rapidocr-onnxruntime==1.4.4 +requests==2.34.2 +shapely==2.1.2 +six==1.17.0 +tqdm==4.70.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.7.0 diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/REPORT.md b/docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/REPORT.md new file mode 100644 index 0000000..8903731 --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/REPORT.md @@ -0,0 +1,53 @@ +# Remote frame-lease delivery: the two safety properties, measured + +openadapt-flow `c9618cc` (1.28.0) lets a consequential remote click be +delivered through the backend's one-shot actuation lease when no typed +delivery receipt exists. That relaxes a refusal, so its two stated +safety properties are measured here rather than trusted. + +## Source and environment + +- Flow commit: `2d225dea9a0ad29ca84ce1b037cc0ac671367e28` (version `1.31.0`; tracked-clean source) +- Release tag: `v1.31.0` +- Wheel SHA-256: `81133db1528ad1bb1f26e3fcb6aea61b0651db6d905cf2e4943e8383c1f3d29c` +- Evals base commit: `1082192e9c2ec299d31330608c1016a939d3b88d` +- Runner SHA-256: `d75c1c62248818d937705e3a7b3170b09ea211b54591f37a57520018b25a7388` +- Platform: `macOS-15.7.3-arm64-arm-64bit` +- Python: `3.12.13` +- Network/provider use: none. No server, no browser, no model API. + +## Counted result + +3 trials per cell. `input edges` counts what the backend was actually asked to deliver. + +| Cell | Runs | Delivered | Refused | Receipts | Actuation tier | `transaction_outcome` (demo / standard / regulated) | +|---|---:|---:|---:|---:|---|---| +| `ungoverned_lease` | 3 | 3 | 0 | 0 | None | COMPLETED_UNVERIFIED / COMPLETED_UNVERIFIED / COMPLETED_UNVERIFIED | +| `governed_lease` | 3 | 0 | 3 | 0 | None | HALTED_BEFORE_EFFECT / HALTED_BEFORE_EFFECT / HALTED_BEFORE_EFFECT | +| `lease_frame_changed` | 3 | 0 | 3 | 0 | None | RECONCILIATION_REQUIRED / RECONCILIATION_REQUIRED / RECONCILIATION_REQUIRED | + +## Invariants + +Every invariant states the denominator it was evaluated over. A `vacuous` invariant had no applicable run and proves nothing. + +| Invariant | Applicable runs | Violations | Holds | +|---|---:|---:|---| +| A governed run must refuse a consequential remote click on a backend that cannot bind its exact fresh frame and target to delivery, before the first input edge. | 3 | 0 | yes | +| A remote write delivered through the frame lease alone carries no typed receipt, so no execution profile may classify it VERIFIED or production-eligible. | 9 | 0 | yes | +| A lease-only delivery must leave the result unlabeled: no typed delivery receipt and no recorded actuation tier. | 3 | 0 | yes | +| The lease is the safety property: a remote frame that changed between the lease and the input edge must stop delivery. | 3 | 0 | yes | +| A compiled remote replay makes no model calls and costs $0. | 9 | 0 | yes | + +## Scope + +- The backend is a fake implementing ONLY the two-phase remote + actuation lease, which is the exact protocol surface a pixel-only + no-DOM canvas backend exposes. This measures a runtime contract, not + a real Citrix or RDP session. +- One synthetic single-step workflow whose only step is the + irreversible write. Resolution is scripted to one fixed point so the + delivery decision is the only thing that varies between cells. +- No claim is made here about wrong-target immunity on a real remote + surface, about identity coverage, or about any hosted lifecycle. + +Reproduce: `python scripts/probe_remote_lease_safety.py --flow-source --flow-wheel --out ` diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/dependency-freeze.txt b/docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/dependency-freeze.txt new file mode 100644 index 0000000..163a80e --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/dependency-freeze.txt @@ -0,0 +1,37 @@ +annotated-types==0.8.0 +anyio==4.14.2 +certifi==2026.7.22 +cffi==2.1.1 +charset-normalizer==3.5.1 +cryptography==50.0.0 +flatbuffers==25.12.19 +greenlet==3.5.5 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.19 +numpy==2.5.2 +onnxruntime==1.29.0 +openadapt-flow @ file:///private/tmp/openadapt_flow-1.31.0-py3-none-any.whl +openadapt-types==0.9.0 +opencv-python==5.0.0.93 +opencv-python-headless==5.0.0.93 +packaging==26.3 +pillow==12.3.0 +pip==26.2.1 +playwright==1.61.0 +protobuf==7.35.1 +pyclipper==1.4.0 +pycparser==3.0 +pydantic==2.13.4 +pydantic_core==2.46.4 +pyee==13.0.1 +PyYAML==6.0.3 +rapidocr-onnxruntime==1.4.4 +requests==2.34.2 +shapely==2.1.2 +six==1.17.0 +tqdm==4.70.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.7.0 diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/results.json b/docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/results.json new file mode 100644 index 0000000..7d00fc5 --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/remote_lease_safety/results.json @@ -0,0 +1,590 @@ +{ + "aggregate": { + "governed_lease": { + "actuation_tiers": [ + "None" + ], + "delivered_count": 0, + "delivery_receipt_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "refused_count": 3, + "replayer_success_count": 0, + "transaction_outcomes": { + "demo": [ + "HALTED_BEFORE_EFFECT" + ], + "regulated": [ + "HALTED_BEFORE_EFFECT" + ], + "standard": [ + "HALTED_BEFORE_EFFECT" + ] + } + }, + "lease_frame_changed": { + "actuation_tiers": [ + "None" + ], + "delivered_count": 0, + "delivery_receipt_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "refused_count": 3, + "replayer_success_count": 0, + "transaction_outcomes": { + "demo": [ + "RECONCILIATION_REQUIRED" + ], + "regulated": [ + "RECONCILIATION_REQUIRED" + ], + "standard": [ + "RECONCILIATION_REQUIRED" + ] + } + }, + "ungoverned_lease": { + "actuation_tiers": [ + "None" + ], + "delivered_count": 3, + "delivery_receipt_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "refused_count": 0, + "replayer_success_count": 3, + "transaction_outcomes": { + "demo": [ + "COMPLETED_UNVERIFIED" + ], + "regulated": [ + "COMPLETED_UNVERIFIED" + ], + "standard": [ + "COMPLETED_UNVERIFIED" + ] + } + } + }, + "caveats": [ + "The backend is a fake implementing only the two-phase remote actuation lease; this measures a runtime contract, not a real remote session.", + "One synthetic single-step workflow; resolution is scripted so the delivery decision is the only variable.", + "No wrong-target immunity, identity coverage, or hosted lifecycle claim is made here." + ], + "cells": [ + "ungoverned_lease", + "governed_lease", + "lease_frame_changed" + ], + "environment": { + "platform": "macOS-15.7.3-arm64-arm-64bit", + "python": "3.12.13" + }, + "generated_at": "2026-08-18T15:59:20.153848+00:00", + "invariants": [ + { + "applicable_count": 3, + "holds": true, + "id": "governed_run_refuses_receiptless_remote_write", + "statement": "A governed run must refuse a consequential remote click on a backend that cannot bind its exact fresh frame and target to delivery, before the first input edge.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 9, + "holds": true, + "id": "lease_delivery_is_never_a_production_success", + "statement": "A remote write delivered through the frame lease alone carries no typed receipt, so no execution profile may classify it VERIFIED or production-eligible.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 3, + "holds": true, + "id": "lease_delivery_carries_no_receipt_or_actuation_tier", + "statement": "A lease-only delivery must leave the result unlabeled: no typed delivery receipt and no recorded actuation tier.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 3, + "holds": true, + "id": "changed_frame_aborts_delivery", + "statement": "The lease is the safety property: a remote frame that changed between the lease and the input edge must stop delivery.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 9, + "holds": true, + "id": "zero_model_calls_and_cost", + "statement": "A compiled remote replay makes no model calls and costs $0.", + "vacuous": false, + "violation_count": 0, + "violations": [] + } + ], + "paid_or_remote_mutations": [], + "profiles": [ + "demo", + "standard", + "regulated" + ], + "reproduce": "python scripts/probe_remote_lease_safety.py --flow-source --flow-wheel --out ", + "runs": [ + { + "actuation_tier": null, + "backend_actions": [ + [ + "click", + 110, + 105, + false + ] + ], + "cell": "ungoverned_lease", + "delivery_receipt_present": false, + "governed": false, + "input_edges_delivered": 1, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": true, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + }, + "regulated": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + }, + "standard": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + } + }, + "replayer_success": true, + "safety_halt": false, + "step_error": null, + "trial": 1 + }, + { + "actuation_tier": null, + "backend_actions": [ + [ + "click", + 110, + 105, + false + ] + ], + "cell": "ungoverned_lease", + "delivery_receipt_present": false, + "governed": false, + "input_edges_delivered": 1, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": true, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + }, + "regulated": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + }, + "standard": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + } + }, + "replayer_success": true, + "safety_halt": false, + "step_error": null, + "trial": 2 + }, + { + "actuation_tier": null, + "backend_actions": [ + [ + "click", + 110, + 105, + false + ] + ], + "cell": "ungoverned_lease", + "delivery_receipt_present": false, + "governed": false, + "input_edges_delivered": 1, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": true, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + }, + "regulated": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + }, + "standard": { + "execution_outcome": "COMPLETED_UNVERIFIED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED" + } + }, + "replayer_success": true, + "safety_halt": false, + "step_error": null, + "trial": 3 + }, + { + "actuation_tier": null, + "backend_actions": [], + "cell": "governed_lease", + "delivery_receipt_present": false, + "governed": true, + "input_edges_delivered": 0, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + }, + "regulated": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + }, + "standard": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + } + }, + "replayer_success": false, + "safety_halt": true, + "step_error": "Step 's1' (click 'Save') is a consequential remote click, but this backend cannot bind its exact fresh frame and target to delivery; run aborted", + "trial": 1 + }, + { + "actuation_tier": null, + "backend_actions": [], + "cell": "governed_lease", + "delivery_receipt_present": false, + "governed": true, + "input_edges_delivered": 0, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + }, + "regulated": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + }, + "standard": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + } + }, + "replayer_success": false, + "safety_halt": true, + "step_error": "Step 's1' (click 'Save') is a consequential remote click, but this backend cannot bind its exact fresh frame and target to delivery; run aborted", + "trial": 2 + }, + { + "actuation_tier": null, + "backend_actions": [], + "cell": "governed_lease", + "delivery_receipt_present": false, + "governed": true, + "input_edges_delivered": 0, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + }, + "regulated": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + }, + "standard": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "HALTED_BEFORE_EFFECT" + } + }, + "replayer_success": false, + "safety_halt": true, + "step_error": "Step 's1' (click 'Save') is a consequential remote click, but this backend cannot bind its exact fresh frame and target to delivery; run aborted", + "trial": 3 + }, + { + "actuation_tier": null, + "backend_actions": [], + "cell": "lease_frame_changed", + "delivery_receipt_present": false, + "governed": false, + "input_edges_delivered": 0, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + }, + "regulated": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + }, + "standard": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + } + }, + "replayer_success": false, + "safety_halt": true, + "step_error": "Action delivery was uncertain and was not retried; the complete postcondition and independent effect contract did not confirm the intended outcome", + "trial": 1 + }, + { + "actuation_tier": null, + "backend_actions": [], + "cell": "lease_frame_changed", + "delivery_receipt_present": false, + "governed": false, + "input_edges_delivered": 0, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + }, + "regulated": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + }, + "standard": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + } + }, + "replayer_success": false, + "safety_halt": true, + "step_error": "Action delivery was uncertain and was not retried; the complete postcondition and independent effect contract did not confirm the intended outcome", + "trial": 2 + }, + { + "actuation_tier": null, + "backend_actions": [], + "cell": "lease_frame_changed", + "delivery_receipt_present": false, + "governed": false, + "input_edges_delivered": 0, + "lease_acquisitions": 1, + "model_calls": 0, + "model_cost_usd": 0.0, + "prepared_pointer_points": [ + [ + 110, + 105 + ] + ], + "profiles": { + "demo": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + }, + "regulated": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + }, + "standard": { + "execution_outcome": "HALTED", + "production_eligible": false, + "reported_success": false, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED" + } + }, + "replayer_success": false, + "safety_halt": true, + "step_error": "Action delivery was uncertain and was not retried; the complete postcondition and independent effect contract did not confirm the intended outcome", + "trial": 3 + } + ], + "runtime": { + "dependency_snapshot_filename": "dependency-freeze.txt", + "dependency_snapshot_sha256": "215571cdf516e7806338348ad72b387730c629e071abb78817d70b147f554b9b", + "openadapt_types_version": "0.9.0", + "python_version": "3.12.13" + }, + "schema_version": 1, + "scope": "remote frame-lease delivery safety properties (runtime contract)", + "source": { + "evals": { + "commit": "1082192e9c2ec299d31330608c1016a939d3b88d" + }, + "flow": { + "artifact": { + "filename": "openadapt_flow-1.31.0-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "81133db1528ad1bb1f26e3fcb6aea61b0651db6d905cf2e4943e8383c1f3d29c" + }, + "commit": "2d225dea9a0ad29ca84ce1b037cc0ac671367e28", + "release_tag": "v1.31.0", + "tags": [ + "v1.31.0" + ], + "tracked_clean": true, + "version": "1.31.0" + }, + "runner_sha256": "d75c1c62248818d937705e3a7b3170b09ea211b54591f37a57520018b25a7388" + }, + "trials_per_cell": 3 +} diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/replication/REPORT.md b/docs/eval_results/current_flow_v1_31_0_local_20260818/replication/REPORT.md new file mode 100644 index 0000000..6de843b --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/replication/REPORT.md @@ -0,0 +1,70 @@ +# Exact-current local Flow performance + +This is a deterministic-runtime overhead and bounded robustness comparison. It is **not** a zero-shot computer-use comparison. + +## Source and environment + +- Flow commit: `2d225dea9a0ad29ca84ce1b037cc0ac671367e28` (version `1.31.0`; tracked-clean source) +- Release tag: `v1.31.0` +- Wheel SHA-256: `81133db1528ad1bb1f26e3fcb6aea61b0651db6d905cf2e4943e8383c1f3d29c` +- Evals base commit: `1082192e9c2ec299d31330608c1016a939d3b88d` +- Runner SHA-256: `fe49e0d9fc9ebf6d66120fc8f16683390071b96af6d252b82afbb94bb7449119` +- Platform: `macOS-15.7.3-arm64-arm-64bit` +- Python: `3.12.13` +- Playwright: `1.61.0` +- Chromium: `Playwright-managed headless Chromium` +- Network/provider use: loopback bundled MockMed only; no cloud VM, hosted runner, or model API + +## Counted result + +One synthetic MockMed workflow: sign in, open the intended referral, create a Triage encounter, enter a trial-unique note, and save. Each arm used a fresh browser. The arm-independent screenshot/OCR oracle required the exact saved note, Triage row, and intended patient, and separately flagged wrong-target writes. + +| Condition | Arm | Runs | Task success | Silent incorrect | Wrong action | Over-halt | Halt/error | Steady median | Steady p95 | End-to-end median | End-to-end p95 | Model calls | Cost | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `clean` | compiled replay | 3 | 3/3 | 0 | 0 | 0 | 0 | 7.110s | 7.229s | 7.920s | 7.955s | 0 | $0.00 | +| `clean` | DOM positional | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.217s | 0.224s | 0.948s | 1.052s | 0 | $0.00 | +| `clean` | DOM name-scoped | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.213s | 0.216s | 0.924s | 1.233s | 0 | $0.00 | +| `theme` | compiled replay | 3 | 3/3 | 0 | 0 | 0 | 0 | 8.062s | 8.962s | 8.740s | 9.774s | 0 | $0.00 | +| `theme` | DOM positional | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.215s | 0.218s | 1.053s | 1.134s | 0 | $0.00 | +| `theme` | DOM name-scoped | 3 | 3/3 | 0 | 0 | 0 | 0 | 0.214s | 0.218s | 1.130s | 1.442s | 0 | $0.00 | +| `rename` | compiled replay | 3 | 3/3 | 0 | 0 | 0 | 0 | 7.776s | 9.919s | 8.475s | 10.894s | 0 | $0.00 | +| `rename` | DOM positional | 3 | 0/3 | 0 | 0 | 0 | 3 | 30.080s | 30.087s | 30.779s | 30.914s | 0 | $0.00 | +| `rename` | DOM name-scoped | 3 | 0/3 | 0 | 0 | 0 | 3 | 30.092s | 30.108s | 30.876s | 30.999s | 0 | $0.00 | + +Steady time wraps only the replay/script action loop. End-to-end time additionally includes fresh browser launch, the independent final screenshot/OCR oracle, and browser teardown. Local server startup and one-time record/compile are excluded from both and reported separately below. p95 is nearest-rank; with n=3 it is the slowest counted trial. + +Outcome definitions: **silent incorrect success** means the arm reported completion but the independent oracle did not confirm the intended effect; **wrong action** means the oracle observed a write to the wrong patient or encounter type; **over-halt** means the arm reported halt/incomplete while the independent oracle confirmed the intended effect; **halt/error** means the arm stopped and the intended effect was absent. The same definitions apply to every arm. The selector controls therefore can count as over-halts when alternate final-state evidence contradicts them; their rename failures did not, because the oracle confirmed no write. + +## One-time setup + +- Bundled app server startup: 0.004s (n=1 diagnostic) +- Setup trial 1: record 2.987s; compile 21.354s; bundle `380cdecdf04dcd29f6d40dc93fa8788def26a61f58312259ce571a8bcc12c1d1` +- Setup trial 2: record 3.208s; compile 9.884s; bundle `62520e2519f3974c911f49627d8ca16280c9685d90721f25203618b917ab8a4d` +- Setup trial 3: record 2.787s; compile 9.862s; bundle `28ed00717caafe85e770e56909c41ecf2314ff61b9e601985a83a39d9e7f455a` + +## What this establishes + +- The independent final-state oracle confirmed the intended effect for every compiled clean, theme-drift, and label-drift trial. Compiled theme runs reported 0/3 over-halts; these are counted rather than relabelled as clean completions. +- The selector controls are steelmanned Playwright scripts. Their clean/theme speed is the correct reminder that API/structural actuation should remain the preferred tier where available. +- The `rename` surface changes `Open` to `View` and `Save Encounter` to `Submit Encounter`. Both selector controls failed loudly at the first renamed locator before mutation. These are unsupported-drift halts, not silent wrong actions. +- Label drift is an intentionally bounded robustness probe, not evidence for arbitrary drift or arbitrary applications. +- The compiled bundle had 5/8 identity-applicable clicks armed. Task completion is not a universal wrong-target-immunity claim for unarmed steps. +- The exact theme postcondition failures are retained in `theme_postcondition_over_halt.json` as a compact regression artifact. + +## Exact-current Flow versus zero-shot: not run + +No current Flow-versus-zero-shot result is claimed. The Azure WAA VM was not started and no model was called. The existing `scripts/eval_flow_on_waa.py` live replay path currently leaves the WAA evaluator unwired (so it cannot independently score success), while its hybrid live path explicitly returns before execution because the adapter is not wired. A valid future run therefore requires, in order: + +1. Wire `WAALiveAdapter.evaluate` into the Flow replay path and wire the same model/adapter into the zero-shot arm. +2. Prepare one exact compiled bundle per retained task and bind Flow/evals/model/environment revisions in the run manifest. +3. Obtain explicit approval for Azure VM start and a hard model spend cap; then confirm VM snapshot/readiness without changing the task set. +4. Run at least three trials per task per condition for both arms, using WAA's evaluator as the oracle and recording correct, silent incorrect, over-halt, timeout/error, latency, tokens, and cost. +5. Deallocate the VM, verify no orphaned resources, and publish the immutable raw summaries plus a normalized report. + +## Caveats + +- Synthetic MockMed, one workflow, one macOS host, headless Chromium. +- This report runs the exact published wheel named above, extracted locally, and binds it to the release-tagged tracked-clean source. +- Browser startup is fresh per run but OS/browser caches are warm after the first launch; the Latin-rotated arm order reduces but does not eliminate host-order effects. +- Final screenshots were inspected and hashed during the run; only their SHA-256 hashes are retained in this committed report. +- No hosted lifecycle, Windows UIA, RDP, Citrix, or real customer application is represented. diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/replication/dependency-freeze.txt b/docs/eval_results/current_flow_v1_31_0_local_20260818/replication/dependency-freeze.txt new file mode 100644 index 0000000..163a80e --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/replication/dependency-freeze.txt @@ -0,0 +1,37 @@ +annotated-types==0.8.0 +anyio==4.14.2 +certifi==2026.7.22 +cffi==2.1.1 +charset-normalizer==3.5.1 +cryptography==50.0.0 +flatbuffers==25.12.19 +greenlet==3.5.5 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.19 +numpy==2.5.2 +onnxruntime==1.29.0 +openadapt-flow @ file:///private/tmp/openadapt_flow-1.31.0-py3-none-any.whl +openadapt-types==0.9.0 +opencv-python==5.0.0.93 +opencv-python-headless==5.0.0.93 +packaging==26.3 +pillow==12.3.0 +pip==26.2.1 +playwright==1.61.0 +protobuf==7.35.1 +pyclipper==1.4.0 +pycparser==3.0 +pydantic==2.13.4 +pydantic_core==2.46.4 +pyee==13.0.1 +PyYAML==6.0.3 +rapidocr-onnxruntime==1.4.4 +requests==2.34.2 +shapely==2.1.2 +six==1.17.0 +tqdm==4.70.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.7.0 diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/replication/results.json b/docs/eval_results/current_flow_v1_31_0_local_20260818/replication/results.json new file mode 100644 index 0000000..b41be13 --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/replication/results.json @@ -0,0 +1,1118 @@ +{ + "aggregate": { + "compiled": { + "clean": { + "browser_oracle_teardown_overhead_s_median": 0.8096417909837328, + "end_to_end_wall_s_median": 7.919623458001297, + "end_to_end_wall_s_p95_nearest_rank": 7.9546537910355255, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 7.109981667017564, + "steady_wall_s_p95_nearest_rank": 7.228747875022236, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "rename": { + "browser_oracle_teardown_overhead_s_median": 0.8359982910333201, + "end_to_end_wall_s_median": 8.47529512498295, + "end_to_end_wall_s_p95_nearest_rank": 10.894376124953851, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 7.776412124978378, + "steady_wall_s_p95_nearest_rank": 9.919221874966752, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "theme": { + "browser_oracle_teardown_overhead_s_median": 0.7007425000192598, + "end_to_end_wall_s_median": 8.739939458027948, + "end_to_end_wall_s_p95_nearest_rank": 9.774056624970399, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 8.062472417019308, + "steady_wall_s_p95_nearest_rank": 8.962266792019363, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + } + }, + "dom": { + "clean": { + "browser_oracle_teardown_overhead_s_median": 0.7311412499984726, + "end_to_end_wall_s_median": 0.9483596250065602, + "end_to_end_wall_s_p95_nearest_rank": 1.0523889170144685, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.2172183750080876, + "steady_wall_s_p95_nearest_rank": 0.2237619999796152, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "rename": { + "browser_oracle_teardown_overhead_s_median": 0.6919755830313079, + "end_to_end_wall_s_median": 30.779383666987997, + "end_to_end_wall_s_p95_nearest_rank": 30.914333416963927, + "failure_taxonomy": { + "halt_or_error": 3 + }, + "halt_or_error_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 0, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 30.079736291023437, + "steady_wall_s_p95_nearest_rank": 30.08740808395669, + "task_success_count": 0, + "task_success_rate": 0.0, + "wrong_action_count": 0 + }, + "theme": { + "browser_oracle_teardown_overhead_s_median": 0.8391102909808978, + "end_to_end_wall_s_median": 1.0527942909975536, + "end_to_end_wall_s_p95_nearest_rank": 1.1343001250061207, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.21523858298314735, + "steady_wall_s_p95_nearest_rank": 0.21841358300298452, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + } + }, + "dom_named": { + "clean": { + "browser_oracle_teardown_overhead_s_median": 0.7110636670258828, + "end_to_end_wall_s_median": 0.9244759170105681, + "end_to_end_wall_s_p95_nearest_rank": 1.2328357090009376, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.21341224998468533, + "steady_wall_s_p95_nearest_rank": 0.2157305409782566, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "rename": { + "browser_oracle_teardown_overhead_s_median": 0.7678752489737235, + "end_to_end_wall_s_median": 30.87557391601149, + "end_to_end_wall_s_p95_nearest_rank": 30.998797707958147, + "failure_taxonomy": { + "halt_or_error": 3 + }, + "halt_or_error_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 0, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 30.09167991700815, + "steady_wall_s_p95_nearest_rank": 30.107698667037766, + "task_success_count": 0, + "task_success_rate": 0.0, + "wrong_action_count": 0 + }, + "theme": { + "browser_oracle_teardown_overhead_s_median": 0.9116156260133721, + "end_to_end_wall_s_median": 1.1300369590171613, + "end_to_end_wall_s_p95_nearest_rank": 1.4417413329938427, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.2138964160112664, + "steady_wall_s_p95_nearest_rank": 0.2184213330037892, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + } + } + }, + "arms": [ + "compiled", + "dom", + "dom_named" + ], + "caveats": [ + "Synthetic MockMed, one workflow, one macOS host, headless Chromium.", + "Published Flow wheel and release-tagged source are exact-bound.", + "Warm OS/browser caches remain after the first fresh-browser launch.", + "Final screenshot bytes were discarded after their hashes were retained.", + "No hosted, Windows, RDP, Citrix, or customer workflow is represented." + ], + "conditions": [ + "clean", + "theme", + "rename" + ], + "environment": { + "chromium": "Playwright-managed headless Chromium", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "playwright": "1.61.0", + "python": "3.12.13" + }, + "generated_at": "2026-08-18T16:04:44.314261+00:00", + "identity_coverage": { + "applicable_steps": 8, + "armed_steps": 5, + "unarmed": [ + { + "reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "step_id": "step_000" + }, + { + "reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "step_id": "step_002" + }, + { + "reason": "the only readable row text is the target's own label (mutable evidence, excluded from identity)", + "step_id": "step_004" + } + ] + }, + "oracle": "arm-independent screenshot/OCR final-state check: exact note and saved banner, Triage row, intended patient, no wrong-type write", + "outcome_definitions": { + "correct": "arm reported completion and independent oracle confirmed effect", + "halt_or_error": "arm reported halt/incomplete and independent oracle found effect absent", + "over_halt": "arm reported halt/incomplete but independent oracle confirmed effect", + "silent_incorrect_success": "arm reported completion but independent oracle did not confirm effect", + "wrong_action": "independent oracle observed a wrong-patient or wrong-type write" + }, + "paid_or_remote_mutations": [], + "runs": [ + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.919623458001297, + "error": null, + "final_screenshot_sha256": "d173cc62630934f0e9dbc37f51536f46fd65172b6bb47d301dec51e398ca0694", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "abd12a9be34bdba88f72c46bfff764fd89275f3b213bc0d2683cd7f45cbc7bef", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.109981667017564, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9232134590274654, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "2b882a4c17309e8276043d804fce7d72b1744c886d8ef335ac6a95f3914e4beb", + "input_tokens": 0, + "note_found": true, + "note_sha256": "487a06d2518e7cbd44c2bf909f5e1c75efb4a74677014fa6a397eb3199abdba0", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.2237619999796152, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9244759170105681, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "9f7a3c464c84d35c0c730cdb3e42785ce956ee55c4d6ec3db5aad588bb7f3e2d", + "input_tokens": 0, + "note_found": true, + "note_sha256": "0bab187ec8f858a6b7c76c7488cb233a0e63bd4b622c9ab5f0e13cd5e9545164", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.21341224998468533, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9483596250065602, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "1774ae9ef0d5e682e45cd963009a5ccd245a1fde0f43a9737a4b569c3af501e6", + "input_tokens": 0, + "note_found": true, + "note_sha256": "fd9a7feca40d5477630dc008704a4cc90c73c8712333125a3fa0c75107e1def9", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.2172183750080876, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9017332500079647, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "7525116feaa3ebb69531bb6c0790e63efc37a63d6858b238932af8ae9599b692", + "input_tokens": 0, + "note_found": true, + "note_sha256": "0a79359c51a1562520f7cd213997e0e339a2cd46bba70365624dd0eda5545753", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.21286412503104657, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.9546537910355255, + "error": null, + "final_screenshot_sha256": "28868357daf60fd146bb494cf7a9840e6507cced77479394cdad1d862f117133", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "ac705b1533fec84a917710f37f46cc3c971668133ac443544636ca5c87730bda", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.228747875022236, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 1.2328357090009376, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "e90e14bd8f07fa67d7a0be4e25e26e3622d0975b363f5e7aab41ae766c4c0799", + "input_tokens": 0, + "note_found": true, + "note_sha256": "0aaceaff9dd0e0f73480518bf40ef1833fff34cc82644c7aca4c7598430b5b0e", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.2157305409782566, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.817519375006668, + "error": null, + "final_screenshot_sha256": "0fbf9eb820d2ec10cf4a6db9f5426a12650074df5e676e9118238dd7b048492f", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "d936959a095f73f37e6bb533e6a491a248117661b9428db1436f23a2518f4a9e", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 6.796473832975607, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 1.0523889170144685, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "561fa687f35b7da42c7df6cbaea2463f1fbcbeebfad4e4ed207570a58a219b7b", + "input_tokens": 0, + "note_found": true, + "note_sha256": "16bd942cfa1315c6494e526f3c06f4e240f6a41aa434725d3558e69c3c639152", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.212980541982688, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 8.739939458027948, + "error": null, + "final_screenshot_sha256": "da8d189ea8ab3cf138a39b48a8609a19d129c1e0a856532088bbf30d68c7dd23", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "cd8f603f0405cf4e119d36cb53864e88c769b27b4ac791f97b063f51e5b035c2", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 8.062472417019308, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9053395420196466, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "f474b4e55ac374801b5c7c9f434692fed4c0f7b71dbbcfbe7cfb78ba0bd82cf9", + "input_tokens": 0, + "note_found": true, + "note_sha256": "c1612f0bac6a65e866e923f68a636bce28368cb3bcb77d9d6fdb690364511f66", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.21523858298314735, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 1.1300369590171613, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "e6df91f58b57fc80a589761c02c3c5c73385bc6832282b398199220ca94a8d09", + "input_tokens": 0, + "note_found": true, + "note_sha256": "96c9f54ba668b83d716997b38595f7eb3ba6003edba85ae47efeb0fe36ba2ec2", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.2184213330037892, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 1.1343001250061207, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "ae3fc4f06795f9df0c9f3c7541fb52143bafb2850126c6ee234e3c67fdb22a1a", + "input_tokens": 0, + "note_found": true, + "note_sha256": "3ac4161fc038822d09fc851a31c46ce6933b378df41173685e962b66083afd63", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.21841358300298452, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 1.4417413329938427, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "c24e1259a0a8c0e602e56446cca872b3b8cb47f439762a14dd1ed151263c9228", + "input_tokens": 0, + "note_found": true, + "note_sha256": "ba7e43d4c36aece2b8383ce9206a4cb77e63f9803ad80eceacf42d44161fc102", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.2126732089673169, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 9.774056624970399, + "error": null, + "final_screenshot_sha256": "32e3c893937fe74e1c09875b4740047c1088bac7671aadd63b8e1662fd43c2d8", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "853b125d6a17c00943f386979192028cdefc5d9e494e89a85254abf4dc048493", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 8.962266792019363, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9412730830372311, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "49e839527a5a933e5b0ecb512366ceef0786dec32f3a1db40020d819f08a510a", + "input_tokens": 0, + "note_found": true, + "note_sha256": "79a353ca71e790724eb96a69724455151e8469bf24405e90565090dd80a3c465", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.2138964160112664, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 8.34083375002956, + "error": null, + "final_screenshot_sha256": "5562db901d104b62f29a46a84d9076c8e19a38e3f2c3dd1350f7931aae1fb7bf", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "859c554a8f0a16f1be8d62fc23b55bef0f1531fcf26544ea26e08aa2eb686b94", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.6400912500103, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 1.0527942909975536, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "5b38789776dcada6b5cc71bb4c91cc38ce21ea72f9e195b4f27ddc2abae48da2", + "input_tokens": 0, + "note_found": true, + "note_sha256": "f3ca35760733ac421075853af9cc0d0e6c44fcaf6bbfbc37f10c63d2e56def1a", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.21368400001665577, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 8.47529512498295, + "error": null, + "final_screenshot_sha256": "69e2242c3c4e20feee7b6fb7eb36a7013606e4cc4f5857dd1a330f14266e5311", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "076b38bf20e6f444fe21df4cf2bf35539439cb1a1e302623fef2398ea623dba6", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.776412124978378, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.914333416963927, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"button\", name=\"Open\").first\n", + "failed_step": "open first referral", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "29c4fef4f1bd398474762a8fe81c754db30e30fe883e16222de95d5c42004aee", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.079736291023437, + "steps_total": 9, + "success": false, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom_named", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.998797707958147, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"row\", name=\"Jane Sample\").get_by_role(\"button\", name=\"Open\")\n", + "failed_step": "open referral by patient name", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "1ab448b77f9ba6bcc9353ab49979cbbf59f3c143ca89b23f0c3acf446089d191", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.09167991700815, + "steps_total": 9, + "success": false, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.779383666987997, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"button\", name=\"Open\").first\n", + "failed_step": "open first referral", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "ac4a87fd1f10c165f46f0bd11ad9ac0566c8c3c5bbf143ee4a38e68fc8581f91", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.08740808395669, + "steps_total": 9, + "success": false, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom_named", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.80269791703904, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"row\", name=\"Jane Sample\").get_by_role(\"button\", name=\"Open\")\n", + "failed_step": "open referral by patient name", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "6b40530e788765ea3d93cbe9bcc5c6b4329da187a82172b6929a0c310206aa40", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.074826167023275, + "steps_total": 9, + "success": false, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 8.260517625021748, + "error": null, + "final_screenshot_sha256": "2afcfd960cc887cb1f9c1b148f333cd4cadc22e472c2151692fcc36a29337ed8", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "81552308938bce81da752bee7715233079b55b434b433390f22c10ff0e0b847a", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.424519333988428, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom_named", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.87557391601149, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"row\", name=\"Jane Sample\").get_by_role(\"button\", name=\"Open\")\n", + "failed_step": "open referral by patient name", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "db3899469ecf79973b1072c9b3c0acab3bd2c3be46995d2c6f509456b449ea8d", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.107698667037766, + "steps_total": 9, + "success": false, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 10.894376124953851, + "error": null, + "final_screenshot_sha256": "07af33aea15bff8129be3686b262630407d95809563bb0a73272ca0932ea51c6", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "8763d35a4b13524e96dc508111a49d65d5ff779a044ff936c063cb8993d2ba3f", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 9.919221874966752, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.75966179103125, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"button\", name=\"Open\").first\n", + "failed_step": "open first referral", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "f68010e8551fe0b53d63da9c2cfa5ec7412e85c700264aedcd369c18b4211856", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.07269195903791, + "steps_total": 9, + "success": false, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + } + ], + "runtime": { + "browser_revision": "1228", + "browser_version": "149.0.7827.55", + "dependency_snapshot_filename": "dependency-freeze.txt", + "dependency_snapshot_sha256": "215571cdf516e7806338348ad72b387730c629e071abb78817d70b147f554b9b", + "openadapt_types_version": "0.9.0", + "python_version": "3.12.13" + }, + "schema_version": 1, + "scope": "exact-current local deterministic runtime overhead and robustness", + "setup": { + "record_compile_trials": [ + { + "bundle_tree_sha256": "380cdecdf04dcd29f6d40dc93fa8788def26a61f58312259ce571a8bcc12c1d1", + "compile_wall_s": 21.353540499985684, + "manifest_sha256": "44c576fcc7dbdb46b673f95a543413c0a8637d45a9c0c76ed12a11941365b791", + "record_wall_s": 2.9867650829837658, + "trial": 1, + "workflow_sha256": "a2ad2fb5cd5e0fd183dc12db522cc7545f0528c09e9bf5405eb8024ac264120b" + }, + { + "bundle_tree_sha256": "62520e2519f3974c911f49627d8ca16280c9685d90721f25203618b917ab8a4d", + "compile_wall_s": 9.88428591703996, + "manifest_sha256": "7cb17466c614a634fbd4f6e141ce089956355b232d9c8d82175c4ac6f2b87672", + "record_wall_s": 3.2075565829873085, + "trial": 2, + "workflow_sha256": "17494b5cdc22aef8b6b27563d50a189bf6aa2d75227ba7d8436f797b8c1930b6" + }, + { + "bundle_tree_sha256": "28ed00717caafe85e770e56909c41ecf2314ff61b9e601985a83a39d9e7f455a", + "compile_wall_s": 9.861876374983694, + "manifest_sha256": "886ab0a2b98bc34131b61055575838e78f9075121bb8b77c6ac86258f2c3a779", + "record_wall_s": 2.786653416987974, + "trial": 3, + "workflow_sha256": "e6544ae3fd3b2f58d217825b2711f467ce53c4ec293fef9678078eb92b8eb276" + } + ], + "server_start_wall_s": 0.003585917002055794 + }, + "source": { + "evals": { + "commit": "1082192e9c2ec299d31330608c1016a939d3b88d" + }, + "flow": { + "artifact": { + "filename": "openadapt_flow-1.31.0-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "81133db1528ad1bb1f26e3fcb6aea61b0651db6d905cf2e4943e8383c1f3d29c" + }, + "commit": "2d225dea9a0ad29ca84ce1b037cc0ac671367e28", + "release_tag": "v1.31.0", + "tags": [ + "v1.31.0" + ], + "tracked_clean": true, + "version": "1.31.0" + }, + "runner_sha256": "fe49e0d9fc9ebf6d66120fc8f16683390071b96af6d252b82afbb94bb7449119" + }, + "task": "MockMed triage: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "trials_per_arm_condition": 3, + "zero_shot_comparison": { + "reason": "requires paid model plus an available paid WAA environment; current live Flow replay also lacks a wired WAA evaluator", + "status": "not_run" + } +} diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/replication/theme_postcondition_over_halt.json b/docs/eval_results/current_flow_v1_31_0_local_20260818/replication/theme_postcondition_over_halt.json new file mode 100644 index 0000000..71f949d --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/replication/theme_postcondition_over_halt.json @@ -0,0 +1,30 @@ +{ + "condition": { + "kind": "cosmetic theme drift", + "query": "?drift=theme" + }, + "expected": "if the independently verified effect succeeded, the runtime must not leave the run in an unresumable false-incomplete state", + "flow": { + "artifact": { + "filename": "openadapt_flow-1.31.0-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "81133db1528ad1bb1f26e3fcb6aea61b0651db6d905cf2e4943e8383c1f3d29c" + }, + "commit": "2d225dea9a0ad29ca84ce1b037cc0ac671367e28", + "release_tag": "v1.31.0", + "tags": [ + "v1.31.0" + ], + "tracked_clean": true, + "version": "1.31.0" + }, + "observations": [], + "observed_count": 0, + "oracle": "arm-independent screenshot/OCR final-state check: exact note and saved banner, Triage row, intended patient, no wrong-type write", + "reproduce": "python scripts/run_current_flow_local_benchmark.py --flow-source --flow-wheel --out ", + "required_trials": 3, + "runner_sha256": "fe49e0d9fc9ebf6d66120fc8f16683390071b96af6d252b82afbb94bb7449119", + "schema_version": 1, + "task": "MockMed triage: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "title": "theme effect succeeds but region_stable reports halt" +} diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/results.json b/docs/eval_results/current_flow_v1_31_0_local_20260818/results.json new file mode 100644 index 0000000..bb2a3ab --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/results.json @@ -0,0 +1,1118 @@ +{ + "aggregate": { + "compiled": { + "clean": { + "browser_oracle_teardown_overhead_s_median": 0.6579682489973493, + "end_to_end_wall_s_median": 7.4064323749626055, + "end_to_end_wall_s_p95_nearest_rank": 7.568909041990992, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 6.7510130000300705, + "steady_wall_s_p95_nearest_rank": 6.818610624992289, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "rename": { + "browser_oracle_teardown_overhead_s_median": 0.812406082986854, + "end_to_end_wall_s_median": 7.703007208008785, + "end_to_end_wall_s_p95_nearest_rank": 7.946615291002672, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 6.890601125021931, + "steady_wall_s_p95_nearest_rank": 7.116168666980229, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "theme": { + "browser_oracle_teardown_overhead_s_median": 0.6550411669886671, + "end_to_end_wall_s_median": 7.572836292034481, + "end_to_end_wall_s_p95_nearest_rank": 7.960791292018257, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 6.902436917007435, + "steady_wall_s_p95_nearest_rank": 7.335612749971915, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + } + }, + "dom": { + "clean": { + "browser_oracle_teardown_overhead_s_median": 0.6965757089783438, + "end_to_end_wall_s_median": 0.9244546249974519, + "end_to_end_wall_s_p95_nearest_rank": 0.9333239579573274, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.21741712500806898, + "steady_wall_s_p95_nearest_rank": 0.2371967500075698, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "rename": { + "browser_oracle_teardown_overhead_s_median": 0.7088715000427328, + "end_to_end_wall_s_median": 30.79098187503405, + "end_to_end_wall_s_p95_nearest_rank": 30.893189374997746, + "failure_taxonomy": { + "halt_or_error": 3 + }, + "halt_or_error_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 0, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 30.082110374991316, + "steady_wall_s_p95_nearest_rank": 30.084733708004933, + "task_success_count": 0, + "task_success_rate": 0.0, + "wrong_action_count": 0 + }, + "theme": { + "browser_oracle_teardown_overhead_s_median": 0.7191490420373157, + "end_to_end_wall_s_median": 0.9318847500253469, + "end_to_end_wall_s_p95_nearest_rank": 1.1377712920075282, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.21486337500391528, + "steady_wall_s_p95_nearest_rank": 0.2157665830454789, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + } + }, + "dom_named": { + "clean": { + "browser_oracle_teardown_overhead_s_median": 0.713215375028085, + "end_to_end_wall_s_median": 0.9274270830210298, + "end_to_end_wall_s_p95_nearest_rank": 0.9645489579997957, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.21667920803884044, + "steady_wall_s_p95_nearest_rank": 0.21807225001975894, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + }, + "rename": { + "browser_oracle_teardown_overhead_s_median": 0.6856826250441372, + "end_to_end_wall_s_median": 30.770152500015683, + "end_to_end_wall_s_p95_nearest_rank": 30.78983787499601, + "failure_taxonomy": { + "halt_or_error": 3 + }, + "halt_or_error_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 0, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 30.08035745797679, + "steady_wall_s_p95_nearest_rank": 30.084469874971546, + "task_success_count": 0, + "task_success_rate": 0.0, + "wrong_action_count": 0 + }, + "theme": { + "browser_oracle_teardown_overhead_s_median": 0.7673537079826929, + "end_to_end_wall_s_median": 0.98067020799499, + "end_to_end_wall_s_p95_nearest_rank": 1.319114207988605, + "failure_taxonomy": { + "correct": 3 + }, + "halt_or_error_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "over_halt_count": 0, + "reported_complete_count": 3, + "silent_incorrect_success_count": 0, + "steady_wall_s_median": 0.22123933397233486, + "steady_wall_s_p95_nearest_rank": 0.22149520798120648, + "task_success_count": 3, + "task_success_rate": 1.0, + "wrong_action_count": 0 + } + } + }, + "arms": [ + "compiled", + "dom", + "dom_named" + ], + "caveats": [ + "Synthetic MockMed, one workflow, one macOS host, headless Chromium.", + "Published Flow wheel and release-tagged source are exact-bound.", + "Warm OS/browser caches remain after the first fresh-browser launch.", + "Final screenshot bytes were discarded after their hashes were retained.", + "No hosted, Windows, RDP, Citrix, or customer workflow is represented." + ], + "conditions": [ + "clean", + "theme", + "rename" + ], + "environment": { + "chromium": "Playwright-managed headless Chromium", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "playwright": "1.61.0", + "python": "3.12.13" + }, + "generated_at": "2026-08-18T15:58:53.798288+00:00", + "identity_coverage": { + "applicable_steps": 8, + "armed_steps": 5, + "unarmed": [ + { + "reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "step_id": "step_000" + }, + { + "reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "step_id": "step_002" + }, + { + "reason": "the only readable row text is the target's own label (mutable evidence, excluded from identity)", + "step_id": "step_004" + } + ] + }, + "oracle": "arm-independent screenshot/OCR final-state check: exact note and saved banner, Triage row, intended patient, no wrong-type write", + "outcome_definitions": { + "correct": "arm reported completion and independent oracle confirmed effect", + "halt_or_error": "arm reported halt/incomplete and independent oracle found effect absent", + "over_halt": "arm reported halt/incomplete but independent oracle confirmed effect", + "silent_incorrect_success": "arm reported completion but independent oracle did not confirm effect", + "wrong_action": "independent oracle observed a wrong-patient or wrong-type write" + }, + "paid_or_remote_mutations": [], + "runs": [ + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.568909041990992, + "error": null, + "final_screenshot_sha256": "d173cc62630934f0e9dbc37f51536f46fd65172b6bb47d301dec51e398ca0694", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "abd12a9be34bdba88f72c46bfff764fd89275f3b213bc0d2683cd7f45cbc7bef", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 6.818610624992289, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9023672500043176, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "2b882a4c17309e8276043d804fce7d72b1744c886d8ef335ac6a95f3914e4beb", + "input_tokens": 0, + "note_found": true, + "note_sha256": "487a06d2518e7cbd44c2bf909f5e1c75efb4a74677014fa6a397eb3199abdba0", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.20579154102597386, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9274270830210298, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "9f7a3c464c84d35c0c730cdb3e42785ce956ee55c4d6ec3db5aad588bb7f3e2d", + "input_tokens": 0, + "note_found": true, + "note_sha256": "0bab187ec8f858a6b7c76c7488cb233a0e63bd4b622c9ab5f0e13cd5e9545164", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.21421170799294487, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9333239579573274, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "1774ae9ef0d5e682e45cd963009a5ccd245a1fde0f43a9737a4b569c3af501e6", + "input_tokens": 0, + "note_found": true, + "note_sha256": "fd9a7feca40d5477630dc008704a4cc90c73c8712333125a3fa0c75107e1def9", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.21741712500806898, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9645489579997957, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "7525116feaa3ebb69531bb6c0790e63efc37a63d6858b238932af8ae9599b692", + "input_tokens": 0, + "note_found": true, + "note_sha256": "0a79359c51a1562520f7cd213997e0e339a2cd46bba70365624dd0eda5545753", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.21807225001975894, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.295838291000109, + "error": null, + "final_screenshot_sha256": "28868357daf60fd146bb494cf7a9840e6507cced77479394cdad1d862f117133", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "ac705b1533fec84a917710f37f46cc3c971668133ac443544636ca5c87730bda", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 6.63787004200276, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9162480420200154, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "e90e14bd8f07fa67d7a0be4e25e26e3622d0975b363f5e7aab41ae766c4c0799", + "input_tokens": 0, + "note_found": true, + "note_sha256": "0aaceaff9dd0e0f73480518bf40ef1833fff34cc82644c7aca4c7598430b5b0e", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.21667920803884044, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.4064323749626055, + "error": null, + "final_screenshot_sha256": "0fbf9eb820d2ec10cf4a6db9f5426a12650074df5e676e9118238dd7b048492f", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "d936959a095f73f37e6bb533e6a491a248117661b9428db1436f23a2518f4a9e", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 6.7510130000300705, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "clean", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9244546249974519, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "561fa687f35b7da42c7df6cbaea2463f1fbcbeebfad4e4ed207570a58a219b7b", + "input_tokens": 0, + "note_found": true, + "note_sha256": "16bd942cfa1315c6494e526f3c06f4e240f6a41aa434725d3558e69c3c639152", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.2371967500075698, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.960791292018257, + "error": null, + "final_screenshot_sha256": "da8d189ea8ab3cf138a39b48a8609a19d129c1e0a856532088bbf30d68c7dd23", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "cd8f603f0405cf4e119d36cb53864e88c769b27b4ac791f97b063f51e5b035c2", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.335612749971915, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9318847500253469, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "f474b4e55ac374801b5c7c9f434692fed4c0f7b71dbbcfbe7cfb78ba0bd82cf9", + "input_tokens": 0, + "note_found": true, + "note_sha256": "c1612f0bac6a65e866e923f68a636bce28368cb3bcb77d9d6fdb690364511f66", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.2127357079880312, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9340829580323771, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "e6df91f58b57fc80a589761c02c3c5c73385bc6832282b398199220ca94a8d09", + "input_tokens": 0, + "note_found": true, + "note_sha256": "96c9f54ba668b83d716997b38595f7eb3ba6003edba85ae47efeb0fe36ba2ec2", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.22123933397233486, + "steps_total": 9, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 1.1377712920075282, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "ae3fc4f06795f9df0c9f3c7541fb52143bafb2850126c6ee234e3c67fdb22a1a", + "input_tokens": 0, + "note_found": true, + "note_sha256": "3ac4161fc038822d09fc851a31c46ce6933b378df41173685e962b66083afd63", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.2157665830454789, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 1.319114207988605, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "c24e1259a0a8c0e602e56446cca872b3b8cb47f439762a14dd1ed151263c9228", + "input_tokens": 0, + "note_found": true, + "note_sha256": "ba7e43d4c36aece2b8383ce9206a4cb77e63f9803ad80eceacf42d44161fc102", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.22149520798120648, + "steps_total": 9, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.5344834590214305, + "error": null, + "final_screenshot_sha256": "32e3c893937fe74e1c09875b4740047c1088bac7671aadd63b8e1662fd43c2d8", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "853b125d6a17c00943f386979192028cdefc5d9e494e89a85254abf4dc048493", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 6.879442292032763, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom_named", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.98067020799499, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "49e839527a5a933e5b0ecb512366ceef0786dec32f3a1db40020d819f08a510a", + "input_tokens": 0, + "note_found": true, + "note_sha256": "79a353ca71e790724eb96a69724455151e8469bf24405e90565090dd80a3c465", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.21331650001229718, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.572836292034481, + "error": null, + "final_screenshot_sha256": "5562db901d104b62f29a46a84d9076c8e19a38e3f2c3dd1350f7931aae1fb7bf", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "859c554a8f0a16f1be8d62fc23b55bef0f1531fcf26544ea26e08aa2eb686b94", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 6.902436917007435, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 9, + "api_calls": 0, + "arm": "dom", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "theme", + "cost_usd": 0.0, + "end_to_end_wall_s": 0.9080072910292074, + "error": null, + "failed_step": null, + "final_hash": "#patient/p1", + "final_screenshot_sha256": "5b38789776dcada6b5cc71bb4c91cc38ce21ea72f9e195b4f27ddc2abae48da2", + "input_tokens": 0, + "note_found": true, + "note_sha256": "f3ca35760733ac421075853af9cc0d0e6c44fcaf6bbfbc37f10c63d2e56def1a", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 0.21486337500391528, + "steps_total": 9, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.946615291002672, + "error": null, + "final_screenshot_sha256": "69e2242c3c4e20feee7b6fb7eb36a7013606e4cc4f5857dd1a330f14266e5311", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "076b38bf20e6f444fe21df4cf2bf35539439cb1a1e302623fef2398ea623dba6", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 7.116168666980229, + "success": true, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.79098187503405, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"button\", name=\"Open\").first\n", + "failed_step": "open first referral", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "29c4fef4f1bd398474762a8fe81c754db30e30fe883e16222de95d5c42004aee", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.082110374991316, + "steps_total": 9, + "success": false, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom_named", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.770152500015683, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"row\", name=\"Jane Sample\").get_by_role(\"button\", name=\"Open\")\n", + "failed_step": "open referral by patient name", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "1ab448b77f9ba6bcc9353ab49979cbbf59f3c143ca89b23f0c3acf446089d191", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.084469874971546, + "steps_total": 9, + "success": false, + "trial": 1, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.739368624985218, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"button\", name=\"Open\").first\n", + "failed_step": "open first referral", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "ac4a87fd1f10c165f46f0bd11ad9ac0566c8c3c5bbf143ee4a38e68fc8581f91", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.070089417044073, + "steps_total": 9, + "success": false, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom_named", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.78983787499601, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"row\", name=\"Jane Sample\").get_by_role(\"button\", name=\"Open\")\n", + "failed_step": "open referral by patient name", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "6b40530e788765ea3d93cbe9bcc5c6b4329da187a82172b6929a0c310206aa40", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.08035745797679, + "steps_total": 9, + "success": false, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.402242999989539, + "error": null, + "final_screenshot_sha256": "2afcfd960cc887cb1f9c1b148f333cd4cadc22e472c2151692fcc36a29337ed8", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "81552308938bce81da752bee7715233079b55b434b433390f22c10ff0e0b847a", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 6.724041874986142, + "success": true, + "trial": 2, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom_named", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.72936125000706, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"row\", name=\"Jane Sample\").get_by_role(\"button\", name=\"Open\")\n", + "failed_step": "open referral by patient name", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "db3899469ecf79973b1072c9b3c0acab3bd2c3be46995d2c6f509456b449ea8d", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.077215957979206, + "steps_total": 9, + "success": false, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 11, + "api_calls": 0, + "arm": "compiled", + "banner_found": true, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 7.703007208008785, + "error": null, + "final_screenshot_sha256": "07af33aea15bff8129be3686b262630407d95809563bb0a73272ca0932ea51c6", + "first_failure": null, + "heal_count": 0, + "input_tokens": 0, + "note_found": true, + "note_sha256": "8763d35a4b13524e96dc508111a49d65d5ff779a044ff936c063cb8993d2ba3f", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "correct", + "replayer_success": true, + "reported_complete": true, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 6.890601125021931, + "success": true, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + }, + { + "actions": 3, + "api_calls": 0, + "arm": "dom", + "banner_found": false, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "condition": "rename", + "cost_usd": 0.0, + "end_to_end_wall_s": 30.893189374997746, + "error": "TimeoutError: Locator.click: Timeout 30000ms exceeded.\nCall log:\n - waiting for get_by_role(\"button\", name=\"Open\").first\n", + "failed_step": "open first referral", + "final_hash": "#tasks", + "final_screenshot_sha256": "cc43073ad320d7140ccb5f9ca6a7e11d9953248e83d2e22684117fdcf3169c89", + "input_tokens": 0, + "note_found": false, + "note_sha256": "f68010e8551fe0b53d63da9c2cfa5ec7412e85c700264aedcd369c18b4211856", + "output_tokens": 0, + "over_halt": false, + "primary_outcome": "halt_or_error", + "reported_complete": false, + "right_patient": true, + "silent_incorrect_success": false, + "steady_wall_s": 30.084733708004933, + "steps_total": 9, + "success": false, + "trial": 3, + "wrong_action": false, + "wrong_type_row": false + } + ], + "runtime": { + "browser_revision": "1228", + "browser_version": "149.0.7827.55", + "dependency_snapshot_filename": "dependency-freeze.txt", + "dependency_snapshot_sha256": "215571cdf516e7806338348ad72b387730c629e071abb78817d70b147f554b9b", + "openadapt_types_version": "0.9.0", + "python_version": "3.12.13" + }, + "schema_version": 1, + "scope": "exact-current local deterministic runtime overhead and robustness", + "setup": { + "record_compile_trials": [ + { + "bundle_tree_sha256": "205a91df82fd3c2b7d2b29553f9fbac1fd577407b591ef55556126e1646e5d8b", + "compile_wall_s": 7.563941375003196, + "manifest_sha256": "dd59cbf4f452558ea83818e2abaacc71bb24e7dc2b65e276e68b5839dec657e9", + "record_wall_s": 2.613822333049029, + "trial": 1, + "workflow_sha256": "bdae6ce2e48a97ed3a2c2a8e99b11c5a49c240585c5492998a8fe54968801cfd" + }, + { + "bundle_tree_sha256": "4e010fd848d8e8b4737251f41d430133f6c77b239b51151c2869b79cd6e7305a", + "compile_wall_s": 7.096329166961368, + "manifest_sha256": "5a0e38c0c7b17922a566b6913777b0b876ada0534b2c699e56f8285e1efabd39", + "record_wall_s": 2.5051433329936117, + "trial": 2, + "workflow_sha256": "73e02218fcc711dc7894499adc761c48dd70476c858a52da6b0e2e7c2bbf891d" + }, + { + "bundle_tree_sha256": "e62dc18a45c067235acc5a331a13004df133adce5c61e2faa57a44c7b9a06490", + "compile_wall_s": 7.819133916986175, + "manifest_sha256": "371809be7563ca9d982e93866a64b08b9b26ca33202fcfd3f0f3ca8cec14d4b7", + "record_wall_s": 2.632781792024616, + "trial": 3, + "workflow_sha256": "5b9355f53a523820af4dae6f2dfe609924e7f0b7e081e44a18c4bcc5e1e57f61" + } + ], + "server_start_wall_s": 0.0034888340160250664 + }, + "source": { + "evals": { + "commit": "1082192e9c2ec299d31330608c1016a939d3b88d" + }, + "flow": { + "artifact": { + "filename": "openadapt_flow-1.31.0-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "81133db1528ad1bb1f26e3fcb6aea61b0651db6d905cf2e4943e8383c1f3d29c" + }, + "commit": "2d225dea9a0ad29ca84ce1b037cc0ac671367e28", + "release_tag": "v1.31.0", + "tags": [ + "v1.31.0" + ], + "tracked_clean": true, + "version": "1.31.0" + }, + "runner_sha256": "fe49e0d9fc9ebf6d66120fc8f16683390071b96af6d252b82afbb94bb7449119" + }, + "task": "MockMed triage: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "trials_per_arm_condition": 3, + "zero_shot_comparison": { + "reason": "requires paid model plus an available paid WAA environment; current live Flow replay also lacks a wired WAA evaluator", + "status": "not_run" + } +} diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/theme_postcondition_over_halt.json b/docs/eval_results/current_flow_v1_31_0_local_20260818/theme_postcondition_over_halt.json new file mode 100644 index 0000000..71f949d --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/theme_postcondition_over_halt.json @@ -0,0 +1,30 @@ +{ + "condition": { + "kind": "cosmetic theme drift", + "query": "?drift=theme" + }, + "expected": "if the independently verified effect succeeded, the runtime must not leave the run in an unresumable false-incomplete state", + "flow": { + "artifact": { + "filename": "openadapt_flow-1.31.0-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "81133db1528ad1bb1f26e3fcb6aea61b0651db6d905cf2e4943e8383c1f3d29c" + }, + "commit": "2d225dea9a0ad29ca84ce1b037cc0ac671367e28", + "release_tag": "v1.31.0", + "tags": [ + "v1.31.0" + ], + "tracked_clean": true, + "version": "1.31.0" + }, + "observations": [], + "observed_count": 0, + "oracle": "arm-independent screenshot/OCR final-state check: exact note and saved banner, Triage row, intended patient, no wrong-type write", + "reproduce": "python scripts/run_current_flow_local_benchmark.py --flow-source --flow-wheel --out ", + "required_trials": 3, + "runner_sha256": "fe49e0d9fc9ebf6d66120fc8f16683390071b96af6d252b82afbb94bb7449119", + "schema_version": 1, + "task": "MockMed triage: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "title": "theme effect succeeds but region_stable reports halt" +} diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/REPORT.md b/docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/REPORT.md new file mode 100644 index 0000000..00e256d --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/REPORT.md @@ -0,0 +1,60 @@ +# Flow transaction outcome taxonomy probe + +Does the terminal transaction outcome match what the system of record can actually prove? Judged only against MockMed's independent store, never against the runtime's report or the screen. **Not** a zero-shot computer-use comparison. + +## Source and environment + +- Flow commit: `2d225dea9a0ad29ca84ce1b037cc0ac671367e28` (version `1.31.0`; tracked-clean source) +- Release tag: `v1.31.0` +- Wheel SHA-256: `81133db1528ad1bb1f26e3fcb6aea61b0651db6d905cf2e4943e8383c1f3d29c` +- Evals base commit: `1082192e9c2ec299d31330608c1016a939d3b88d` +- Runner SHA-256: `e27ecc7fcda287c9f946bf002f02443c8628c3a2cbc50f88ed03b1ad3abf6a42` +- Platform: `macOS-15.7.3-arm64-arm-64bit` +- Python: `3.12.13` +- Playwright: `1.61.0` +- Chromium: `Playwright-managed headless Chromium` +- Network/provider use: loopback bundled MockMed fault server only; no cloud VM, hosted runner, or model API + +## Counted result + +3 trials per cell, no retries. The oracle is the system-of-record snapshot delta at `GET /api/db`: the intended effect is exactly one new `Triage` encounter for the intended patient carrying this run's note, with no pre-existing row destroyed. + +| Fault mode | Verification | Runs | Ground-truth effect | Transaction outcome | Billable | Verification performed | Model calls | +|---|---|---:|---|---|---:|---:|---:| +| `ok` | `unverified` | 3 | intended_once 3 | COMPLETED_UNVERIFIED 3 | 0 | 0 | 0 | +| `ok` | `effect_verified` | 3 | intended_once 3 | COMPLETED_UNVERIFIED 3 | 0 | 3 | 0 | +| `timeout` | `unverified` | 3 | intended_once 3 | RECONCILIATION_REQUIRED 3 | 0 | 0 | 0 | +| `timeout` | `effect_verified` | 3 | intended_once 3 | RECONCILIATION_REQUIRED 3 | 0 | 0 | 0 | +| `optimistic` | `unverified` | 3 | absent 3 | COMPLETED_UNVERIFIED 3 | 0 | 0 | 0 | +| `optimistic` | `effect_verified` | 3 | absent 3 | RECONCILIATION_REQUIRED 3 | 0 | 3 | 0 | +| `session` | `unverified` | 3 | absent 3 | RECONCILIATION_REQUIRED 3 | 0 | 0 | 0 | +| `session` | `effect_verified` | 3 | absent 3 | RECONCILIATION_REQUIRED 3 | 0 | 0 | 0 | +| `duplicate` | `unverified` | 3 | duplicate 3 | COMPLETED_UNVERIFIED 3 | 0 | 0 | 0 | +| `duplicate` | `effect_verified` | 3 | duplicate 3 | RECONCILIATION_REQUIRED 3 | 0 | 3 | 0 | + +`timeout` commits the row and then hangs past the client timeout, so the client sees an error though the write landed. `optimistic` paints a success banner the server then rejects. `session` returns 401 and persists nothing. `duplicate` accepts every arriving write. + +## Invariants + +An invariant no counted run could exercise is reported as `vacuous`; it proves nothing and is not a pass. + +| Invariant | Holds | Applicable runs | Violations | +|---|---|---:|---:| +| COMPLETED_UNVERIFIED must never be reported as a production success and must never be billable. | yes | 12 | 0 | +| Only VERIFIED may be billable. | yes | 30 | 0 | +| An outcome that asserts no business effect occurred (HALTED_BEFORE_EFFECT / FAILED_PLATFORM / CANCELED / REJECTED_POLICY) must not be reported when the system of record shows the write landed. | vacuous | 0 | 0 | +| A consequential step that reached actuation but whose effect was never verified cannot be classified as a proven absence; it must be RECONCILIATION_REQUIRED. | yes | 21 | 0 | +| A single run must never write the intended record more than once. A duplicate row from a run whose backend is NOT the double-accepting 'duplicate' mode would be a blind retry. | yes | 24 | 0 | +| VERIFIED must never be reported when nothing landed. | vacuous | 0 | 0 | +| A healthy compiled run makes no model calls and costs $0. | yes | 30 | 0 | + +## Findings + +- Every invariant held across every counted trial. + +## Caveats + +- Synthetic MockMed fault server, one workflow, one macOS host, headless Chromium. +- The probe runs the exact published wheel named above, extracted locally, bound to the release-tagged tracked-clean source. +- The `effect_verified` configuration is authored by this script, not mined by the compiler; it shows what the taxonomy can prove when a verifier IS configured, and is not a claim that a compiled bundle ships one by default. +- No hosted lifecycle, Windows UIA, RDP, Citrix, or real customer application is represented. diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/dependency-freeze.txt b/docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/dependency-freeze.txt new file mode 100644 index 0000000..163a80e --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/dependency-freeze.txt @@ -0,0 +1,37 @@ +annotated-types==0.8.0 +anyio==4.14.2 +certifi==2026.7.22 +cffi==2.1.1 +charset-normalizer==3.5.1 +cryptography==50.0.0 +flatbuffers==25.12.19 +greenlet==3.5.5 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.19 +numpy==2.5.2 +onnxruntime==1.29.0 +openadapt-flow @ file:///private/tmp/openadapt_flow-1.31.0-py3-none-any.whl +openadapt-types==0.9.0 +opencv-python==5.0.0.93 +opencv-python-headless==5.0.0.93 +packaging==26.3 +pillow==12.3.0 +pip==26.2.1 +playwright==1.61.0 +protobuf==7.35.1 +pyclipper==1.4.0 +pycparser==3.0 +pydantic==2.13.4 +pydantic_core==2.46.4 +pyee==13.0.1 +PyYAML==6.0.3 +rapidocr-onnxruntime==1.4.4 +requests==2.34.2 +shapely==2.1.2 +six==1.17.0 +tqdm==4.70.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.7.0 diff --git a/docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/results.json b/docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/results.json new file mode 100644 index 0000000..8f7d9d2 --- /dev/null +++ b/docs/eval_results/current_flow_v1_31_0_local_20260818/transaction_probe/results.json @@ -0,0 +1,1178 @@ +{ + "aggregate": { + "effect_verified": { + "duplicate": { + "billable_count": 0, + "business_effects": { + "duplicate": 3 + }, + "effect_landed_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 12.152340332977474, + "transaction_outcomes": { + "RECONCILIATION_REQUIRED": 3 + }, + "verification_performed_count": 3 + }, + "ok": { + "billable_count": 0, + "business_effects": { + "intended_once": 3 + }, + "effect_landed_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 6.840629791957326, + "transaction_outcomes": { + "COMPLETED_UNVERIFIED": 3 + }, + "verification_performed_count": 3 + }, + "optimistic": { + "billable_count": 0, + "business_effects": { + "absent": 3 + }, + "effect_landed_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 13.139672042045277, + "transaction_outcomes": { + "RECONCILIATION_REQUIRED": 3 + }, + "verification_performed_count": 3 + }, + "session": { + "billable_count": 0, + "business_effects": { + "absent": 3 + }, + "effect_landed_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 14.253758292004932, + "transaction_outcomes": { + "RECONCILIATION_REQUIRED": 3 + }, + "verification_performed_count": 0 + }, + "timeout": { + "billable_count": 0, + "business_effects": { + "intended_once": 3 + }, + "effect_landed_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 12.550642750051338, + "transaction_outcomes": { + "RECONCILIATION_REQUIRED": 3 + }, + "verification_performed_count": 0 + } + }, + "unverified": { + "duplicate": { + "billable_count": 0, + "business_effects": { + "duplicate": 3 + }, + "effect_landed_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 7.29670950002037, + "transaction_outcomes": { + "COMPLETED_UNVERIFIED": 3 + }, + "verification_performed_count": 0 + }, + "ok": { + "billable_count": 0, + "business_effects": { + "intended_once": 3 + }, + "effect_landed_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 8.57075791695388, + "transaction_outcomes": { + "COMPLETED_UNVERIFIED": 3 + }, + "verification_performed_count": 0 + }, + "optimistic": { + "billable_count": 0, + "business_effects": { + "absent": 3 + }, + "effect_landed_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 7.762287415971514, + "transaction_outcomes": { + "COMPLETED_UNVERIFIED": 3 + }, + "verification_performed_count": 0 + }, + "session": { + "billable_count": 0, + "business_effects": { + "absent": 3 + }, + "effect_landed_count": 0, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 12.89615316700656, + "transaction_outcomes": { + "RECONCILIATION_REQUIRED": 3 + }, + "verification_performed_count": 0 + }, + "timeout": { + "billable_count": 0, + "business_effects": { + "intended_once": 3 + }, + "effect_landed_count": 3, + "model_calls_total": 0, + "model_cost_usd_total": 0.0, + "n": 3, + "production_success_claimed_count": 0, + "steady_wall_s_median": 13.081712124985643, + "transaction_outcomes": { + "RECONCILIATION_REQUIRED": 3 + }, + "verification_performed_count": 0 + } + } + }, + "business_effects": [ + "intended_once", + "absent", + "duplicate", + "partial", + "collateral_loss" + ], + "caveats": [ + "Synthetic MockMed fault server, one workflow, one macOS host, headless Chromium.", + "Published Flow wheel and release-tagged source are exact-bound.", + "The effect contract in the effect_verified cells is authored by this script.", + "No hosted, Windows, RDP, Citrix, or customer workflow is represented." + ], + "consequential_step_id": "step_010", + "environment": { + "chromium": "Playwright-managed headless Chromium", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "playwright": "1.61.0", + "python": "3.12.13" + }, + "fault_modes": [ + "ok", + "timeout", + "optimistic", + "session", + "duplicate" + ], + "generated_at": "2026-08-18T16:05:26.396138+00:00", + "invariants": [ + { + "applicable_count": 12, + "holds": true, + "id": "completed_unverified_is_never_success", + "statement": "COMPLETED_UNVERIFIED must never be reported as a production success and must never be billable.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 30, + "holds": true, + "id": "only_verified_is_billable", + "statement": "Only VERIFIED may be billable.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 0, + "holds": true, + "id": "no_false_absence_claim", + "statement": "An outcome that asserts no business effect occurred (HALTED_BEFORE_EFFECT / FAILED_PLATFORM / CANCELED / REJECTED_POLICY) must not be reported when the system of record shows the write landed.", + "vacuous": true, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 21, + "holds": true, + "id": "unverified_delivered_write_needs_reconciliation", + "statement": "A consequential step that reached actuation but whose effect was never verified cannot be classified as a proven absence; it must be RECONCILIATION_REQUIRED.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 24, + "holds": true, + "id": "no_blind_retry_of_consequential_write", + "statement": "A single run must never write the intended record more than once. A duplicate row from a run whose backend is NOT the double-accepting 'duplicate' mode would be a blind retry.", + "vacuous": false, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 0, + "holds": true, + "id": "verified_requires_a_real_effect", + "statement": "VERIFIED must never be reported when nothing landed.", + "vacuous": true, + "violation_count": 0, + "violations": [] + }, + { + "applicable_count": 30, + "holds": true, + "id": "zero_model_calls_and_cost", + "statement": "A healthy compiled run makes no model calls and costs $0.", + "vacuous": false, + "violation_count": 0, + "violations": [] + } + ], + "oracle": "system-of-record snapshot delta at GET /api/db: exactly one new Triage encounter for the intended patient carrying this run's note, with no pre-existing row destroyed", + "paid_or_remote_mutations": [], + "runs": [ + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "ok", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "abd12a9be34bdba88f72c46bfff764fd89275f3b213bc0d2683cd7f45cbc7bef", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 8.897482333006337, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "ok", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "ac705b1533fec84a917710f37f46cc3c971668133ac443544636ca5c87730bda", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 8.57075791695388, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "ok", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "d936959a095f73f37e6bb533e6a491a248117661b9428db1436f23a2518f4a9e", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 8.039484166016337, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "HALTED", + "fault_mode": "timeout", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "cd8f603f0405cf4e119d36cb53864e88c769b27b4ac791f97b063f51e5b035c2", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.683551708003506, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "HALTED", + "fault_mode": "timeout", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "853b125d6a17c00943f386979192028cdefc5d9e494e89a85254abf4dc048493", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 13.081712124985643, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "HALTED", + "fault_mode": "timeout", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "859c554a8f0a16f1be8d62fc23b55bef0f1531fcf26544ea26e08aa2eb686b94", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 13.496374499984086, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "optimistic", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "076b38bf20e6f444fe21df4cf2bf35539439cb1a1e302623fef2398ea623dba6", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 9.135909667005762, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "optimistic", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "81552308938bce81da752bee7715233079b55b434b433390f22c10ff0e0b847a", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.595087042020168, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "optimistic", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "8763d35a4b13524e96dc508111a49d65d5ff779a044ff936c063cb8993d2ba3f", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.762287415971514, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "session", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "b84e97c170e65ff3a7b9712f233507b2a589dec44cf353cd0034de4a31a8d525", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.89615316700656, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "session", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "f34561628adb2837d87c87d953e1780f9fee2702b135af1bb096c18a0a4e9cf4", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.573105915973429, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "session", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "bbf6c8f549bb48f6303e19264a2f0de4bb587910dcfdf60c49032bff48b869b6", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 17.070802958041895, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "duplicate", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 2, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "duplicate", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 2, + "note_sha256": "5d210517b20451b7a7f52c057e211a2277797520edd3cf6fd76c7055d184b4ac", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 2, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.29670950002037, + "targeted_record_count": 2, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "duplicate", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 2, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "duplicate", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 2, + "note_sha256": "3207eb9c32f1aa33fe88e313eae8cb1a0adf184be9cb2cf35018f8b2cb0dfa88", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 2, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.408017291978467, + "targeted_record_count": 2, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "duplicate", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 2, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "duplicate", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 2, + "note_sha256": "80a6eec7c17331ecb26644d81df16377714db78149d2acd0a497d6768618abf7", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 2, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.262766333005857, + "targeted_record_count": 2, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "unverified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": true, + "error": null, + "exact_record_count": 1, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "ok", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "29fde12e51ae8bd734dc568e564c038c2bb5dd227c572c64ad3610815a284cfe", + "observed_effect": "present", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 7.290414916002192, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": true, + "error": null, + "exact_record_count": 1, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "ok", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "5aa693a87142dcceb4c8db74b5b93165c79313a01579adf3b24fbd1e2336a2c6", + "observed_effect": "present", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 6.816782207984943, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": true, + "error": null, + "exact_record_count": 1, + "execution_outcome": "COMPLETED_UNVERIFIED", + "fault_mode": "ok", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "7052069910c3085280370c9aed389f62a16e3d9370f73bb8472a7918acbef17e", + "observed_effect": "present", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": true, + "steady_wall_s": 6.840629791957326, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "HALTED", + "fault_mode": "timeout", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "929ced0d8f74be8a17242d43e7c61c2e894bfaaf7850937a15450d7e99d95668", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.896964624989778, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "effect_verified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "HALTED", + "fault_mode": "timeout", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "1c6102e34a227a0101775c627e73c880cab193c34848bdda0f43fc4da96cab2e", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.328748499974608, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "effect_verified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "intended_once", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": null, + "error": null, + "exact_record_count": 1, + "execution_outcome": "HALTED", + "fault_mode": "timeout", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 1, + "note_sha256": "f43d194861e88ab46f2cf911b7eb96170709e51cf0adf0296394b6877edef10c", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 1, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.550642750051338, + "targeted_record_count": 1, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "effect_verified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": false, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "optimistic", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "1f77084f54803a5f2796ed15d54d1f0198e95883f75b5e2b7e48e72dc74ebc6c", + "observed_effect": "absent", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.547751334030181, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": false, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "optimistic", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "609b93e6fc3f4060c19a1dd54aa5b40597cd07d6838e04e8715f569f158c7ac9", + "observed_effect": "absent", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 13.180746166966856, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": false, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "optimistic", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "765ae64e90cf992a6c19c701b6d84450431ec4a23a84507465fa02e3873b20b7", + "observed_effect": "absent", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 13.139672042045277, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "session", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "754e4d054274c16b7c1e99b44172b015ff2a5fd86be82d843ab4b5400cddd85d", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 16.24890491605038, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "effect_verified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "session", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "5f565c53fb7c15406a3c21440b9af972cfdfc65bd7ed0f6a3f7573eb7e7c9498", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 14.253758292004932, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "effect_verified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "absent", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": false, + "effect_verified": null, + "error": null, + "exact_record_count": 0, + "execution_outcome": "HALTED", + "fault_mode": "session", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 0, + "note_sha256": "5b2bcb6c3fa0b9850b340ad9fed810503d86b8de259a87b867064960eaa457ff", + "observed_effect": "unknown", + "production_success_claimed": false, + "records_after": 0, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.423827874998096, + "targeted_record_count": 0, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "effect_verified", + "verification_performed": false + }, + { + "attempt_state": "delivered", + "business_effect": "duplicate", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": false, + "error": null, + "exact_record_count": 2, + "execution_outcome": "HALTED", + "fault_mode": "duplicate", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 2, + "note_sha256": "7292d8f70933cee714c91112d2d2908aeeadf60c25665ed89c51414ae829e09d", + "observed_effect": "conflicting", + "production_success_claimed": false, + "records_after": 2, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.291226708039176, + "targeted_record_count": 2, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 1, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "duplicate", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": false, + "error": null, + "exact_record_count": 2, + "execution_outcome": "HALTED", + "fault_mode": "duplicate", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 2, + "note_sha256": "cac5d1be233ba75af6729a3aa4fcd4f49d764706c7cd6d22b06cb5d68a69dc15", + "observed_effect": "conflicting", + "production_success_claimed": false, + "records_after": 2, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.152340332977474, + "targeted_record_count": 2, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 2, + "verification": "effect_verified", + "verification_performed": true + }, + { + "attempt_state": "delivered", + "business_effect": "duplicate", + "collateral_lost_count": 0, + "effect_journal_entries": 1, + "effect_landed": true, + "effect_verified": false, + "error": null, + "exact_record_count": 2, + "execution_outcome": "HALTED", + "fault_mode": "duplicate", + "model_calls": 0, + "model_cost_usd": 0.0, + "new_record_count": 2, + "note_sha256": "2512a0a87f2c04343e5ea49e761c92aea70efe5ebe2247d73b0db52e6b305d1c", + "observed_effect": "conflicting", + "production_success_claimed": false, + "records_after": 2, + "records_before": 0, + "runtime_reported_success": false, + "steady_wall_s": 12.117220833024476, + "targeted_record_count": 2, + "transaction_billable": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_platform_fault": false, + "trial": 3, + "verification": "effect_verified", + "verification_performed": true + } + ], + "runtime": { + "browser_revision": "1228", + "browser_version": "149.0.7827.55", + "dependency_snapshot_filename": "dependency-freeze.txt", + "dependency_snapshot_sha256": "215571cdf516e7806338348ad72b387730c629e071abb78817d70b147f554b9b", + "openadapt_types_version": "0.9.0", + "python_version": "3.12.13" + }, + "schema_version": 1, + "scope": "transaction outcome taxonomy against a real persistence boundary", + "setup": { + "bundle_tree_sha256": "a63ab5cf68bcd2b2e5cbb047592c2a9a98dc9de9db77b3468c720cbc11f6cad7", + "compile_wall_s": 21.03327008400811, + "manifest_sha256": "f0dae2057a3a184f992b9ceb734a6ff8b42e7f385d5cba8868d7be9e708b4751", + "record_wall_s": 2.951541707967408, + "workflow_sha256": "5fd2a99bb645efecbdf8ac1bab1c15eb1d2e5f891e4dd96538574efade756227" + }, + "source": { + "evals": { + "commit": "1082192e9c2ec299d31330608c1016a939d3b88d" + }, + "flow": { + "artifact": { + "filename": "openadapt_flow-1.31.0-py3-none-any.whl", + "import_mode": "locally extracted published wheel", + "sha256": "81133db1528ad1bb1f26e3fcb6aea61b0651db6d905cf2e4943e8383c1f3d29c" + }, + "commit": "2d225dea9a0ad29ca84ce1b037cc0ac671367e28", + "release_tag": "v1.31.0", + "tags": [ + "v1.31.0" + ], + "tracked_clean": true, + "version": "1.31.0" + }, + "runner_sha256": "e27ecc7fcda287c9f946bf002f02443c8628c3a2cbc50f88ed03b1ad3abf6a42" + }, + "task": "MockMed triage against the fault-injection persistence boundary: sign in, open intended referral, create Triage encounter, enter a trial-unique note, save", + "trials_per_cell": 3, + "verification_modes": [ + "unverified", + "effect_verified" + ] +} diff --git a/scripts/build_evidence_manifest.py b/scripts/build_evidence_manifest.py index eb20ac5..233621c 100644 --- a/scripts/build_evidence_manifest.py +++ b/scripts/build_evidence_manifest.py @@ -49,6 +49,22 @@ def _verifier_for(name: str) -> str: raise ValueError(f"unknown campaign directory: {name}") +def _evidence_scope(name: str) -> dict[str, Any]: + """Return the bounded maturity scope for one public campaign.""" + + if name == "remote_lease_safety": + evidence_class = "contract_fixture" + else: + evidence_class = "local_synthetic" + return { + "class": evidence_class, + "production_acceptance": False, + "customer_workflow": False, + "hosted_execution": False, + "real_remote_session": False, + } + + def build(evidence: Path, *, sdist_sha256: str) -> dict[str, Any]: evidence = evidence.resolve() results = sorted(evidence.rglob("results.json")) @@ -68,6 +84,8 @@ def build(evidence: Path, *, sdist_sha256: str) -> dict[str, Any]: "verifier_path": _verifier_for(name), "environment": document["environment"], "runtime": document["runtime"], + "evidence_scope": _evidence_scope(name), + "metric_coverage": CHECK._metric_coverage(document), } ) contract = CHECK._task_contract(document) @@ -85,10 +103,12 @@ def build(evidence: Path, *, sdist_sha256: str) -> dict[str, Any]: if path.is_file() and path.suffix != ".md" and path.name != "EVIDENCE_MANIFEST.json" ) return { - "schema_version": 2, + "schema_version": 3, "evals_commit": root["source"]["evals"]["commit"], "flow": { "version": flow["version"], + "source_commit": flow["commit"], + "release_tag": flow["release_tag"], "wheel_sha256": flow["artifact"]["sha256"], "sdist_sha256": sdist_sha256, }, diff --git a/scripts/check_published_evidence_freshness.py b/scripts/check_published_evidence_freshness.py index 90479de..2de3544 100644 --- a/scripts/check_published_evidence_freshness.py +++ b/scripts/check_published_evidence_freshness.py @@ -19,8 +19,11 @@ 5. Every ``superseded`` set names the set that replaced it. 6. Its verifier, result, replication, public-report, task/oracle, campaign, browser, and installed-dependency inventories are complete and exact. +7. Every campaign states whether it is production acceptance, and the checker + derives silent-incorrect-success and over-halt coverage from the retained + result instead of accepting an unsupported maturity label. -Checks 1, 2, 5 and 6 are offline and always run. Checks 3 and 4 need PyPI: pass +Checks 1, 2 and 5-7 are offline and always run. Checks 3 and 4 need PyPI: pass ``--current-version``/``--current-wheel-sha256`` to supply the release out-of-band, or ``--offline`` to skip them. A network failure is reported and skipped rather than failing the build -- it is not evidence of drift. Actual @@ -187,20 +190,93 @@ def _task_contract(document: dict[str, Any]) -> dict[str, Any]: """Return the claim-defining task and oracle fields from one result file.""" keys = ( + "scope", "task", "oracle", "outcome_definitions", "trials_per_arm_condition", + "trials_per_cell", "arms", "conditions", "fault_modes", "verification_modes", "cells", "profiles", + "caveats", + "invariants", + "identity_coverage", + "paid_or_remote_mutations", ) return {key: document[key] for key in keys if key in document} +def _contains_numeric_metric(value: Any, names: set[str]) -> bool: + """Return whether a retained result contains one counted metric.""" + + if isinstance(value, dict): + for key, item in value.items(): + if key in names and isinstance(item, (int, float)) and not isinstance(item, bool): + return True + if _contains_numeric_metric(item, names): + return True + return False + if isinstance(value, list): + return any(_contains_numeric_metric(item, names) for item in value) + return False + + +def _metric_coverage(document: dict[str, Any]) -> dict[str, str]: + """Derive required reliability-metric coverage from retained results.""" + + return { + "silent_incorrect_success": ( + "counted" + if _contains_numeric_metric( + document, + {"silent_incorrect_success", "silent_incorrect_success_count"}, + ) + else "not_counted" + ), + "over_halt": ( + "counted" + if _contains_numeric_metric(document, {"over_halt", "over_halt_count"}) + else "not_counted" + ), + } + + +def _check_task_standard(document: dict[str, Any], campaign_name: object) -> list[str]: + """Check the minimum public evaluation contract for one campaign.""" + + label = repr(campaign_name) + problems: list[str] = [] + task = document.get("task") or document.get("scope") + if not isinstance(task, str) or not task.strip(): + problems.append(f"campaign task/scope is missing: {label}") + trials = document.get("trials_per_arm_condition", document.get("trials_per_cell")) + if not isinstance(trials, int) or isinstance(trials, bool) or trials < 3: + problems.append(f"campaign has fewer than 3 trials per condition: {label}") + oracle = document.get("oracle") + invariants = document.get("invariants") + if not (isinstance(oracle, str) and oracle.strip()) and not ( + isinstance(invariants, list) and invariants + ): + problems.append(f"campaign oracle/invariants are missing: {label}") + caveats = document.get("caveats") + if ( + not isinstance(caveats, list) + or not caveats + or not all(isinstance(item, str) and item.strip() for item in caveats) + ): + problems.append(f"campaign caveats are missing: {label}") + taxonomy = document.get("outcome_definitions") + if not (isinstance(taxonomy, dict) and taxonomy) and not ( + isinstance(invariants, list) and invariants + ): + problems.append(f"campaign failure taxonomy is missing: {label}") + return problems + + def check_evidence_manifest(entry: dict[str, Any], repo_root: Path) -> list[str]: """Verify the current result set is still the exact measurement set.""" @@ -213,14 +289,16 @@ def check_evidence_manifest(entry: dict[str, Any], repo_root: Path) -> list[str] except (OSError, json.JSONDecodeError) as exc: return [f"{entry['path']}: evidence_manifest cannot be read: {exc}"] prefix = f"{entry['path']}: " - if binding.get("schema_version") != 2: - problems.append(prefix + "evidence manifest schema_version must be 2") + if binding.get("schema_version") != 3: + problems.append(prefix + "evidence manifest schema_version must be 3") flow = binding.get("flow") if not isinstance(flow, dict): problems.append(f"{entry['path']}: evidence manifest flow binding is missing") else: for key, expected in ( ("version", entry.get("flow_version")), + ("source_commit", entry.get("flow_source_commit")), + ("release_tag", entry.get("flow_release_tag")), ("wheel_sha256", entry.get("wheel_sha256")), ("sdist_sha256", entry.get("sdist_sha256")), ): @@ -260,6 +338,7 @@ def check_evidence_manifest(entry: dict[str, Any], repo_root: Path) -> list[str] ) campaigns = binding.get("campaigns") + production_acceptance_values: list[bool] = [] seen_results: set[str] = set() if not isinstance(campaigns, list) or not campaigns: problems.append(prefix + "campaigns must be a non-empty list") @@ -314,6 +393,12 @@ def check_evidence_manifest(entry: dict[str, Any], repo_root: Path) -> list[str] problems.append( prefix + f"campaign Flow binding differs from {normalized_result!r}" ) + if campaign_flow.get("commit") != flow_binding.get( + "source_commit" + ) or campaign_flow.get("release_tag") != flow_binding.get("release_tag"): + problems.append( + prefix + f"campaign Flow source binding differs from {normalized_result!r}" + ) if campaign.get("environment") != document.get("environment"): problems.append(prefix + f"campaign environment differs from {normalized_result!r}") runtime = campaign.get("runtime") @@ -363,6 +448,53 @@ def check_evidence_manifest(entry: dict[str, Any], repo_root: Path) -> list[str] ) if runtime != document.get("runtime"): problems.append(prefix + f"campaign runtime differs from {normalized_result!r}") + problems.extend(prefix + problem for problem in _check_task_standard(document, name)) + metric_coverage = _metric_coverage(document) + if campaign.get("metric_coverage") != metric_coverage: + problems.append( + prefix + f"campaign metric coverage differs from {normalized_result!r}" + ) + evidence_scope = campaign.get("evidence_scope") + if not isinstance(evidence_scope, dict): + problems.append(prefix + f"campaign evidence scope is missing: {name!r}") + else: + if evidence_scope.get("class") not in { + "contract_fixture", + "local_synthetic", + "customer_environment", + "production", + }: + problems.append(prefix + f"campaign evidence class is invalid: {name!r}") + for scope_field in ( + "customer_workflow", + "hosted_execution", + "real_remote_session", + ): + if not isinstance(evidence_scope.get(scope_field), bool): + problems.append( + prefix + + f"campaign evidence scope field is not boolean: {name!r}.{scope_field}" + ) + production_acceptance = evidence_scope.get("production_acceptance") + if not isinstance(production_acceptance, bool): + problems.append( + prefix + f"campaign production acceptance is not boolean: {name!r}" + ) + elif production_acceptance and set(metric_coverage.values()) != {"counted"}: + problems.append( + prefix + + f"campaign claims production acceptance without required reliability metrics: {name!r}" + ) + elif production_acceptance and evidence_scope.get("class") not in { + "customer_environment", + "production", + }: + problems.append( + prefix + + f"campaign claims production acceptance from a synthetic or fixture class: {name!r}" + ) + else: + production_acceptance_values.append(production_acceptance) result_artifacts = { path for path in bound_artifacts @@ -374,6 +506,15 @@ def check_evidence_manifest(entry: dict[str, Any], repo_root: Path) -> list[str] + "campaign results inventory differs from retained results: " + f"missing={sorted(result_artifacts - seen_results)}, extra={sorted(seen_results - result_artifacts)}" ) + entry_production_acceptance = entry.get("production_acceptance") + if not isinstance(entry_production_acceptance, bool): + problems.append(prefix + "PUBLISHED_EVIDENCE production_acceptance must be boolean") + elif entry_production_acceptance != ( + bool(production_acceptance_values) and all(production_acceptance_values) + ): + problems.append( + prefix + "PUBLISHED_EVIDENCE production_acceptance disagrees with campaign scopes" + ) contracts = binding.get("task_contracts") if not isinstance(contracts, list) or not contracts: problems.append(f"{entry['path']}: task_contracts are missing") diff --git a/tests/test_published_evidence_freshness.py b/tests/test_published_evidence_freshness.py index 472a862..7bd81d2 100644 --- a/tests/test_published_evidence_freshness.py +++ b/tests/test_published_evidence_freshness.py @@ -28,11 +28,12 @@ def test_repo_manifest_is_internally_consistent() -> None: def test_repo_manifest_keeps_old_evidence_stale_and_names_the_exact_rerun() -> None: manifest = MODULE.load_manifest(MANIFEST) current = MODULE.current_entry(manifest) - old = next(item for item in manifest["evidence"] if item["flow_version"] == "1.28.0") - assert current["flow_version"] == "1.30.0" + old = next(item for item in manifest["evidence"] if item["flow_version"] == "1.30.0") + assert current["flow_version"] == "1.31.0" + assert current["production_acceptance"] is False assert old["status"] == "superseded" assert old["superseded_by"] == current["path"] - assert "not relabeled" in old["stale_reason"] + assert "not relabeled" in old["stale_reason"].lower() def test_drift_is_detected_when_a_newer_release_is_published() -> None: @@ -138,8 +139,20 @@ def _valid_bound_evidence(tmp_path: Path) -> tuple[dict[str, object], Path, dict report.write_text("# Bound report\n", encoding="utf-8") result = evidence / "results.json" result_value = { + "scope": "local synthetic write acceptance", "task": "write one record", "oracle": "independent record read", + "trials_per_cell": 3, + "outcome_definitions": { + "correct": "reported complete and independently confirmed", + "silent_incorrect_success": "reported complete without the effect", + "over_halt": "reported incomplete when the effect was confirmed", + }, + "caveats": ["Synthetic fixture only."], + "aggregate": { + "silent_incorrect_success_count": 0, + "over_halt_count": 0, + }, "environment": { "chromium": "Chromium 140.0.7339.16", "platform": "test-platform", @@ -151,6 +164,8 @@ def _valid_bound_evidence(tmp_path: Path) -> tuple[dict[str, object], Path, dict "runner_sha256": MODULE._sha256(verifier), "flow": { "version": "1.30.0", + "commit": "c" * 40, + "release_tag": "v1.30.0", "artifact": {"sha256": "a" * 64}, }, }, @@ -166,9 +181,14 @@ def _valid_bound_evidence(tmp_path: Path) -> tuple[dict[str, object], Path, dict _write_json(result, result_value) contract = MODULE._task_contract(result_value) binding: dict[str, object] = { - "schema_version": 2, + "schema_version": 3, "evals_commit": "b" * 40, - "flow": {"version": "1.30.0", "wheel_sha256": "a" * 64}, + "flow": { + "version": "1.30.0", + "source_commit": "c" * 40, + "release_tag": "v1.30.0", + "wheel_sha256": "a" * 64, + }, "verifiers": [{"path": "scripts/verify.py", "sha256": MODULE._sha256(verifier)}], "campaigns": [ { @@ -177,6 +197,14 @@ def _valid_bound_evidence(tmp_path: Path) -> tuple[dict[str, object], Path, dict "verifier_path": "scripts/verify.py", "environment": result_value["environment"], "runtime": result_value["runtime"], + "evidence_scope": { + "class": "local_synthetic", + "production_acceptance": False, + "customer_workflow": False, + "hosted_execution": False, + "real_remote_session": False, + }, + "metric_coverage": MODULE._metric_coverage(result_value), } ], "task_contracts": [ @@ -198,8 +226,11 @@ def _valid_bound_evidence(tmp_path: Path) -> tuple[dict[str, object], Path, dict "path": "evidence", "status": "current", "flow_version": "1.30.0", + "flow_source_commit": "c" * 40, + "flow_release_tag": "v1.30.0", "wheel_sha256": "a" * 64, "evidence_manifest": "evidence/EVIDENCE_MANIFEST.json", + "production_acceptance": False, } return entry, manifest_path, binding @@ -234,10 +265,32 @@ def test_complete_evidence_binding_passes(tmp_path: Path) -> None: lambda value: value["campaigns"][0].__setitem__("environment", {}), "campaign environment differs", ), + ( + lambda value: value["campaigns"][0]["metric_coverage"].__setitem__( + "over_halt", "not_counted" + ), + "metric coverage differs", + ), + ( + lambda value: value["campaigns"][0]["evidence_scope"].__setitem__( + "production_acceptance", "yes" + ), + "production acceptance is not boolean", + ), + ( + lambda value: value["campaigns"][0]["evidence_scope"].__setitem__( + "hosted_execution", "yes" + ), + "evidence scope field is not boolean", + ), ( lambda value: value.__setitem__("evals_commit", "c" * 40), "campaign evals commit differs", ), + ( + lambda value: value["flow"].__setitem__("source_commit", "d" * 40), + "campaign Flow source binding differs", + ), ( lambda value: value["task_contracts"].append(dict(value["task_contracts"][0])), "duplicate task contract", @@ -273,3 +326,51 @@ def test_evidence_binding_refuses_any_bound_file_mutation(tmp_path: Path, path: target.write_bytes(target.read_bytes() + b"tampered") assert MODULE.check_evidence_manifest(entry, tmp_path) + + +def test_production_acceptance_requires_both_reliability_metrics(tmp_path: Path) -> None: + entry, manifest_path, binding = _valid_bound_evidence(tmp_path) + result_path = tmp_path / "evidence" / "results.json" + result = MODULE.load_manifest(result_path) + del result["aggregate"]["over_halt_count"] + _write_json(result_path, result) + binding["artifacts"][1]["sha256"] = MODULE._sha256(result_path) + contract = MODULE._task_contract(result) + binding["task_contracts"][0]["value"] = contract + binding["task_contracts"][0]["sha256"] = MODULE._canonical_sha256(contract) + binding["campaigns"][0]["metric_coverage"] = MODULE._metric_coverage(result) + binding["campaigns"][0]["evidence_scope"]["production_acceptance"] = True + _write_json(manifest_path, binding) + + problems = MODULE.check_evidence_manifest(entry, tmp_path) + + assert any("without required reliability metrics" in problem for problem in problems) + + +def test_synthetic_campaign_cannot_claim_production_acceptance(tmp_path: Path) -> None: + entry, manifest_path, binding = _valid_bound_evidence(tmp_path) + binding["campaigns"][0]["evidence_scope"]["production_acceptance"] = True + entry["production_acceptance"] = True + _write_json(manifest_path, binding) + + problems = MODULE.check_evidence_manifest(entry, tmp_path) + + assert any("from a synthetic or fixture class" in problem for problem in problems) + + +def test_registry_production_acceptance_must_match_campaigns(tmp_path: Path) -> None: + entry, _, _ = _valid_bound_evidence(tmp_path) + entry["production_acceptance"] = True + + problems = MODULE.check_evidence_manifest(entry, tmp_path) + + assert any("disagrees with campaign scopes" in problem for problem in problems) + + +def test_task_standard_requires_count_oracle_taxonomy_and_caveats() -> None: + problems = MODULE._check_task_standard({"task": "x", "trials_per_cell": 2}, "weak") + + assert any("fewer than 3" in problem for problem in problems) + assert any("oracle/invariants" in problem for problem in problems) + assert any("failure taxonomy" in problem for problem in problems) + assert any("caveats" in problem for problem in problems) From 20b8ffab8c193c294847453d8b35cd772f01a620 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Wed, 19 Aug 2026 17:41:16 -0400 Subject: [PATCH 2/2] docs(evidence): remove the accepted-workflow claim from the readiness boundary The readiness boundary opened by saying OpenAdapt "has production-capable mechanisms and bounded accepted workflows", and said other repositories retain "bounded substrate acceptance records". Both read as acceptance claims in a public repository whose whole point is that no campaign here is production acceptance. Nothing in this repo backs an acceptance claim, so state the mechanism and the campaign scope only. Co-Authored-By: Claude Opus 5 --- docs/eval_results/PRODUCTION_READINESS.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/eval_results/PRODUCTION_READINESS.md b/docs/eval_results/PRODUCTION_READINESS.md index 288ff5e..a7567be 100644 --- a/docs/eval_results/PRODUCTION_READINESS.md +++ b/docs/eval_results/PRODUCTION_READINESS.md @@ -1,9 +1,9 @@ # Production-readiness evidence boundary -OpenAdapt has production-capable mechanisms and bounded accepted workflows. -It does not have one global production-readiness state. Each workflow, -application, version, environment, identity rule, effect oracle, and deployment -boundary needs its own qualification. +OpenAdapt has production-capable mechanisms. It does not have one global +production-readiness state. Each workflow, application, version, environment, +identity rule, effect oracle, and deployment boundary needs its own +qualification. The `current` label in `PUBLISHED_EVIDENCE.json` has one narrow meaning: the evidence set matches the current published `openadapt-flow` release. It does @@ -28,9 +28,10 @@ cannot satisfy a production-acceptance gate. No campaign in this set represents hosted execution, a customer workflow, Windows UIA, native macOS Accessibility, Linux AT-SPI, a real RDP session, or a -real Citrix ICA/HDX session. Other repositories retain bounded substrate -acceptance records. Those records remain specific to their exact tasks and -environments. +real Citrix ICA/HDX session. Other repositories retain their own bounded +substrate campaigns. Those results remain specific to their exact tasks and +environments, and no campaign in this repository establishes production +acceptance. ## Acceptance exit conditions