From 42a869db7662867fef62b807d5fded5a0a1db708 Mon Sep 17 00:00:00 2001 From: Varun Ursekar Date: Sat, 12 Sep 2026 14:01:22 -0700 Subject: [PATCH 1/8] Make the compiled optimizer task frozen and verifiable Nothing per-run is baked any more. The three scope tokens and the producer allow-list arrive by environment (VERO_PRODUCER_TOKEN, VERO_EVALUATION_TOKEN, VERO_FINALIZATION_TOKEN, VERO_PRODUCER_SCOPE): vero harbor run mints or chooses them per launch, task.toml passes them to compose like secrets, and the gateway and sidecar read them once at container start, so a later change to the launcher's environment cannot move a running gateway's allowances. The baseline commit is stamped with a fixed date so its hash depends on the seed alone. Two compiles of one config are now byte-identical. vero harbor build --manifest writes a SHA-256 manifest (one hash each for the task-data and vero-source subtrees); --check recompiles and fails on drift. A compiled.manifest.json is committed for every baseline that compiles. Co-Authored-By: Claude Fable 5.1 --- .../baseline/compiled.manifest.json | 93 ++++++++++ .../gaia/baseline/compiled.manifest.json | 92 ++++++++++ .../officeqa/baseline/compiled.manifest.json | 93 ++++++++++ .../baseline/compiled.manifest.json | 92 ++++++++++ vero/src/vero/gateway/inference.py | 98 +++++++++- vero/src/vero/harbor/backend.py | 16 +- vero/src/vero/harbor/build/compiler.py | 76 +++++--- .../build/templates/docker-compose.yaml.j2 | 16 +- vero/src/vero/harbor/cli.py | 172 +++++++++++++++++- vero/src/vero/harbor/deployment.py | 58 +++++- vero/src/vero/layout.py | 20 ++ vero/tests/test_v05_harbor_build.py | 106 +++++++++-- vero/tests/test_v05_harbor_deployment.py | 44 +++++ vero/tests/test_v05_harbor_inference.py | 59 ++++++ 14 files changed, 970 insertions(+), 65 deletions(-) create mode 100644 harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json create mode 100644 harness-opt-bench/gaia/baseline/compiled.manifest.json create mode 100644 harness-opt-bench/officeqa/baseline/compiled.manifest.json create mode 100644 harness-opt-bench/terminal-bench/baseline/compiled.manifest.json diff --git a/harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json b/harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json new file mode 100644 index 00000000..288e517b --- /dev/null +++ b/harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json @@ -0,0 +1,93 @@ +{ + "config": "build.yaml", + "files": { + "environment/Dockerfile": "dd5da42d7a39fb291ed9fc68cdcc566a0668f45fe4859dae5ddc43bff3445e7c", + "environment/agent-baseline/.git/COMMIT_EDITMSG": "4b654bd1437066b13498661f3ca14774daf1066d072036beffaf06f0c014250e", + "environment/agent-baseline/.git/HEAD": "28d25bf82af4c0e2b72f50959b2beb859e3e60b9630a5e8c603dad4ddb2b6e80", + "environment/agent-baseline/.git/config": "cae33efdb02cf774435c1ff9cb16bcc1014606908530c6e1dc727615fe3e8cda", + "environment/agent-baseline/.git/description": "85ab6c163d43a17ea9cf7788308bca1466f1b0a8d1cc92e26e9bf63da4062aee", + "environment/agent-baseline/.git/hooks/applypatch-msg.sample": "0223497a0b8b033aa58a3a521b8629869386cf7ab0e2f101963d328aa62193f7", + "environment/agent-baseline/.git/hooks/commit-msg.sample": "1f74d5e9292979b573ebd59741d46cb93ff391acdd083d340b94370753d92437", + "environment/agent-baseline/.git/hooks/fsmonitor-watchman.sample": "e0549964e93897b519bd8e333c037e51fff0f88ba13e086a331592bf801fa1d0", + "environment/agent-baseline/.git/hooks/post-update.sample": "81765af2daef323061dcbc5e61fc16481cb74b3bac9ad8a174b186523586f6c5", + "environment/agent-baseline/.git/hooks/pre-applypatch.sample": "e15c5b469ea3e0a695bea6f2c82bcf8e62821074939ddd85b77e0007ff165475", + "environment/agent-baseline/.git/hooks/pre-commit.sample": "f9af7d95eb1231ecf2eba9770fedfa8d4797a12b02d7240e98d568201251244a", + "environment/agent-baseline/.git/hooks/pre-merge-commit.sample": "d3825a70337940ebbd0a5c072984e13245920cdf8898bd225c8d27a6dfc9cb53", + "environment/agent-baseline/.git/hooks/pre-push.sample": "ecce9c7e04d3f5dd9d8ada81753dd1d549a9634b26770042b58dda00217d086a", + "environment/agent-baseline/.git/hooks/pre-rebase.sample": "4febce867790052338076f4e66cc47efb14879d18097d1d61c8261859eaaa7b3", + "environment/agent-baseline/.git/hooks/pre-receive.sample": "a4c3d2b9c7bb3fd8d1441c31bd4ee71a595d66b44fcf49ddb310252320169989", + "environment/agent-baseline/.git/hooks/prepare-commit-msg.sample": "e9ddcaa4189fddd25ed97fc8c789eca7b6ca16390b2392ae3276f0c8e1aa4619", + "environment/agent-baseline/.git/hooks/push-to-checkout.sample": "a53d0741798b287c6dd7afa64aee473f305e65d3f49463bb9d7408ec3b12bf5f", + "environment/agent-baseline/.git/hooks/update.sample": "8d5f2fa83e103cf08b57eaa67521df9194f45cbdbcb37da52ad586097a14d106", + "environment/agent-baseline/.git/info/exclude": "6671fe83b7a07c8932ee89164d1f2793b2318058eb8b98dc5c06ee0a5a3b0ec1", + "environment/agent-baseline/.git/objects/5f/2636354b5457a26944fbf282ebfa275a3745c5": "246993306b280236c31c6336c2932d02734c68ba76fecd79968926e7b4fff4fe", + "environment/agent-baseline/.git/objects/61/89527dbb3f5ac1db123f6f92399314c6812703": "ac3a0bc2cefca2d886c40fac8c1276ce277e8073c08ea691f6248859247e00be", + "environment/agent-baseline/.git/objects/9b/6c67980bc3cf2957e8e6b4960f487b1bfd07e7": "73fffb38d33b5a2fbcfaf7b6d090b19ac6ed54aeadd328021655990663ac5674", + "environment/agent-baseline/.git/objects/9e/65d7134852d9d3b22e8aaa16cd726f40dc26a9": "662391cc972df55112973ddec56c2a11b63c611cc3cc648fd9f98513fd2929df", + "environment/agent-baseline/.git/objects/ab/d511df3da84ec0b7474b8d708e377eac293daa": "4fc35c570b8fb08ebd22d7b87d4ed7fc4e123f0a1a748323ae09df115febabf9", + "environment/agent-baseline/.git/objects/b0/8d3b1a269d8e63d7866045de9997ee0004414a": "46050abfded713f037e1ed79e922b1f17739834e4538ddd1d0513cc073712dcc", + "environment/agent-baseline/.git/objects/c3/a61651a17cbbd6ac0c3b64cce690872fbf7ac0": "1fa61a457ca37f48543d2cfd169d2627c0f26530ce9e86489e04dba3334d61e8", + "environment/agent-baseline/.git/objects/ea/a6e48e68f232948f83031d9f0c40f6ae801a05": "7acb4ab281ea56da580579691a80e40170c95fda0c684f0fd27464e239a5f2e1", + "environment/agent-baseline/.git/objects/f8/8b33edccb69502dffa7ace8bbcb37ced00a30d": "0fbec58ce560e6b17dafca6b38a479dbf3a839e358b0eba419184bf47468061d", + "environment/agent-baseline/.git/objects/fc/e903367fdc39e8a500a7e421a582882f985a61": "3fc0fad06bdc2e363bea3b7d7c8e7a291ea18aed983eeefc53a17729f1327bd8", + "environment/agent-baseline/.git/refs/heads/main": "297f048998d3343607b4615d46c24ed80699575251cd0ad2aa41978c620ad93c", + "environment/agent-baseline/pyproject.toml": "0ec8bcff29f38533524ecbfa391b2176ba111bcd6e38f64af7ddb82c460a6108", + "environment/agent-baseline/src/browsecomp_plus_agent/__init__.py": "a04ce857e8a57436f145d7d8713aa2566776edae95f798dbaae33f0045f877f2", + "environment/agent-baseline/src/browsecomp_plus_agent/agent.py": "137790f566ee7bc8939fb918fa5235c1e254f25598714f96e1eda10dd4d6f35e", + "environment/agent-baseline/tests/test_agent.py": "25788ac708cf995f557b195e37bab7153af4291b3ae8f7d29fa4c48d2d02539d", + "environment/agent-baseline/uv.lock": "e8f297525100fbd6c660b85970c7f02ec779ee35bbd8d3020108f7fd3146d05e", + "environment/agent-env-blanks.json": "90b8150c7380714e1b169d902c8cc153c58596bc8000dd8adeb49469f9af5699", + "environment/agent-seed/.git/COMMIT_EDITMSG": "4b654bd1437066b13498661f3ca14774daf1066d072036beffaf06f0c014250e", + "environment/agent-seed/.git/HEAD": "28d25bf82af4c0e2b72f50959b2beb859e3e60b9630a5e8c603dad4ddb2b6e80", + "environment/agent-seed/.git/config": "cae33efdb02cf774435c1ff9cb16bcc1014606908530c6e1dc727615fe3e8cda", + "environment/agent-seed/.git/description": "85ab6c163d43a17ea9cf7788308bca1466f1b0a8d1cc92e26e9bf63da4062aee", + "environment/agent-seed/.git/hooks/applypatch-msg.sample": "0223497a0b8b033aa58a3a521b8629869386cf7ab0e2f101963d328aa62193f7", + "environment/agent-seed/.git/hooks/commit-msg.sample": "1f74d5e9292979b573ebd59741d46cb93ff391acdd083d340b94370753d92437", + "environment/agent-seed/.git/hooks/fsmonitor-watchman.sample": "e0549964e93897b519bd8e333c037e51fff0f88ba13e086a331592bf801fa1d0", + "environment/agent-seed/.git/hooks/post-update.sample": "81765af2daef323061dcbc5e61fc16481cb74b3bac9ad8a174b186523586f6c5", + "environment/agent-seed/.git/hooks/pre-applypatch.sample": "e15c5b469ea3e0a695bea6f2c82bcf8e62821074939ddd85b77e0007ff165475", + "environment/agent-seed/.git/hooks/pre-commit.sample": "f9af7d95eb1231ecf2eba9770fedfa8d4797a12b02d7240e98d568201251244a", + "environment/agent-seed/.git/hooks/pre-merge-commit.sample": "d3825a70337940ebbd0a5c072984e13245920cdf8898bd225c8d27a6dfc9cb53", + "environment/agent-seed/.git/hooks/pre-push.sample": "ecce9c7e04d3f5dd9d8ada81753dd1d549a9634b26770042b58dda00217d086a", + "environment/agent-seed/.git/hooks/pre-rebase.sample": "4febce867790052338076f4e66cc47efb14879d18097d1d61c8261859eaaa7b3", + "environment/agent-seed/.git/hooks/pre-receive.sample": "a4c3d2b9c7bb3fd8d1441c31bd4ee71a595d66b44fcf49ddb310252320169989", + "environment/agent-seed/.git/hooks/prepare-commit-msg.sample": "e9ddcaa4189fddd25ed97fc8c789eca7b6ca16390b2392ae3276f0c8e1aa4619", + "environment/agent-seed/.git/hooks/push-to-checkout.sample": "a53d0741798b287c6dd7afa64aee473f305e65d3f49463bb9d7408ec3b12bf5f", + "environment/agent-seed/.git/hooks/update.sample": "8d5f2fa83e103cf08b57eaa67521df9194f45cbdbcb37da52ad586097a14d106", + "environment/agent-seed/.git/info/exclude": "6671fe83b7a07c8932ee89164d1f2793b2318058eb8b98dc5c06ee0a5a3b0ec1", + "environment/agent-seed/.git/objects/5f/2636354b5457a26944fbf282ebfa275a3745c5": "246993306b280236c31c6336c2932d02734c68ba76fecd79968926e7b4fff4fe", + "environment/agent-seed/.git/objects/61/89527dbb3f5ac1db123f6f92399314c6812703": "ac3a0bc2cefca2d886c40fac8c1276ce277e8073c08ea691f6248859247e00be", + "environment/agent-seed/.git/objects/9b/6c67980bc3cf2957e8e6b4960f487b1bfd07e7": "73fffb38d33b5a2fbcfaf7b6d090b19ac6ed54aeadd328021655990663ac5674", + "environment/agent-seed/.git/objects/9e/65d7134852d9d3b22e8aaa16cd726f40dc26a9": "662391cc972df55112973ddec56c2a11b63c611cc3cc648fd9f98513fd2929df", + "environment/agent-seed/.git/objects/ab/d511df3da84ec0b7474b8d708e377eac293daa": "4fc35c570b8fb08ebd22d7b87d4ed7fc4e123f0a1a748323ae09df115febabf9", + "environment/agent-seed/.git/objects/b0/8d3b1a269d8e63d7866045de9997ee0004414a": "46050abfded713f037e1ed79e922b1f17739834e4538ddd1d0513cc073712dcc", + "environment/agent-seed/.git/objects/c3/a61651a17cbbd6ac0c3b64cce690872fbf7ac0": "1fa61a457ca37f48543d2cfd169d2627c0f26530ce9e86489e04dba3334d61e8", + "environment/agent-seed/.git/objects/ea/a6e48e68f232948f83031d9f0c40f6ae801a05": "7acb4ab281ea56da580579691a80e40170c95fda0c684f0fd27464e239a5f2e1", + "environment/agent-seed/.git/objects/f8/8b33edccb69502dffa7ace8bbcb37ced00a30d": "0fbec58ce560e6b17dafca6b38a479dbf3a839e358b0eba419184bf47468061d", + "environment/agent-seed/.git/objects/fc/e903367fdc39e8a500a7e421a582882f985a61": "3fc0fad06bdc2e363bea3b7d7c8e7a291ea18aed983eeefc53a17729f1327bd8", + "environment/agent-seed/.git/refs/heads/main": "297f048998d3343607b4615d46c24ed80699575251cd0ad2aa41978c620ad93c", + "environment/agent-seed/pyproject.toml": "0ec8bcff29f38533524ecbfa391b2176ba111bcd6e38f64af7ddb82c460a6108", + "environment/agent-seed/src/browsecomp_plus_agent/__init__.py": "a04ce857e8a57436f145d7d8713aa2566776edae95f798dbaae33f0045f877f2", + "environment/agent-seed/src/browsecomp_plus_agent/agent.py": "137790f566ee7bc8939fb918fa5235c1e254f25598714f96e1eda10dd4d6f35e", + "environment/agent-seed/tests/test_agent.py": "25788ac708cf995f557b195e37bab7153af4291b3ae8f7d29fa4c48d2d02539d", + "environment/agent-seed/uv.lock": "e8f297525100fbd6c660b85970c7f02ec779ee35bbd8d3020108f7fd3146d05e", + "environment/docker-compose.yaml": "db36414cd7b7015e76b36da47a7d6c5d93d8a9d1c53ac39fbf31363ef94a9e04", + "environment/gateway/Dockerfile": "70d149f1928b135b8e11b7bb8d4ef70d81f331a6766bb3947192ff876e2eea26", + "environment/gateway/config.json": "650eabf9ff9899771d5d39b0aa6f233f8405bdd98a1261aab9c3bc5beaeb6f97", + "environment/gateway/launch.json": "462bcc8d2749f1364d79e14d36be7144a84441b6bb74c985f798115ef26636ae", + "environment/main/seed.sh": "f51b70ef8fbc646ab6f129d2d7f8f3a72a1c37a9331691dcbd4f9b899dc7bc90", + "environment/overlay/skills/evals/SKILL.md": "32cd544a37264d478b01985f11178f6476b093f6c6b2f682d4495b43a14e28cc", + "environment/sidecar/Dockerfile": "3260908e6c9cd5392bcc8dad756dcdff7705bb1da0c6e63d972bf3d724971913", + "environment/sidecar/cases/development.jsonl": "becd3eef6841ad0174b8ec4bc73088432221f8849c48a5c822b955e62003c17d", + "environment/sidecar/cases/test.jsonl": "b20f6750e35cabdfb65c186ff886026f78ed4b37e3cb11e67fe7538531e1dc6d", + "environment/sidecar/cases/validation.jsonl": "4d46fc533edc1862d9ee04f8b6fcb835c017a9360c8a1a88cb7a6963cd54aa91", + "environment/sidecar/serve.json": "92aa6ad2ec2e7967bcf50d694f55a1efc00f911a1979a0d3e297bab6ebf903d3", + "environment/sidecar/task-source/": "abfee46a96f0ffa72ad3b4b287227ffb32a42cdc9188e1dddc72685dfcfa7c30", + "environment/vero/": "9ccf5b0d3ae10a57f561f9e92777eccc905ee159396b004498bdbca0d71e9160", + "instruction.md": "2665ef594b9dffcc447e8013ac15c633cd6baf69b7e019fef92382f479c71cb7", + "solution/solve.sh": "906e1e8dcd72edd2fb1f11f36bb29f2df7c203269ffd7badf9fa7632bcd98544", + "task.toml": "a23272759e2304e32610da127c97098121e980bc38568a602175d039d807c7e5", + "tests/test.sh": "b65e90e6d67aacedd559c6b1c9d0f1d5c5071ecc9178a5bd7f9e1066f332bb0d" + }, + "vero_commit": "21cb81954019c5db6e54f196d2bf048c2e3d2ba0" +} diff --git a/harness-opt-bench/gaia/baseline/compiled.manifest.json b/harness-opt-bench/gaia/baseline/compiled.manifest.json new file mode 100644 index 00000000..5ee80e73 --- /dev/null +++ b/harness-opt-bench/gaia/baseline/compiled.manifest.json @@ -0,0 +1,92 @@ +{ + "config": "build.yaml", + "files": { + "environment/Dockerfile": "dd5da42d7a39fb291ed9fc68cdcc566a0668f45fe4859dae5ddc43bff3445e7c", + "environment/agent-baseline/.git/COMMIT_EDITMSG": "4b654bd1437066b13498661f3ca14774daf1066d072036beffaf06f0c014250e", + "environment/agent-baseline/.git/HEAD": "28d25bf82af4c0e2b72f50959b2beb859e3e60b9630a5e8c603dad4ddb2b6e80", + "environment/agent-baseline/.git/config": "cae33efdb02cf774435c1ff9cb16bcc1014606908530c6e1dc727615fe3e8cda", + "environment/agent-baseline/.git/description": "85ab6c163d43a17ea9cf7788308bca1466f1b0a8d1cc92e26e9bf63da4062aee", + "environment/agent-baseline/.git/hooks/applypatch-msg.sample": "0223497a0b8b033aa58a3a521b8629869386cf7ab0e2f101963d328aa62193f7", + "environment/agent-baseline/.git/hooks/commit-msg.sample": "1f74d5e9292979b573ebd59741d46cb93ff391acdd083d340b94370753d92437", + "environment/agent-baseline/.git/hooks/fsmonitor-watchman.sample": "e0549964e93897b519bd8e333c037e51fff0f88ba13e086a331592bf801fa1d0", + "environment/agent-baseline/.git/hooks/post-update.sample": "81765af2daef323061dcbc5e61fc16481cb74b3bac9ad8a174b186523586f6c5", + "environment/agent-baseline/.git/hooks/pre-applypatch.sample": "e15c5b469ea3e0a695bea6f2c82bcf8e62821074939ddd85b77e0007ff165475", + "environment/agent-baseline/.git/hooks/pre-commit.sample": "f9af7d95eb1231ecf2eba9770fedfa8d4797a12b02d7240e98d568201251244a", + "environment/agent-baseline/.git/hooks/pre-merge-commit.sample": "d3825a70337940ebbd0a5c072984e13245920cdf8898bd225c8d27a6dfc9cb53", + "environment/agent-baseline/.git/hooks/pre-push.sample": "ecce9c7e04d3f5dd9d8ada81753dd1d549a9634b26770042b58dda00217d086a", + "environment/agent-baseline/.git/hooks/pre-rebase.sample": "4febce867790052338076f4e66cc47efb14879d18097d1d61c8261859eaaa7b3", + "environment/agent-baseline/.git/hooks/pre-receive.sample": "a4c3d2b9c7bb3fd8d1441c31bd4ee71a595d66b44fcf49ddb310252320169989", + "environment/agent-baseline/.git/hooks/prepare-commit-msg.sample": "e9ddcaa4189fddd25ed97fc8c789eca7b6ca16390b2392ae3276f0c8e1aa4619", + "environment/agent-baseline/.git/hooks/push-to-checkout.sample": "a53d0741798b287c6dd7afa64aee473f305e65d3f49463bb9d7408ec3b12bf5f", + "environment/agent-baseline/.git/hooks/update.sample": "8d5f2fa83e103cf08b57eaa67521df9194f45cbdbcb37da52ad586097a14d106", + "environment/agent-baseline/.git/info/exclude": "6671fe83b7a07c8932ee89164d1f2793b2318058eb8b98dc5c06ee0a5a3b0ec1", + "environment/agent-baseline/.git/objects/00/14b2f620b16ec4909d214c8d7786765231e7eb": "d87f0e26961b3f6fc0dc8a30a78f96db41c4f8edff56acc741846ae240e088e0", + "environment/agent-baseline/.git/objects/01/6d6341285b83888295d27c1601ab2a583b64e5": "a81e4e66cc44c3f284783b2c0f009d88b574654e42e3b996f37358c564cf458f", + "environment/agent-baseline/.git/objects/0c/c9b031040f400b23cf0e65311281f4588ddac4": "a45f02b9fd619a2f205d2930e01ce72f50c6c8aba5019f5ff9ad8a5d3369a68e", + "environment/agent-baseline/.git/objects/17/948c7a74baffb634f719ab1c00bcb7eab971d2": "ca688545811780d5fbef104404815c9eddf75ceb66661783adc6ddd6147c220d", + "environment/agent-baseline/.git/objects/26/5ca4ba578b94a887c58157868e5cc7ed03449a": "338b33024261a51997e5763459b4bc4bc9c7102a768a7136f674a2e9da00fb83", + "environment/agent-baseline/.git/objects/5d/966b7e8a646a68cebeab4cd2d157b99fc81348": "1fc19a26ac9dbabb1d5da7109f4eb50e889ededf07f32d8466b3834ee1ca673a", + "environment/agent-baseline/.git/objects/71/4769e4d9bb67a27fff227de1090b2197d6bbe7": "12412806914db77eaed203f665aeac00e1e5ec72aefb3acf2d7d6364fac64152", + "environment/agent-baseline/.git/objects/cd/a252128d1fad24069c5610c676c3664da8a209": "2742aab1e75c635d64371459701a0b834ae6333f25ed325bccb6da14d8cb7a38", + "environment/agent-baseline/.git/objects/ec/4fcded3b3545e78179343c102a3add240ec8cf": "23045cc3137175880adc9b0ca74a6c5ade5ec162e6a3c89ea62bcb2e2bdad48e", + "environment/agent-baseline/.git/objects/f4/3fd7966d8530968ca96d70a029b1b285789215": "1cf3a1a32863da6d03c44c3354299cb69092e0940d09c181b4ba6c791d849d39", + "environment/agent-baseline/.git/refs/heads/main": "08c61ed6f931362e30f1388638145c6effb22f812d62ebf1acb07c11e8be1951", + "environment/agent-baseline/pyproject.toml": "9b4fb7f9bdd4847b857a266719679b934f5a9bf2e85fb88cef027fe4b9b76aeb", + "environment/agent-baseline/src/gaia_agent/__init__.py": "55d83acc25d775ac01f155f45035a5a211343736980097bf84936c683b831d02", + "environment/agent-baseline/src/gaia_agent/agent.py": "54272d5adbad89b254e2a64789b4f37054c398e153e722f0d9f0e452e27c6926", + "environment/agent-baseline/tests/test_agent.py": "11a9f3ea6b9a0daae3558f66b1ad044fda886ca86d5011a774ce20c24f57b70f", + "environment/agent-baseline/uv.lock": "5e94e388c3050cf90b59b08cbe55030dd102e30475c171415a190da9969cf2c2", + "environment/agent-env-blanks.json": "90b8150c7380714e1b169d902c8cc153c58596bc8000dd8adeb49469f9af5699", + "environment/agent-seed/.git/COMMIT_EDITMSG": "4b654bd1437066b13498661f3ca14774daf1066d072036beffaf06f0c014250e", + "environment/agent-seed/.git/HEAD": "28d25bf82af4c0e2b72f50959b2beb859e3e60b9630a5e8c603dad4ddb2b6e80", + "environment/agent-seed/.git/config": "cae33efdb02cf774435c1ff9cb16bcc1014606908530c6e1dc727615fe3e8cda", + "environment/agent-seed/.git/description": "85ab6c163d43a17ea9cf7788308bca1466f1b0a8d1cc92e26e9bf63da4062aee", + "environment/agent-seed/.git/hooks/applypatch-msg.sample": "0223497a0b8b033aa58a3a521b8629869386cf7ab0e2f101963d328aa62193f7", + "environment/agent-seed/.git/hooks/commit-msg.sample": "1f74d5e9292979b573ebd59741d46cb93ff391acdd083d340b94370753d92437", + "environment/agent-seed/.git/hooks/fsmonitor-watchman.sample": "e0549964e93897b519bd8e333c037e51fff0f88ba13e086a331592bf801fa1d0", + "environment/agent-seed/.git/hooks/post-update.sample": "81765af2daef323061dcbc5e61fc16481cb74b3bac9ad8a174b186523586f6c5", + "environment/agent-seed/.git/hooks/pre-applypatch.sample": "e15c5b469ea3e0a695bea6f2c82bcf8e62821074939ddd85b77e0007ff165475", + "environment/agent-seed/.git/hooks/pre-commit.sample": "f9af7d95eb1231ecf2eba9770fedfa8d4797a12b02d7240e98d568201251244a", + "environment/agent-seed/.git/hooks/pre-merge-commit.sample": "d3825a70337940ebbd0a5c072984e13245920cdf8898bd225c8d27a6dfc9cb53", + "environment/agent-seed/.git/hooks/pre-push.sample": "ecce9c7e04d3f5dd9d8ada81753dd1d549a9634b26770042b58dda00217d086a", + "environment/agent-seed/.git/hooks/pre-rebase.sample": "4febce867790052338076f4e66cc47efb14879d18097d1d61c8261859eaaa7b3", + "environment/agent-seed/.git/hooks/pre-receive.sample": "a4c3d2b9c7bb3fd8d1441c31bd4ee71a595d66b44fcf49ddb310252320169989", + "environment/agent-seed/.git/hooks/prepare-commit-msg.sample": "e9ddcaa4189fddd25ed97fc8c789eca7b6ca16390b2392ae3276f0c8e1aa4619", + "environment/agent-seed/.git/hooks/push-to-checkout.sample": "a53d0741798b287c6dd7afa64aee473f305e65d3f49463bb9d7408ec3b12bf5f", + "environment/agent-seed/.git/hooks/update.sample": "8d5f2fa83e103cf08b57eaa67521df9194f45cbdbcb37da52ad586097a14d106", + "environment/agent-seed/.git/info/exclude": "6671fe83b7a07c8932ee89164d1f2793b2318058eb8b98dc5c06ee0a5a3b0ec1", + "environment/agent-seed/.git/objects/00/14b2f620b16ec4909d214c8d7786765231e7eb": "d87f0e26961b3f6fc0dc8a30a78f96db41c4f8edff56acc741846ae240e088e0", + "environment/agent-seed/.git/objects/01/6d6341285b83888295d27c1601ab2a583b64e5": "a81e4e66cc44c3f284783b2c0f009d88b574654e42e3b996f37358c564cf458f", + "environment/agent-seed/.git/objects/0c/c9b031040f400b23cf0e65311281f4588ddac4": "a45f02b9fd619a2f205d2930e01ce72f50c6c8aba5019f5ff9ad8a5d3369a68e", + "environment/agent-seed/.git/objects/17/948c7a74baffb634f719ab1c00bcb7eab971d2": "ca688545811780d5fbef104404815c9eddf75ceb66661783adc6ddd6147c220d", + "environment/agent-seed/.git/objects/26/5ca4ba578b94a887c58157868e5cc7ed03449a": "338b33024261a51997e5763459b4bc4bc9c7102a768a7136f674a2e9da00fb83", + "environment/agent-seed/.git/objects/5d/966b7e8a646a68cebeab4cd2d157b99fc81348": "1fc19a26ac9dbabb1d5da7109f4eb50e889ededf07f32d8466b3834ee1ca673a", + "environment/agent-seed/.git/objects/71/4769e4d9bb67a27fff227de1090b2197d6bbe7": "12412806914db77eaed203f665aeac00e1e5ec72aefb3acf2d7d6364fac64152", + "environment/agent-seed/.git/objects/cd/a252128d1fad24069c5610c676c3664da8a209": "2742aab1e75c635d64371459701a0b834ae6333f25ed325bccb6da14d8cb7a38", + "environment/agent-seed/.git/objects/ec/4fcded3b3545e78179343c102a3add240ec8cf": "23045cc3137175880adc9b0ca74a6c5ade5ec162e6a3c89ea62bcb2e2bdad48e", + "environment/agent-seed/.git/objects/f4/3fd7966d8530968ca96d70a029b1b285789215": "1cf3a1a32863da6d03c44c3354299cb69092e0940d09c181b4ba6c791d849d39", + "environment/agent-seed/.git/refs/heads/main": "08c61ed6f931362e30f1388638145c6effb22f812d62ebf1acb07c11e8be1951", + "environment/agent-seed/pyproject.toml": "9b4fb7f9bdd4847b857a266719679b934f5a9bf2e85fb88cef027fe4b9b76aeb", + "environment/agent-seed/src/gaia_agent/__init__.py": "55d83acc25d775ac01f155f45035a5a211343736980097bf84936c683b831d02", + "environment/agent-seed/src/gaia_agent/agent.py": "54272d5adbad89b254e2a64789b4f37054c398e153e722f0d9f0e452e27c6926", + "environment/agent-seed/tests/test_agent.py": "11a9f3ea6b9a0daae3558f66b1ad044fda886ca86d5011a774ce20c24f57b70f", + "environment/agent-seed/uv.lock": "5e94e388c3050cf90b59b08cbe55030dd102e30475c171415a190da9969cf2c2", + "environment/docker-compose.yaml": "f67edf521e9b2c5a3983f27a9e941d824039fbd17caac20cc05fcbc317e8c4a6", + "environment/gateway/Dockerfile": "70d149f1928b135b8e11b7bb8d4ef70d81f331a6766bb3947192ff876e2eea26", + "environment/gateway/config.json": "52f4f8064ce5a310cf4a7cbec7d067233fea68e26432f610f0d59660cb4898dd", + "environment/gateway/launch.json": "462bcc8d2749f1364d79e14d36be7144a84441b6bb74c985f798115ef26636ae", + "environment/main/seed.sh": "f51b70ef8fbc646ab6f129d2d7f8f3a72a1c37a9331691dcbd4f9b899dc7bc90", + "environment/overlay/skills/evals/SKILL.md": "32cd544a37264d478b01985f11178f6476b093f6c6b2f682d4495b43a14e28cc", + "environment/sidecar/Dockerfile": "ace4dc9aad7cdd3e830423e55f9ca4346988f6afff8d3187751b991679e4d526", + "environment/sidecar/cases/development.jsonl": "ef9a63afb40f7f08d9b95995857c5f81eb2f817006d14176cfb3e8676f688555", + "environment/sidecar/cases/test.jsonl": "9cf01d06d194f4e7de3257189b7629dea6d33c572a913d98147ec3b05c338044", + "environment/sidecar/cases/validation.jsonl": "292ddd833d0a584c648349dcea0eca5c8dec170fe596110df6a300935d0e3c1e", + "environment/sidecar/serve.json": "0befd4c90605d1aa2d798a89222f62b1da80426c4a2d0c1325ea28a82c72606c", + "environment/vero/": "9ccf5b0d3ae10a57f561f9e92777eccc905ee159396b004498bdbca0d71e9160", + "instruction.md": "ee539ac13ad8bdcaf5c3c2bd7465eb44db72001cb7181793c02fe99d440d88f6", + "solution/solve.sh": "10923f62ac228055ab21388df27d9131a457bb51bafbb883cfd425e3c7d0a4ad", + "task.toml": "473186cb0a19314df24cf7114469da5c7504de1a91b27830123b5f7e42490d68", + "tests/test.sh": "b65e90e6d67aacedd559c6b1c9d0f1d5c5071ecc9178a5bd7f9e1066f332bb0d" + }, + "vero_commit": "21cb81954019c5db6e54f196d2bf048c2e3d2ba0" +} diff --git a/harness-opt-bench/officeqa/baseline/compiled.manifest.json b/harness-opt-bench/officeqa/baseline/compiled.manifest.json new file mode 100644 index 00000000..f993f032 --- /dev/null +++ b/harness-opt-bench/officeqa/baseline/compiled.manifest.json @@ -0,0 +1,93 @@ +{ + "config": "build.yaml", + "files": { + "environment/Dockerfile": "dd5da42d7a39fb291ed9fc68cdcc566a0668f45fe4859dae5ddc43bff3445e7c", + "environment/agent-baseline/.git/COMMIT_EDITMSG": "4b654bd1437066b13498661f3ca14774daf1066d072036beffaf06f0c014250e", + "environment/agent-baseline/.git/HEAD": "28d25bf82af4c0e2b72f50959b2beb859e3e60b9630a5e8c603dad4ddb2b6e80", + "environment/agent-baseline/.git/config": "cae33efdb02cf774435c1ff9cb16bcc1014606908530c6e1dc727615fe3e8cda", + "environment/agent-baseline/.git/description": "85ab6c163d43a17ea9cf7788308bca1466f1b0a8d1cc92e26e9bf63da4062aee", + "environment/agent-baseline/.git/hooks/applypatch-msg.sample": "0223497a0b8b033aa58a3a521b8629869386cf7ab0e2f101963d328aa62193f7", + "environment/agent-baseline/.git/hooks/commit-msg.sample": "1f74d5e9292979b573ebd59741d46cb93ff391acdd083d340b94370753d92437", + "environment/agent-baseline/.git/hooks/fsmonitor-watchman.sample": "e0549964e93897b519bd8e333c037e51fff0f88ba13e086a331592bf801fa1d0", + "environment/agent-baseline/.git/hooks/post-update.sample": "81765af2daef323061dcbc5e61fc16481cb74b3bac9ad8a174b186523586f6c5", + "environment/agent-baseline/.git/hooks/pre-applypatch.sample": "e15c5b469ea3e0a695bea6f2c82bcf8e62821074939ddd85b77e0007ff165475", + "environment/agent-baseline/.git/hooks/pre-commit.sample": "f9af7d95eb1231ecf2eba9770fedfa8d4797a12b02d7240e98d568201251244a", + "environment/agent-baseline/.git/hooks/pre-merge-commit.sample": "d3825a70337940ebbd0a5c072984e13245920cdf8898bd225c8d27a6dfc9cb53", + "environment/agent-baseline/.git/hooks/pre-push.sample": "ecce9c7e04d3f5dd9d8ada81753dd1d549a9634b26770042b58dda00217d086a", + "environment/agent-baseline/.git/hooks/pre-rebase.sample": "4febce867790052338076f4e66cc47efb14879d18097d1d61c8261859eaaa7b3", + "environment/agent-baseline/.git/hooks/pre-receive.sample": "a4c3d2b9c7bb3fd8d1441c31bd4ee71a595d66b44fcf49ddb310252320169989", + "environment/agent-baseline/.git/hooks/prepare-commit-msg.sample": "e9ddcaa4189fddd25ed97fc8c789eca7b6ca16390b2392ae3276f0c8e1aa4619", + "environment/agent-baseline/.git/hooks/push-to-checkout.sample": "a53d0741798b287c6dd7afa64aee473f305e65d3f49463bb9d7408ec3b12bf5f", + "environment/agent-baseline/.git/hooks/update.sample": "8d5f2fa83e103cf08b57eaa67521df9194f45cbdbcb37da52ad586097a14d106", + "environment/agent-baseline/.git/info/exclude": "6671fe83b7a07c8932ee89164d1f2793b2318058eb8b98dc5c06ee0a5a3b0ec1", + "environment/agent-baseline/.git/objects/26/65ad0a820166f2ef273eaa9a4bdcd3305f3eb1": "6b45b885b2a368e7508e82451eb2789c679d6173f8df910993ab9ca35c5ce829", + "environment/agent-baseline/.git/objects/27/451a99a8b9132d1fd641eed91d8eaf5c8fe716": "61dccea9b24fbb734a15f64719d84c0d6acf3ac03dc8e31b2e8f4c1266975c8f", + "environment/agent-baseline/.git/objects/33/39a11db058e57074764b30c58b34cedb8745ad": "094549ecc2f5c2c8de68b5e2b95fb56b67331f61b535dbb1f002cb4607831677", + "environment/agent-baseline/.git/objects/37/258d7d225c0db583e2ee504779f513265ff036": "40be39b9b014c501ca106fe97365dedee1b4deb2a6052a7b4d37d91e20d825e3", + "environment/agent-baseline/.git/objects/47/cbabcb41fa2257120534d685627831a59ca89d": "3ce84f4991af7597d418b9b58f1ed492f7e5499cb348460dbce784c6b7d727fe", + "environment/agent-baseline/.git/objects/79/bcbd6a293b3aa50de7d27cd1a4a2945412fe6a": "695ff52fbc626ce1969bf79488170130f1bb255de0edf64a21597946e66c0293", + "environment/agent-baseline/.git/objects/7b/41de5aa01633bf12cb6857653007b912ebbd7f": "26e236e2e351803f9c7232bd8ae48efa5e1491c3ef585c619a459a6a52199e1b", + "environment/agent-baseline/.git/objects/82/d42f4fb61049b354d38099ffe21256d60dcf67": "3b1561a04a9cdfef6024360d59e8f7662f3dde759f0c90a130edb7cc8b217521", + "environment/agent-baseline/.git/objects/ce/7a3af1cf6955de1d85f9eb4acfd696109f811d": "a951f04be51a217a6427b8c1e0ff77f37fae5bff8823b41c4777a908ad3d75ea", + "environment/agent-baseline/.git/objects/d2/2abbed35e466b7d3abfe7bbb615418e5f9b614": "12e2a37b45f05fadd5aece9facc07059dde3898f8d092af2e37a408168fe7b33", + "environment/agent-baseline/.git/refs/heads/main": "5355a86579de14e80c5bce320b5c16df952e4e4797344b40b4e8c34be509e279", + "environment/agent-baseline/pyproject.toml": "d643c1f07ae904b59bd66af86c784c0363bc8e5419077671091219e43d10daee", + "environment/agent-baseline/src/officeqa_agent/__init__.py": "6a5042033bbcd93f2cef92bc0eaa9f33f37b2496388e7c29d75d092c8e827897", + "environment/agent-baseline/src/officeqa_agent/agent.py": "290e0c83797589b640ba93635edf2bd5c53f3880008c3c34f76496153b448367", + "environment/agent-baseline/tests/test_agent.py": "1a2a6dd660deab8ce5c619756be0a6cad493d488c4217f28981e601d868ce986", + "environment/agent-baseline/uv.lock": "256b8b915d8a1b7a8cc19619fa96d2f32869d615e9fe1055686228c02f138658", + "environment/agent-env-blanks.json": "90b8150c7380714e1b169d902c8cc153c58596bc8000dd8adeb49469f9af5699", + "environment/agent-seed/.git/COMMIT_EDITMSG": "4b654bd1437066b13498661f3ca14774daf1066d072036beffaf06f0c014250e", + "environment/agent-seed/.git/HEAD": "28d25bf82af4c0e2b72f50959b2beb859e3e60b9630a5e8c603dad4ddb2b6e80", + "environment/agent-seed/.git/config": "cae33efdb02cf774435c1ff9cb16bcc1014606908530c6e1dc727615fe3e8cda", + "environment/agent-seed/.git/description": "85ab6c163d43a17ea9cf7788308bca1466f1b0a8d1cc92e26e9bf63da4062aee", + "environment/agent-seed/.git/hooks/applypatch-msg.sample": "0223497a0b8b033aa58a3a521b8629869386cf7ab0e2f101963d328aa62193f7", + "environment/agent-seed/.git/hooks/commit-msg.sample": "1f74d5e9292979b573ebd59741d46cb93ff391acdd083d340b94370753d92437", + "environment/agent-seed/.git/hooks/fsmonitor-watchman.sample": "e0549964e93897b519bd8e333c037e51fff0f88ba13e086a331592bf801fa1d0", + "environment/agent-seed/.git/hooks/post-update.sample": "81765af2daef323061dcbc5e61fc16481cb74b3bac9ad8a174b186523586f6c5", + "environment/agent-seed/.git/hooks/pre-applypatch.sample": "e15c5b469ea3e0a695bea6f2c82bcf8e62821074939ddd85b77e0007ff165475", + "environment/agent-seed/.git/hooks/pre-commit.sample": "f9af7d95eb1231ecf2eba9770fedfa8d4797a12b02d7240e98d568201251244a", + "environment/agent-seed/.git/hooks/pre-merge-commit.sample": "d3825a70337940ebbd0a5c072984e13245920cdf8898bd225c8d27a6dfc9cb53", + "environment/agent-seed/.git/hooks/pre-push.sample": "ecce9c7e04d3f5dd9d8ada81753dd1d549a9634b26770042b58dda00217d086a", + "environment/agent-seed/.git/hooks/pre-rebase.sample": "4febce867790052338076f4e66cc47efb14879d18097d1d61c8261859eaaa7b3", + "environment/agent-seed/.git/hooks/pre-receive.sample": "a4c3d2b9c7bb3fd8d1441c31bd4ee71a595d66b44fcf49ddb310252320169989", + "environment/agent-seed/.git/hooks/prepare-commit-msg.sample": "e9ddcaa4189fddd25ed97fc8c789eca7b6ca16390b2392ae3276f0c8e1aa4619", + "environment/agent-seed/.git/hooks/push-to-checkout.sample": "a53d0741798b287c6dd7afa64aee473f305e65d3f49463bb9d7408ec3b12bf5f", + "environment/agent-seed/.git/hooks/update.sample": "8d5f2fa83e103cf08b57eaa67521df9194f45cbdbcb37da52ad586097a14d106", + "environment/agent-seed/.git/info/exclude": "6671fe83b7a07c8932ee89164d1f2793b2318058eb8b98dc5c06ee0a5a3b0ec1", + "environment/agent-seed/.git/objects/26/65ad0a820166f2ef273eaa9a4bdcd3305f3eb1": "6b45b885b2a368e7508e82451eb2789c679d6173f8df910993ab9ca35c5ce829", + "environment/agent-seed/.git/objects/27/451a99a8b9132d1fd641eed91d8eaf5c8fe716": "61dccea9b24fbb734a15f64719d84c0d6acf3ac03dc8e31b2e8f4c1266975c8f", + "environment/agent-seed/.git/objects/33/39a11db058e57074764b30c58b34cedb8745ad": "094549ecc2f5c2c8de68b5e2b95fb56b67331f61b535dbb1f002cb4607831677", + "environment/agent-seed/.git/objects/37/258d7d225c0db583e2ee504779f513265ff036": "40be39b9b014c501ca106fe97365dedee1b4deb2a6052a7b4d37d91e20d825e3", + "environment/agent-seed/.git/objects/47/cbabcb41fa2257120534d685627831a59ca89d": "3ce84f4991af7597d418b9b58f1ed492f7e5499cb348460dbce784c6b7d727fe", + "environment/agent-seed/.git/objects/79/bcbd6a293b3aa50de7d27cd1a4a2945412fe6a": "695ff52fbc626ce1969bf79488170130f1bb255de0edf64a21597946e66c0293", + "environment/agent-seed/.git/objects/7b/41de5aa01633bf12cb6857653007b912ebbd7f": "26e236e2e351803f9c7232bd8ae48efa5e1491c3ef585c619a459a6a52199e1b", + "environment/agent-seed/.git/objects/82/d42f4fb61049b354d38099ffe21256d60dcf67": "3b1561a04a9cdfef6024360d59e8f7662f3dde759f0c90a130edb7cc8b217521", + "environment/agent-seed/.git/objects/ce/7a3af1cf6955de1d85f9eb4acfd696109f811d": "a951f04be51a217a6427b8c1e0ff77f37fae5bff8823b41c4777a908ad3d75ea", + "environment/agent-seed/.git/objects/d2/2abbed35e466b7d3abfe7bbb615418e5f9b614": "12e2a37b45f05fadd5aece9facc07059dde3898f8d092af2e37a408168fe7b33", + "environment/agent-seed/.git/refs/heads/main": "5355a86579de14e80c5bce320b5c16df952e4e4797344b40b4e8c34be509e279", + "environment/agent-seed/pyproject.toml": "d643c1f07ae904b59bd66af86c784c0363bc8e5419077671091219e43d10daee", + "environment/agent-seed/src/officeqa_agent/__init__.py": "6a5042033bbcd93f2cef92bc0eaa9f33f37b2496388e7c29d75d092c8e827897", + "environment/agent-seed/src/officeqa_agent/agent.py": "290e0c83797589b640ba93635edf2bd5c53f3880008c3c34f76496153b448367", + "environment/agent-seed/tests/test_agent.py": "1a2a6dd660deab8ce5c619756be0a6cad493d488c4217f28981e601d868ce986", + "environment/agent-seed/uv.lock": "256b8b915d8a1b7a8cc19619fa96d2f32869d615e9fe1055686228c02f138658", + "environment/docker-compose.yaml": "f67edf521e9b2c5a3983f27a9e941d824039fbd17caac20cc05fcbc317e8c4a6", + "environment/gateway/Dockerfile": "70d149f1928b135b8e11b7bb8d4ef70d81f331a6766bb3947192ff876e2eea26", + "environment/gateway/config.json": "f218803a48ce1f927020df0c7443e5229689985cfc88f5f715b2658bf89e0507", + "environment/gateway/launch.json": "462bcc8d2749f1364d79e14d36be7144a84441b6bb74c985f798115ef26636ae", + "environment/main/seed.sh": "f51b70ef8fbc646ab6f129d2d7f8f3a72a1c37a9331691dcbd4f9b899dc7bc90", + "environment/overlay/skills/evals/SKILL.md": "32cd544a37264d478b01985f11178f6476b093f6c6b2f682d4495b43a14e28cc", + "environment/sidecar/Dockerfile": "3260908e6c9cd5392bcc8dad756dcdff7705bb1da0c6e63d972bf3d724971913", + "environment/sidecar/cases/development.jsonl": "8c4df163f6fa4b495392552e3fa9da8a1f743486783888d5a9150a391dd06e75", + "environment/sidecar/cases/test.jsonl": "d96a98c9ae504045f9cb2a8d7fb8fa1815528237bb68855d5e0fde8c6b031b81", + "environment/sidecar/cases/validation.jsonl": "14d86ab3071fb4091efd7b80687b1b8d8176c6b76dc1676d9a72a27f9d99032f", + "environment/sidecar/serve.json": "f70daf21febb1f19b3cc63c109d92d85a4ebb862a8375a8ecd992ff28f1386ae", + "environment/sidecar/task-source/": "be9f7f935eac438753a93bc1dccb66dc82a47c2911a9befad507d8f1e4e0480b", + "environment/vero/": "9ccf5b0d3ae10a57f561f9e92777eccc905ee159396b004498bdbca0d71e9160", + "instruction.md": "c9f995857987ecbf98ca9d1bb35dd3ecee240fc50c906cda2a464f75d879471b", + "solution/solve.sh": "6472c890bec7a80fbbf5a07e232ee0aa33a2636ca34e919bb4083e470bf11630", + "task.toml": "82490115a3279e0a53465b7a70b00f9e154cd3999179799e083befd538d010f8", + "tests/test.sh": "b65e90e6d67aacedd559c6b1c9d0f1d5c5071ecc9178a5bd7f9e1066f332bb0d" + }, + "vero_commit": "21cb81954019c5db6e54f196d2bf048c2e3d2ba0" +} diff --git a/harness-opt-bench/terminal-bench/baseline/compiled.manifest.json b/harness-opt-bench/terminal-bench/baseline/compiled.manifest.json new file mode 100644 index 00000000..300b3bde --- /dev/null +++ b/harness-opt-bench/terminal-bench/baseline/compiled.manifest.json @@ -0,0 +1,92 @@ +{ + "config": "build.yaml", + "files": { + "environment/Dockerfile": "dd5da42d7a39fb291ed9fc68cdcc566a0668f45fe4859dae5ddc43bff3445e7c", + "environment/agent-baseline/.git/COMMIT_EDITMSG": "4b654bd1437066b13498661f3ca14774daf1066d072036beffaf06f0c014250e", + "environment/agent-baseline/.git/HEAD": "28d25bf82af4c0e2b72f50959b2beb859e3e60b9630a5e8c603dad4ddb2b6e80", + "environment/agent-baseline/.git/config": "cae33efdb02cf774435c1ff9cb16bcc1014606908530c6e1dc727615fe3e8cda", + "environment/agent-baseline/.git/description": "85ab6c163d43a17ea9cf7788308bca1466f1b0a8d1cc92e26e9bf63da4062aee", + "environment/agent-baseline/.git/hooks/applypatch-msg.sample": "0223497a0b8b033aa58a3a521b8629869386cf7ab0e2f101963d328aa62193f7", + "environment/agent-baseline/.git/hooks/commit-msg.sample": "1f74d5e9292979b573ebd59741d46cb93ff391acdd083d340b94370753d92437", + "environment/agent-baseline/.git/hooks/fsmonitor-watchman.sample": "e0549964e93897b519bd8e333c037e51fff0f88ba13e086a331592bf801fa1d0", + "environment/agent-baseline/.git/hooks/post-update.sample": "81765af2daef323061dcbc5e61fc16481cb74b3bac9ad8a174b186523586f6c5", + "environment/agent-baseline/.git/hooks/pre-applypatch.sample": "e15c5b469ea3e0a695bea6f2c82bcf8e62821074939ddd85b77e0007ff165475", + "environment/agent-baseline/.git/hooks/pre-commit.sample": "f9af7d95eb1231ecf2eba9770fedfa8d4797a12b02d7240e98d568201251244a", + "environment/agent-baseline/.git/hooks/pre-merge-commit.sample": "d3825a70337940ebbd0a5c072984e13245920cdf8898bd225c8d27a6dfc9cb53", + "environment/agent-baseline/.git/hooks/pre-push.sample": "ecce9c7e04d3f5dd9d8ada81753dd1d549a9634b26770042b58dda00217d086a", + "environment/agent-baseline/.git/hooks/pre-rebase.sample": "4febce867790052338076f4e66cc47efb14879d18097d1d61c8261859eaaa7b3", + "environment/agent-baseline/.git/hooks/pre-receive.sample": "a4c3d2b9c7bb3fd8d1441c31bd4ee71a595d66b44fcf49ddb310252320169989", + "environment/agent-baseline/.git/hooks/prepare-commit-msg.sample": "e9ddcaa4189fddd25ed97fc8c789eca7b6ca16390b2392ae3276f0c8e1aa4619", + "environment/agent-baseline/.git/hooks/push-to-checkout.sample": "a53d0741798b287c6dd7afa64aee473f305e65d3f49463bb9d7408ec3b12bf5f", + "environment/agent-baseline/.git/hooks/update.sample": "8d5f2fa83e103cf08b57eaa67521df9194f45cbdbcb37da52ad586097a14d106", + "environment/agent-baseline/.git/info/exclude": "6671fe83b7a07c8932ee89164d1f2793b2318058eb8b98dc5c06ee0a5a3b0ec1", + "environment/agent-baseline/.git/objects/18/4c58a1641968e28f5488959d1175d0edde2fe0": "303a619ab424e01f536d9949973d0a8c55016406150c3fc14fbef77cc55b3170", + "environment/agent-baseline/.git/objects/1a/618b66df3626790759a1fd42b573e55c69b06a": "d650e178ee14df2c718dcd1c39fdfae6ddb1938ce11ba98123bfeb1101c05dbe", + "environment/agent-baseline/.git/objects/4c/ea8b6c762988df262bd01f0a65720b74b0452c": "ab70261beb215202798200bf0e1803d491a1154ed74a983b251395d01cd0a308", + "environment/agent-baseline/.git/objects/64/2558f80d40ff47e6a7468c53053707c9353f4d": "7a1743797c5d049b45aff402944fbf925fc25084e5f9d6b71d92b4a5b2344b56", + "environment/agent-baseline/.git/objects/a5/6f4cda998027bdc3dc27f0c26e81b841a9ba49": "d9aa8300620c08d632f619f76a755715b427b75bf8706380158ad9b582eb7ca2", + "environment/agent-baseline/.git/objects/ab/ab8909f2166fdce594bd6a906b71504a73c050": "f376424a74c67e0645f690d6b98c7eb0227b4fccaaac722bab4f738c3df3a63f", + "environment/agent-baseline/.git/objects/d7/ae6a44dd2e67c174bbea43d27f507e610a9a93": "7373d2c507e430685d8e136fcce80d23eb96c4ba601a23cbe1f95f6c5b5b5f28", + "environment/agent-baseline/.git/objects/db/0940f06c319c3c5b6b6ce6123832fb91ea146b": "03b18a24a636dc82d379a9f34980bfcac8b5bbc8528fea40b3b619f4d0f90828", + "environment/agent-baseline/.git/objects/e0/6639285e77952e526c761163b4cc98f54116b7": "4650595d7bd2bc10fbd776ee057a2954b144e0e03960aba0ef2e156d2b6e9cc4", + "environment/agent-baseline/.git/objects/ef/c7d68248387c4389e00d49bb93697216af3929": "8551a4fabfa5fea254a0602d6c72186604443b7dea62a0ed98d22f8838f6552e", + "environment/agent-baseline/.git/refs/heads/main": "f202d3d105aa71cefd764b1bb413bdd2cfaa63f151147b2a89147570a58c6016", + "environment/agent-baseline/pyproject.toml": "405868f59e0f8147957f42f430115dcefbbe23a7a984f75e43943dd96a39a938", + "environment/agent-baseline/src/terminal_bench_agent/__init__.py": "e4e8df882659a584e5e59e46b770431d30613e7eb7ad8fb7b811e48767283181", + "environment/agent-baseline/src/terminal_bench_agent/agent.py": "2749fdf6dd72e9e6ac67de22f226df3d09cb5d08d3b5bb2fbb1f8ce6f589d394", + "environment/agent-baseline/tests/test_agent.py": "a6cc1afb3be8d4f9d6e484d7e1f5aa4e058806eb8096cd9ebfa08a7addf5d6d2", + "environment/agent-baseline/uv.lock": "c7e865882c9ee4ea963fcc0511472286c3c22c8bb35051d5d4d9c8523e8a205a", + "environment/agent-env-blanks.json": "90b8150c7380714e1b169d902c8cc153c58596bc8000dd8adeb49469f9af5699", + "environment/agent-seed/.git/COMMIT_EDITMSG": "4b654bd1437066b13498661f3ca14774daf1066d072036beffaf06f0c014250e", + "environment/agent-seed/.git/HEAD": "28d25bf82af4c0e2b72f50959b2beb859e3e60b9630a5e8c603dad4ddb2b6e80", + "environment/agent-seed/.git/config": "cae33efdb02cf774435c1ff9cb16bcc1014606908530c6e1dc727615fe3e8cda", + "environment/agent-seed/.git/description": "85ab6c163d43a17ea9cf7788308bca1466f1b0a8d1cc92e26e9bf63da4062aee", + "environment/agent-seed/.git/hooks/applypatch-msg.sample": "0223497a0b8b033aa58a3a521b8629869386cf7ab0e2f101963d328aa62193f7", + "environment/agent-seed/.git/hooks/commit-msg.sample": "1f74d5e9292979b573ebd59741d46cb93ff391acdd083d340b94370753d92437", + "environment/agent-seed/.git/hooks/fsmonitor-watchman.sample": "e0549964e93897b519bd8e333c037e51fff0f88ba13e086a331592bf801fa1d0", + "environment/agent-seed/.git/hooks/post-update.sample": "81765af2daef323061dcbc5e61fc16481cb74b3bac9ad8a174b186523586f6c5", + "environment/agent-seed/.git/hooks/pre-applypatch.sample": "e15c5b469ea3e0a695bea6f2c82bcf8e62821074939ddd85b77e0007ff165475", + "environment/agent-seed/.git/hooks/pre-commit.sample": "f9af7d95eb1231ecf2eba9770fedfa8d4797a12b02d7240e98d568201251244a", + "environment/agent-seed/.git/hooks/pre-merge-commit.sample": "d3825a70337940ebbd0a5c072984e13245920cdf8898bd225c8d27a6dfc9cb53", + "environment/agent-seed/.git/hooks/pre-push.sample": "ecce9c7e04d3f5dd9d8ada81753dd1d549a9634b26770042b58dda00217d086a", + "environment/agent-seed/.git/hooks/pre-rebase.sample": "4febce867790052338076f4e66cc47efb14879d18097d1d61c8261859eaaa7b3", + "environment/agent-seed/.git/hooks/pre-receive.sample": "a4c3d2b9c7bb3fd8d1441c31bd4ee71a595d66b44fcf49ddb310252320169989", + "environment/agent-seed/.git/hooks/prepare-commit-msg.sample": "e9ddcaa4189fddd25ed97fc8c789eca7b6ca16390b2392ae3276f0c8e1aa4619", + "environment/agent-seed/.git/hooks/push-to-checkout.sample": "a53d0741798b287c6dd7afa64aee473f305e65d3f49463bb9d7408ec3b12bf5f", + "environment/agent-seed/.git/hooks/update.sample": "8d5f2fa83e103cf08b57eaa67521df9194f45cbdbcb37da52ad586097a14d106", + "environment/agent-seed/.git/info/exclude": "6671fe83b7a07c8932ee89164d1f2793b2318058eb8b98dc5c06ee0a5a3b0ec1", + "environment/agent-seed/.git/objects/18/4c58a1641968e28f5488959d1175d0edde2fe0": "303a619ab424e01f536d9949973d0a8c55016406150c3fc14fbef77cc55b3170", + "environment/agent-seed/.git/objects/1a/618b66df3626790759a1fd42b573e55c69b06a": "d650e178ee14df2c718dcd1c39fdfae6ddb1938ce11ba98123bfeb1101c05dbe", + "environment/agent-seed/.git/objects/4c/ea8b6c762988df262bd01f0a65720b74b0452c": "ab70261beb215202798200bf0e1803d491a1154ed74a983b251395d01cd0a308", + "environment/agent-seed/.git/objects/64/2558f80d40ff47e6a7468c53053707c9353f4d": "7a1743797c5d049b45aff402944fbf925fc25084e5f9d6b71d92b4a5b2344b56", + "environment/agent-seed/.git/objects/a5/6f4cda998027bdc3dc27f0c26e81b841a9ba49": "d9aa8300620c08d632f619f76a755715b427b75bf8706380158ad9b582eb7ca2", + "environment/agent-seed/.git/objects/ab/ab8909f2166fdce594bd6a906b71504a73c050": "f376424a74c67e0645f690d6b98c7eb0227b4fccaaac722bab4f738c3df3a63f", + "environment/agent-seed/.git/objects/d7/ae6a44dd2e67c174bbea43d27f507e610a9a93": "7373d2c507e430685d8e136fcce80d23eb96c4ba601a23cbe1f95f6c5b5b5f28", + "environment/agent-seed/.git/objects/db/0940f06c319c3c5b6b6ce6123832fb91ea146b": "03b18a24a636dc82d379a9f34980bfcac8b5bbc8528fea40b3b619f4d0f90828", + "environment/agent-seed/.git/objects/e0/6639285e77952e526c761163b4cc98f54116b7": "4650595d7bd2bc10fbd776ee057a2954b144e0e03960aba0ef2e156d2b6e9cc4", + "environment/agent-seed/.git/objects/ef/c7d68248387c4389e00d49bb93697216af3929": "8551a4fabfa5fea254a0602d6c72186604443b7dea62a0ed98d22f8838f6552e", + "environment/agent-seed/.git/refs/heads/main": "f202d3d105aa71cefd764b1bb413bdd2cfaa63f151147b2a89147570a58c6016", + "environment/agent-seed/pyproject.toml": "405868f59e0f8147957f42f430115dcefbbe23a7a984f75e43943dd96a39a938", + "environment/agent-seed/src/terminal_bench_agent/__init__.py": "e4e8df882659a584e5e59e46b770431d30613e7eb7ad8fb7b811e48767283181", + "environment/agent-seed/src/terminal_bench_agent/agent.py": "2749fdf6dd72e9e6ac67de22f226df3d09cb5d08d3b5bb2fbb1f8ce6f589d394", + "environment/agent-seed/tests/test_agent.py": "a6cc1afb3be8d4f9d6e484d7e1f5aa4e058806eb8096cd9ebfa08a7addf5d6d2", + "environment/agent-seed/uv.lock": "c7e865882c9ee4ea963fcc0511472286c3c22c8bb35051d5d4d9c8523e8a205a", + "environment/docker-compose.yaml": "f67edf521e9b2c5a3983f27a9e941d824039fbd17caac20cc05fcbc317e8c4a6", + "environment/gateway/Dockerfile": "70d149f1928b135b8e11b7bb8d4ef70d81f331a6766bb3947192ff876e2eea26", + "environment/gateway/config.json": "51047a5c509fca6a38d44d5cd275a779df1e03fb0005b1cd15c90312bb6a157f", + "environment/gateway/launch.json": "462bcc8d2749f1364d79e14d36be7144a84441b6bb74c985f798115ef26636ae", + "environment/main/seed.sh": "f51b70ef8fbc646ab6f129d2d7f8f3a72a1c37a9331691dcbd4f9b899dc7bc90", + "environment/overlay/skills/evals/SKILL.md": "32cd544a37264d478b01985f11178f6476b093f6c6b2f682d4495b43a14e28cc", + "environment/sidecar/Dockerfile": "ace4dc9aad7cdd3e830423e55f9ca4346988f6afff8d3187751b991679e4d526", + "environment/sidecar/cases/development.jsonl": "57896d3f3d12ceef4650246ae4f7e836879e0d0f40fbf4579156e7567f5bd25f", + "environment/sidecar/cases/test.jsonl": "f294e63d251b10418e318a392b7a2825e4fed834e65a46dd5d63e1d32b059357", + "environment/sidecar/cases/validation.jsonl": "30c87786e8441a2d50c14e0b09d8f2a4caf3b35709acee2bdbbca97c0e121fcb", + "environment/sidecar/serve.json": "b9576b9435c5f71a871c5f2a7a9ecc59fc8ba66df81c47a70315f916a4ed8ad8", + "environment/vero/": "9ccf5b0d3ae10a57f561f9e92777eccc905ee159396b004498bdbca0d71e9160", + "instruction.md": "044b5f8e0e5be1cb293898923b97cb3801399f2932dbec4b8a71f960c335576f", + "solution/solve.sh": "c53ec7868ab20f6fa5bebc19c3c9a742e0419010be3f06e04e985d299c2de88c", + "task.toml": "6832271042299c0131829efea187a85fb7598a2029b0dea3f675d7eebd62dda6", + "tests/test.sh": "b65e90e6d67aacedd559c6b1c9d0f1d5c5071ecc9178a5bd7f9e1066f332bb0d" + }, + "vero_commit": "21cb81954019c5db6e54f196d2bf048c2e3d2ba0" +} diff --git a/vero/src/vero/gateway/inference.py b/vero/src/vero/gateway/inference.py index b9b1d0f6..b45da635 100644 --- a/vero/src/vero/gateway/inference.py +++ b/vero/src/vero/gateway/inference.py @@ -11,7 +11,7 @@ import secrets import time from collections import OrderedDict -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Mapping from contextlib import asynccontextmanager from datetime import UTC, datetime from pathlib import Path @@ -40,9 +40,19 @@ def generate_inference_token() -> str: class InferenceScopeConfig(StrictModel): - """One independently authenticated and metered inference consumer.""" + """One independently authenticated and metered inference consumer. + + A compiled task names the token by environment variable (``token_env``) and + the gateway digests it once at start-up (``resolve_gateway_config``); a + hand-written or test config may carry the digest directly. ``override_env`` + names a variable holding JSON ``{"allowed_models": [...], "model_aliases": + {...}}`` that replaces those two fields at start-up, which is how the + optimizer model becomes a launch choice on an otherwise frozen task. + """ - token_sha256: str + token_sha256: str | None = None + token_env: str | None = None + override_env: str | None = None allowed_models: list[str] # Applied AFTER the allow-list check, on the way upstream. Keeps two # concerns apart: allowed_models says what the caller may ask for, and @@ -57,13 +67,77 @@ class InferenceScopeConfig(StrictModel): @field_validator("token_sha256") @classmethod - def validate_token_digest(cls, value: str) -> str: - if len(value) != 64 or any( - character not in "0123456789abcdef" for character in value + def validate_token_digest(cls, value: str | None) -> str | None: + if value is not None and ( + len(value) != 64 + or any(character not in "0123456789abcdef" for character in value) ): raise ValueError("token_sha256 must be a lowercase SHA-256 digest") return value + @field_validator("token_env", "override_env") + @classmethod + def validate_env_name(cls, value: str | None) -> str | None: + if value is not None and not _is_identifier(value): + raise ValueError("scope environment names must be valid identifiers") + return value + + @model_validator(mode="after") + def validate_token_source(self): + if self.token_sha256 is None and self.token_env is None: + raise ValueError("a scope needs token_sha256 or token_env") + return self + + +def _is_identifier(value: str) -> bool: + return bool(value) and (value[0].isalpha() or value[0] == "_") and all( + character.isalnum() or character == "_" for character in value + ) + + +def resolve_gateway_config( + config: InferenceGatewayConfig, environ: Mapping[str, str] | None = None +) -> InferenceGatewayConfig: + """Read every scope's per-run inputs from the environment, once. + + Called at gateway start. A scope with ``token_env`` gets its digest from that + variable; a scope with ``override_env`` whose variable is non-empty gets its + allow-list and aliases replaced. The result carries digests only, so the + running gateway holds no raw token and nothing it reads later can move the + allowances. + """ + environ = os.environ if environ is None else environ + scopes: dict[str, InferenceScopeConfig] = {} + for name, scope in config.scopes.items(): + update: dict[str, Any] = {} + if scope.token_env is not None: + raw = environ.get(scope.token_env, "") + if not raw.strip(): + raise RuntimeError( + f"inference scope {name!r}: {scope.token_env} is not set" + ) + update["token_sha256"] = token_digest(raw) + if scope.override_env is not None and environ.get(scope.override_env, "").strip(): + try: + override = json.loads(environ[scope.override_env]) + except json.JSONDecodeError as error: + raise RuntimeError( + f"inference scope {name!r}: {scope.override_env} is not JSON" + ) from error + unknown = set(override) - {"allowed_models", "model_aliases"} + if not isinstance(override, dict) or unknown: + raise RuntimeError( + f"inference scope {name!r}: {scope.override_env} may only set " + "allowed_models and model_aliases" + ) + update.update(override) + scopes[name] = ( + InferenceScopeConfig.model_validate({**scope.model_dump(), **update}) + if update + else scope + ) + return config.model_copy(update={"scopes": scopes}) + @field_validator("allowed_models") @classmethod def validate_models(cls, value: list[str]) -> list[str]: @@ -755,6 +829,12 @@ def create_inference_gateway_app( """Create a scoped reverse proxy without exposing the upstream credential.""" if not upstream_api_key.strip(): raise ValueError("upstream API key must not be empty") + unresolved = [n for n, s in config.scopes.items() if s.token_sha256 is None] + if unresolved: + raise ValueError( + "scopes without a token digest (run resolve_gateway_config first): " + + ", ".join(sorted(unresolved)) + ) try: import httpx except ImportError as error: @@ -806,7 +886,7 @@ async def usage_status( if ( scope is None or token is None - or not secrets.compare_digest(token_digest(token), scope.token_sha256) + or not secrets.compare_digest(token_digest(token), scope.token_sha256 or "") ): raise HTTPException(status_code=403, detail="invalid inference scope token") return store.public_status()[scope_name] @@ -832,7 +912,7 @@ async def proxy( if ( scope is None or token is None - or not secrets.compare_digest(token_digest(token), scope.token_sha256) + or not secrets.compare_digest(token_digest(token), scope.token_sha256 or "") ): return _provider_error( 403, "invalid inference scope token", "invalid_token" @@ -1180,7 +1260,7 @@ def serve_inference_gateway( ) -> None: import uvicorn - config = load_inference_gateway_config(config_path) + config = resolve_gateway_config(load_inference_gateway_config(config_path)) upstream_api_key = os.environ.get(config.upstream_api_key_env) if not upstream_api_key: raise RuntimeError( diff --git a/vero/src/vero/harbor/backend.py b/vero/src/vero/harbor/backend.py index 61a44a03..6dad3540 100644 --- a/vero/src/vero/harbor/backend.py +++ b/vero/src/vero/harbor/backend.py @@ -142,6 +142,11 @@ class HarborBackendConfig(StrictModel): # so the optimizer's search evaluations cannot exhaust the budget the mandatory # re-score needs. When unset, finalization falls back to the evaluation scope. inference_gateway_finalization_token: str | None = None + # A compiled task names the tokens by environment variable instead of carrying + # them, so the compiled tree holds nothing per-run. The sidecar fills the two + # fields above from these at start (deployment.resolve_runtime_secrets). + inference_gateway_token_env: str | None = None + inference_gateway_finalization_token_env: str | None = None # When True, task-owned evaluation services (e.g. LLM user-simulators or graders # that run *inside* the task containers and cannot reach the compose-internal # gateway) receive the real upstream credentials via OPENAI_*, while the @@ -287,11 +292,14 @@ def validate_filesystem_and_environment(self) -> HarborBackendConfig: "environment and passthrough_environment overlap for: " + ", ".join(sorted(overlap)) ) - if (self.inference_gateway_url is None) != ( - self.inference_gateway_token is None - ): + has_token = ( + self.inference_gateway_token is not None + or self.inference_gateway_token_env is not None + ) + if (self.inference_gateway_url is None) != (not has_token): raise ValueError( - "inference_gateway_url and inference_gateway_token must be set together" + "inference_gateway_url and inference_gateway_token (or its _env) " + "must be set together" ) if ( self.inference_gateway_url is not None diff --git a/vero/src/vero/harbor/build/compiler.py b/vero/src/vero/harbor/build/compiler.py index ac1b1656..c4f1ddc0 100644 --- a/vero/src/vero/harbor/build/compiler.py +++ b/vero/src/vero/harbor/build/compiler.py @@ -21,7 +21,7 @@ EvaluationSet, RetryPolicy, ) -from vero.gateway.inference import generate_inference_token, token_digest +from vero.gateway.inference import token_digest from vero.harbor.build.config import HarborBuildConfig from vero.harbor.build.specs import WorkspaceOverlaySpec from vero.layout import LAYOUT @@ -129,6 +129,11 @@ def _safe_extract_tar(payload: bytes, destination: Path) -> None: archive.extractall(destination, filter="data") +#: Every compiled baseline commit is stamped with this instant, not the wall +#: clock, so its hash is a function of the seed's contents only. +BASELINE_COMMIT_DATE = "2000-01-01T00:00:00Z" + + def _prepare_baseline_repo( source: Path, destination: Path, @@ -168,6 +173,9 @@ def _prepare_baseline_repo( raise ValueError("agent baseline contains reserved path '.evals'") def git(*arguments: str) -> str: + # Fixed identity AND fixed dates: the baseline commit hash then depends on + # the tree alone, so two compiles of one seed are byte-identical and the + # compiled task can be frozen and checked (`vero harbor build --check`). result = subprocess.run( [ "git", @@ -182,6 +190,11 @@ def git(*arguments: str) -> str: check=True, capture_output=True, text=True, + env={ + **os.environ, + "GIT_AUTHOR_DATE": BASELINE_COMMIT_DATE, + "GIT_COMMITTER_DATE": BASELINE_COMMIT_DATE, + }, ) return result.stdout.strip() @@ -252,8 +265,6 @@ def _deployment_config( *, baseline_version: str, local_task_source: bool, - evaluation_inference_token: str | None, - finalization_inference_token: str | None, ) -> dict: task_source = TASK_SOURCE_DIR if local_task_source else config.task_source backends = {} @@ -339,8 +350,16 @@ def _deployment_config( if config.inference_gateway is not None else None ), - "inference_gateway_token": evaluation_inference_token, - "inference_gateway_finalization_token": finalization_inference_token, + "inference_gateway_token_env": ( + LAYOUT.evaluation_token_env + if config.inference_gateway is not None + else None + ), + "inference_gateway_finalization_token_env": ( + LAYOUT.finalization_token_env + if config.inference_gateway is not None + else None + ), "harness_user": config.harness_user, "task_services_use_upstream": config.task_services_use_upstream, "upstream_api_key_env": ( @@ -571,6 +590,12 @@ def compile_harbor_task( gateway_environment.append(UPSTREAM_BASE_URL_ENV) credential_sources.append(config.inference_gateway.upstream_base_url_env) task_environment = list(dict.fromkeys([*config.secrets, *gateway_environment])) + # Per-run inference inputs the launcher supplies (tokens, producer scope). They + # travel like secrets -- task.toml passes them to compose -- but are never + # baked, which is what lets the compiled tree be frozen and diffed. + runtime_environment = ( + list(LAYOUT.runtime_inference_envs) if config.inference_gateway is not None else [] + ) # One list, two consumers. The compose template blanks these on the candidate's # main service; the launcher blanks the same names for the optimizer's agent # exec, which compose never sees. Keeping it computed once is the point: the @@ -581,7 +606,9 @@ def compile_harbor_task( # caller's own spelling. scrubbed_main_environment = [ name - for name in dict.fromkeys([*task_environment, *credential_sources]) + for name in dict.fromkeys( + [*task_environment, *credential_sources, *runtime_environment] + ) if name not in GATEWAY_ROUTED_CREDENTIALS ] if os.environ.get("VERO_SKIP_SECRET_CHECK") is None: @@ -659,30 +686,18 @@ def compile_harbor_task( Path(config.task_source), sidecar_dir / "task-source", ) - producer_inference_token = ( - generate_inference_token() if config.inference_gateway is not None else None - ) - evaluation_inference_token = ( - generate_inference_token() if config.inference_gateway is not None else None - ) - finalization_inference_token = ( - generate_inference_token() if config.inference_gateway is not None else None - ) + # Tokens are not minted here. The launcher mints them per run and hands them + # over by environment variable; every service reads its own once at start. deployment = _deployment_config( config, baseline_version=baseline, local_task_source=local_task_source, - evaluation_inference_token=evaluation_inference_token, - finalization_inference_token=finalization_inference_token, ) (sidecar_dir / "serve.json").write_text( json.dumps(deployment, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) if config.inference_gateway is not None: - assert producer_inference_token is not None - assert evaluation_inference_token is not None - assert finalization_inference_token is not None # Finalization reserves its own budget; default to the evaluation policy. finalization_spec = ( config.inference_gateway.finalization or config.inference_gateway.evaluation @@ -710,15 +725,18 @@ def compile_harbor_task( ), "scopes": { "producer": { - "token_sha256": token_digest(producer_inference_token), + "token_env": LAYOUT.producer_token_env, + # The launcher's --param optimizer_model lands here at run + # time; the compiled list below is only the build's default. + "override_env": LAYOUT.producer_scope_env, **config.inference_gateway.producer.model_dump(mode="json"), }, "evaluation": { - "token_sha256": token_digest(evaluation_inference_token), + "token_env": LAYOUT.evaluation_token_env, **config.inference_gateway.evaluation.model_dump(mode="json"), }, "finalization": { - "token_sha256": token_digest(finalization_inference_token), + "token_env": LAYOUT.finalization_token_env, **finalization_spec.model_dump(mode="json"), }, }, @@ -738,8 +756,13 @@ def compile_harbor_task( config.inference_gateway.upstream_base_url_env ), "upstream_base_url_target": UPSTREAM_BASE_URL_ENV, - "producer_api_key": producer_inference_token, "producer_base_url": (PRODUCER_BASE_URL), + # Names, not values: the launcher mints the tokens and chooses + # the producer scope, then exports them under these names. + "producer_token_env": LAYOUT.producer_token_env, + "evaluation_token_env": LAYOUT.evaluation_token_env, + "finalization_token_env": LAYOUT.finalization_token_env, + "producer_scope_env": LAYOUT.producer_scope_env, }, ensure_ascii=False, indent=2, @@ -772,7 +795,8 @@ def compile_harbor_task( ) ), "harbor_requirement": config.harbor_requirement, - "secrets": task_environment, + "secrets": [*task_environment, *runtime_environment], + "runtime_environment": runtime_environment, "sidecar_secrets": config.secrets, "inference_gateway": config.inference_gateway, "gateway_environment": gateway_environment, @@ -783,8 +807,6 @@ def compile_harbor_task( # avoid emitting the same key twice, not to permit anything: adding a new # credential to `secrets` gets it blanked automatically. "scrubbed_main_environment": scrubbed_main_environment, - "producer_inference_token": producer_inference_token, - "evaluation_inference_token": evaluation_inference_token, "inference_gateway_url": INFERENCE_GATEWAY_URL, "read_only_paths": config.read_only_paths, "local_task_source": local_task_source, diff --git a/vero/src/vero/harbor/build/templates/docker-compose.yaml.j2 b/vero/src/vero/harbor/build/templates/docker-compose.yaml.j2 index 94d940dc..c3de0977 100644 --- a/vero/src/vero/harbor/build/templates/docker-compose.yaml.j2 +++ b/vero/src/vero/harbor/build/templates/docker-compose.yaml.j2 @@ -8,7 +8,7 @@ services: {{ secret }}: "" {% endfor %} {% if inference_gateway %} - {{ layout.producer_api_key_env }}: "{{ producer_inference_token }}" + {{ layout.producer_api_key_env }}: "${{ '{' }}{{ layout.producer_token_env }}:?{{ layout.producer_token_env }} must be set for the optimizer{{ '}' }}" {{ layout.producer_base_url_env }}: "{{ producer_base_url }}" {% elif layout.producer_api_key_env in secrets %} {{ layout.producer_api_key_env }}: "" @@ -50,6 +50,13 @@ services: {% else %} LANG: "C.UTF-8" {% endif %} +{% if inference_gateway %} +{# The sidecar's own scope tokens, and the launcher's producer scope so the budget + it shows the optimizer names the models the gateway will accept. #} + {{ layout.evaluation_token_env }}: "${{ '{' }}{{ layout.evaluation_token_env }}:?{{ layout.evaluation_token_env }} must be set for the eval sidecar{{ '}' }}" + {{ layout.finalization_token_env }}: "${{ '{' }}{{ layout.finalization_token_env }}:?{{ layout.finalization_token_env }} must be set for the eval sidecar{{ '}' }}" + {{ layout.producer_scope_env }}: "${{ '{' }}{{ layout.producer_scope_env }}:-{{ '}' }}" +{% endif %} {% if task_services_use_upstream %} {# Trusted sidecar receives the real upstream so it can hand task-owned eval services (LLM user-sims/graders) a reachable provider. The optimizer/main keeps @@ -87,6 +94,13 @@ services: environment: {% for secret in gateway_environment %} {{ secret }}: "${{ '{' }}{{ secret }}:?{{ secret }} must be set for the inference gateway{{ '}' }}" +{% endfor %} +{% for name in runtime_environment %} +{% if name == layout.producer_scope_env %} + {{ name }}: "${{ '{' }}{{ name }}:-{{ '}' }}" +{% else %} + {{ name }}: "${{ '{' }}{{ name }}:?{{ name }} must be set for the inference gateway{{ '}' }}" +{% endif %} {% endfor %} volumes: - inference_state:{{ layout.inference_dir }} diff --git a/vero/src/vero/harbor/cli.py b/vero/src/vero/harbor/cli.py index 82cd35f5..014af261 100644 --- a/vero/src/vero/harbor/cli.py +++ b/vero/src/vero/harbor/cli.py @@ -13,6 +13,7 @@ import tempfile import urllib.error import urllib.request +from collections.abc import Mapping from pathlib import Path import click @@ -279,7 +280,9 @@ def _litellm_base_url_args(agent: str, task: Path) -> list[str]: ] -def _kimi_gateway_args(agent: str, task: Path) -> list[str]: +def _kimi_gateway_args( + agent: str, task: Path, environment: Mapping[str, str] | None = None +) -> list[str]: """Point kimi-cli's provider at the gateway instead of api.openai.com. kimi-cli only accepts a model whose provider half is in its own table, and @@ -307,7 +310,7 @@ def _kimi_gateway_args(agent: str, task: Path) -> list[str]: try: launch = json.loads(path.read_text(encoding="utf-8")) base_url = launch["producer_base_url"] - api_key = launch["producer_api_key"] + api_key = _producer_token(launch, environment) except (OSError, json.JSONDecodeError, KeyError): return [] return [ @@ -487,6 +490,54 @@ def _agent_environment_blanks(task: Path) -> list[str]: return arguments +def _producer_token(launch: dict, environment: Mapping[str, str] | None) -> str: + """The optimizer's scope token for this run. + + A compiled task names the variable (``producer_token_env``) and the launcher + minted the value into ``environment``; a task compiled before tokens moved + out of the tree still carries ``producer_api_key`` directly. + """ + name = launch.get("producer_token_env") + if isinstance(name, str) and name: + value = (environment or os.environ).get(name, "") + if not value: + raise KeyError(name) + return value + return launch["producer_api_key"] + + +def _runtime_inference_environment(config, task: Path) -> dict[str, str]: + """Mint this run's scope tokens and choose its producer scope. + + Nothing per-run is baked into a compiled task; these four variables are how + the run reaches it. Each service reads its own once at container start, so + the values are fixed for the run however the launcher's environment changes + afterwards. The producer scope is the build's resolved ``--param`` values, so + the optimizer model stays a launch choice on a frozen task. + """ + path = task / "environment/gateway/launch.json" + if not path.exists(): + return {} + launch = json.loads(path.read_text(encoding="utf-8")) + names = [launch.get(k) for k in ("producer_token_env", "evaluation_token_env", + "finalization_token_env", "producer_scope_env")] + if not all(isinstance(n, str) and n for n in names): + return {} # compiled before tokens moved out of the tree + from vero.gateway.inference import generate_inference_token + + producer = config.inference_gateway.producer + scope = { + "allowed_models": list(producer.allowed_models), + "model_aliases": dict(producer.model_aliases), + } + return { + names[0]: generate_inference_token(), + names[1]: generate_inference_token(), + names[2]: generate_inference_token(), + names[3]: json.dumps(scope, separators=(",", ":")), + } + + def _compiled_run_environment( task: Path, overrides: dict[str, str] | None = None ) -> dict[str, str]: @@ -506,7 +557,7 @@ def _compiled_run_environment( launch = json.loads(path.read_text(encoding="utf-8")) api_source = launch["upstream_api_key_source"] api_target = launch["upstream_api_key_target"] - producer_api_key = launch["producer_api_key"] + producer_api_key = _producer_token(launch, environment) producer_base_url = launch["producer_base_url"] base_source = launch.get("upstream_base_url_source") base_target = launch["upstream_base_url_target"] @@ -614,16 +665,116 @@ def _parse_build_params(values: tuple[str, ...]) -> dict[str, str]: required=True, type=click.Path(path_type=Path, file_okay=False), ) +@click.option( + "--manifest", + "manifest_path", + type=click.Path(path_type=Path, dir_okay=False), + help="Also write a JSON manifest of every compiled file's SHA-256 here.", +) +@click.option( + "--check", + "check_path", + type=click.Path(path_type=Path, exists=True, dir_okay=False), + help="Compile to a temporary directory and fail if it differs from this manifest.", +) @_PARAM_OPTION -def build_command(config_path, output, params): - """Compile a build YAML into a runnable Harbor task directory.""" +def build_command(config_path, output, params, manifest_path, check_path): + """Compile a build YAML into a runnable Harbor task directory. + + The output holds nothing per-run (tokens and the producer scope arrive by + environment at launch), so two compiles of the same sources at the same vero + commit are byte-identical. `--manifest` records that state; `--check` + verifies a checkout still reproduces it, which is what "frozen" means here. + """ from vero.harbor.build import compile_harbor_task, load_harbor_build_config - compiled = compile_harbor_task( - load_harbor_build_config(config_path, params=_parse_build_params(params)), - output, - ) + config = load_harbor_build_config(config_path, params=_parse_build_params(params)) + if check_path is not None: + expected = json.loads(check_path.read_text(encoding="utf-8")) + with tempfile.TemporaryDirectory(prefix="vero-harbor-check-") as temporary: + actual = _compiled_manifest( + compile_harbor_task(config, Path(temporary) / "task"), config_path + ) + drift = _manifest_drift(expected, actual) + if drift: + raise click.ClickException( + f"compiled task drifts from {check_path}:\n " + "\n ".join(drift) + ) + click.echo(f"Compiled task reproduces {check_path} ({len(actual['files'])} files)") + return + compiled = compile_harbor_task(config, output) click.echo(f"Compiled Harbor task: {compiled}") + if manifest_path is not None: + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text( + json.dumps(_compiled_manifest(compiled, config_path), indent=1, sort_keys=True) + + "\n", + encoding="utf-8", + ) + click.echo(f"Wrote manifest: {manifest_path}") + + +def _compiled_manifest(compiled: Path, config_path: Path) -> dict: + """Every file in a compiled task with its SHA-256, plus what produced it.""" + import hashlib + + from vero.harbor.build import compiler as _compiler + + files: dict[str, str] = {} + # Two subtrees are copies of things pinned elsewhere (the vendored task data and + # the vero source), thousands of files between them. One hash per subtree keeps + # the manifest reviewable; drift inside them still shows as that subtree changing. + trees = ("environment/sidecar/task-source/", "environment/vero/") + tree_lines: dict[str, list[str]] = {t: [] for t in trees} + for path in sorted(p for p in Path(compiled).rglob("*") if p.is_file()): + relative = path.relative_to(compiled) + # A git index carries inode and mtime stat data, and reflogs carry wall-clock + # timestamps; neither is content. Objects and refs are, and stay in. + if ".git" in relative.parts and ( + relative.name == "index" or "logs" in relative.parts + ): + continue + posix = relative.as_posix() + digest = hashlib.sha256(path.read_bytes()).hexdigest() + tree = next((t for t in trees if posix.startswith(t)), None) + if tree is None: + files[posix] = digest + else: + tree_lines[tree].append(f"{posix}\0{digest}\n") + for tree, lines in tree_lines.items(): + if lines: + files[tree] = hashlib.sha256("".join(lines).encode()).hexdigest() + vero_root = Path(_compiler.__file__).resolve().parents[4] + try: + commit = subprocess.run( + ["git", "-C", str(vero_root), "rev-parse", "HEAD"], + capture_output=True, text=True, check=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + commit = None + return { + "config": Path(config_path).name, + # Informational: the manifest is committed alongside the code it hashes, so + # its own commit can never equal HEAD. Drift is decided by the files alone. + "vero_commit": commit, + "files": files, + } + + +def _manifest_drift(expected: dict, actual: dict) -> list[str]: + """Human-readable differences between two manifests' file tables.""" + before, after = expected.get("files", {}), actual.get("files", {}) + drift = [f"missing: {p}" for p in sorted(set(before) - set(after))] + drift += [f"new: {p}" for p in sorted(set(after) - set(before))] + drift += [f"changed: {p}" for p in sorted(set(before) & set(after)) if before[p] != after[p]] + if drift and expected.get("vero_commit") and actual.get("vero_commit") and ( + expected["vero_commit"] != actual["vero_commit"] + ): + drift.append( + f"(manifest written at vero {expected['vero_commit'][:12]}, " + f"now {actual['vero_commit'][:12]})" + ) + return drift #: The two request shapes an OpenAI-compatible upstream may accept. Agents in @@ -812,6 +963,7 @@ def run_command(config_path, agent, model, environment, params, env_file, extra) config, Path(temporary) / "task", ) + overrides = {**(overrides or {}), **_runtime_inference_environment(config, task)} command = [ uvx, "--python", @@ -838,7 +990,7 @@ def run_command(config_path, agent, model, environment, params, env_file, extra) command.extend(_agent_environment_blanks(task)) command.extend(_opencode_gateway_args(agent, model, task)) command.extend(_litellm_base_url_args(agent, task)) - command.extend(_kimi_gateway_args(agent, task)) + command.extend(_kimi_gateway_args(agent, task, overrides)) # Build-declared outer-trial flags first, so a command-line arg can still # override them (harbor's `--ek` takes the last value for a key). command.extend(config.optimizer_harbor_args) diff --git a/vero/src/vero/harbor/deployment.py b/vero/src/vero/harbor/deployment.py index 0bf66710..8056e90f 100644 --- a/vero/src/vero/harbor/deployment.py +++ b/vero/src/vero/harbor/deployment.py @@ -2,8 +2,11 @@ from __future__ import annotations +import json import logging +import os import shutil +from collections.abc import Mapping from pathlib import Path, PurePosixPath from typing import Annotated, Literal @@ -24,6 +27,7 @@ from vero.evaluation.backends.command import CommandBackend, CommandBackendConfig from vero.evaluation.engine import EvaluationEngine from vero.harbor.backend import HarborBackend, HarborBackendConfig +from vero.layout import LAYOUT from vero.models import StrictModel from vero.runtime.artifacts import ArtifactStore from vero.sandbox import LocalSandbox @@ -113,7 +117,53 @@ class SidecarWandbConfig(StrictModel): def _build_backend(config: DeploymentBackendConfig) -> EvaluationBackend: if isinstance(config, CommandBackendConfig): return CommandBackend(config) - return HarborBackend(config) + return HarborBackend(resolve_runtime_secrets(config)) + + +def resolve_runtime_secrets( + config: HarborBackendConfig, environ: Mapping[str, str] | None = None +) -> HarborBackendConfig: + """Fill the gateway tokens from the environment the sidecar started with. + + Read once, here, at component build; a running container's environment + cannot change, so the tokens are fixed for the run. A named variable that is + unset is an error rather than a silent fall-back to the evaluation scope. + """ + environ = os.environ if environ is None else environ + update: dict[str, str] = {} + for field, env_field in ( + ("inference_gateway_token", "inference_gateway_token_env"), + ("inference_gateway_finalization_token", "inference_gateway_finalization_token_env"), + ): + name = getattr(config, env_field) + if name is None or getattr(config, field) is not None: + continue + value = environ.get(name, "") + if not value.strip(): + raise RuntimeError(f"{env_field} names {name}, which is not set") + update[field] = value + return config.model_copy(update=update) if update else config + + +def producer_scope_override( + environ: Mapping[str, str] | None = None, +) -> dict[str, JsonValue] | None: + """The launcher's producer allow-list, if it supplied one. + + Same variable and same shape the gateway reads (``LAYOUT.producer_scope_env``), + so the budget the sidecar shows the optimizer names the same models the + gateway will accept. + """ + environ = os.environ if environ is None else environ + raw = environ.get(LAYOUT.producer_scope_env, "") + if not raw.strip(): + return None + override = json.loads(raw) + if not isinstance(override, dict) or set(override) - {"allowed_models", "model_aliases"}: + raise RuntimeError( + f"{LAYOUT.producer_scope_env} may only set allowed_models and model_aliases" + ) + return override class HarborDeploymentConfig(StrictModel): @@ -269,6 +319,12 @@ def _ledger( async def build_harbor_components(config: dict) -> SidecarComponents: """Build the standard compiled-task topology from trusted JSON config.""" parsed = HarborDeploymentConfig.model_validate(config) + override = producer_scope_override() + if override is not None and "producer" in parsed.inference_limits: + parsed.inference_limits["producer"] = { + **parsed.inference_limits["producer"], + **override, + } session_dir = Path(parsed.session_dir) session_dir.mkdir(parents=True, exist_ok=True) # The session dir holds the trusted state — held-out evaluation records and diff --git a/vero/src/vero/layout.py b/vero/src/vero/layout.py index 741df26c..1e9f8080 100644 --- a/vero/src/vero/layout.py +++ b/vero/src/vero/layout.py @@ -85,6 +85,16 @@ class TaskLayout: # in, so routed_credential_envs below is the single list both sides use. producer_api_key_env: str = "OPENAI_API_KEY" producer_base_url_env: str = "OPENAI_BASE_URL" + # Per-run inference inputs, minted or chosen by the launcher and read ONCE by + # each service at container start. They exist so a compiled task can be frozen: + # nothing run-specific is baked into it, and the environment of a running + # container cannot be changed from outside, so the values are fixed for the run. + producer_token_env: str = "VERO_PRODUCER_TOKEN" + evaluation_token_env: str = "VERO_EVALUATION_TOKEN" + finalization_token_env: str = "VERO_FINALIZATION_TOKEN" + # JSON {"allowed_models": [...], "model_aliases": {...}} replacing the compiled + # producer scope, so the optimizer model is a launch choice, not a compile one. + producer_scope_env: str = "VERO_PRODUCER_SCOPE" # The gateway's proxy route, with the FastAPI parameter names it binds. Both # the route the gateway serves and every URL built for it derive from this one # string, so a caller cannot construct a path the gateway will not match. @@ -140,6 +150,16 @@ def routed_credential_envs(self) -> tuple[str, ...]: """Names the compose file sets explicitly, so must not also blank.""" return (self.producer_api_key_env, self.producer_base_url_env) + @property + def runtime_inference_envs(self) -> tuple[str, ...]: + """Every per-run inference input the launcher must supply.""" + return ( + self.producer_token_env, + self.evaluation_token_env, + self.finalization_token_env, + self.producer_scope_env, + ) + @property def sidecar_url(self) -> str: return f"http://{self.sidecar_host}:{self.sidecar_port}" diff --git a/vero/tests/test_v05_harbor_build.py b/vero/tests/test_v05_harbor_build.py index f6f1b564..7951a14d 100644 --- a/vero/tests/test_v05_harbor_build.py +++ b/vero/tests/test_v05_harbor_build.py @@ -325,20 +325,23 @@ def test_compiler_reserves_finalization_scope_defaulting_to_evaluation(tmp_path) assert "finalization" in scopes assert scopes["finalization"]["allowed_models"] == ["gpt-target"] assert scopes["finalization"]["max_tokens"] == 100000000 - # its token is distinct from the evaluation token (optimizer can't drain it) - assert ( - scopes["finalization"]["token_sha256"] != scopes["evaluation"]["token_sha256"] - ) - # and the sidecar backend carries a finalization token distinct from the eval one + # its token arrives under its own variable, distinct from the evaluation one + # (so the optimizer cannot drain it); nothing is baked, only the names. + assert scopes["finalization"]["token_env"] == LAYOUT.finalization_token_env + assert scopes["evaluation"]["token_env"] == LAYOUT.evaluation_token_env + assert scopes["finalization"]["token_env"] != scopes["evaluation"]["token_env"] + assert "token_sha256" not in scopes["finalization"] + # and the sidecar backend names the same two variables serve = json.loads( (output / "environment/sidecar/serve.json").read_text(encoding="utf-8") ) backend = next(iter(serve["backends"].values())) - assert backend["inference_gateway_finalization_token"] is not None + assert backend["inference_gateway_token_env"] == LAYOUT.evaluation_token_env assert ( - backend["inference_gateway_finalization_token"] - != backend["inference_gateway_token"] + backend["inference_gateway_finalization_token_env"] + == LAYOUT.finalization_token_env ) + assert backend.get("inference_gateway_token") is None # Per-request logging is on by default: the gateway captures every # request/response on its state volume and the sidecar mirrors it. The # experimental thread-attribution stamping stays off unless opted into. @@ -1133,6 +1136,11 @@ def test_compiler_isolates_upstream_inference_credentials(tmp_path, monkeypatch) "TEST_MODAL_TOKEN": "${TEST_MODAL_TOKEN}", "VERO_INFERENCE_UPSTREAM_API_KEY": "${VERO_INFERENCE_UPSTREAM_API_KEY}", "VERO_INFERENCE_UPSTREAM_BASE_URL": "${VERO_INFERENCE_UPSTREAM_BASE_URL}", + # per-run inputs the launcher mints; harbor passes them to compose like secrets + "VERO_PRODUCER_TOKEN": "${VERO_PRODUCER_TOKEN}", + "VERO_EVALUATION_TOKEN": "${VERO_EVALUATION_TOKEN}", + "VERO_FINALIZATION_TOKEN": "${VERO_FINALIZATION_TOKEN}", + "VERO_PRODUCER_SCOPE": "${VERO_PRODUCER_SCOPE}", } compose = yaml.safe_load((output / "environment/docker-compose.yaml").read_text()) assert set(compose["services"]) == { @@ -1147,27 +1155,44 @@ def test_compiler_isolates_upstream_inference_credentials(tmp_path, monkeypatch) assert main_environment["OPENAI_API_KEY"] != "real-provider-secret" assert main_environment["OPENAI_BASE_URL"].endswith("/scopes/producer/optimizer/v1") assert compose["services"]["eval-sidecar"]["environment"] == { - "TEST_MODAL_TOKEN": "${TEST_MODAL_TOKEN:?TEST_MODAL_TOKEN must be set for the eval sidecar}" + "TEST_MODAL_TOKEN": "${TEST_MODAL_TOKEN:?TEST_MODAL_TOKEN must be set for the eval sidecar}", + "VERO_EVALUATION_TOKEN": "${VERO_EVALUATION_TOKEN:?VERO_EVALUATION_TOKEN must be set for the eval sidecar}", + "VERO_FINALIZATION_TOKEN": "${VERO_FINALIZATION_TOKEN:?VERO_FINALIZATION_TOKEN must be set for the eval sidecar}", + "VERO_PRODUCER_SCOPE": "${VERO_PRODUCER_SCOPE:-}", } assert compose["services"]["inference-gateway"]["environment"] == { "VERO_INFERENCE_UPSTREAM_API_KEY": "${VERO_INFERENCE_UPSTREAM_API_KEY:?VERO_INFERENCE_UPSTREAM_API_KEY must be set for the inference gateway}", "VERO_INFERENCE_UPSTREAM_BASE_URL": "${VERO_INFERENCE_UPSTREAM_BASE_URL:?VERO_INFERENCE_UPSTREAM_BASE_URL must be set for the inference gateway}", + "VERO_PRODUCER_TOKEN": "${VERO_PRODUCER_TOKEN:?VERO_PRODUCER_TOKEN must be set for the inference gateway}", + "VERO_EVALUATION_TOKEN": "${VERO_EVALUATION_TOKEN:?VERO_EVALUATION_TOKEN must be set for the inference gateway}", + "VERO_FINALIZATION_TOKEN": "${VERO_FINALIZATION_TOKEN:?VERO_FINALIZATION_TOKEN must be set for the inference gateway}", + "VERO_PRODUCER_SCOPE": "${VERO_PRODUCER_SCOPE:-}", } gateway = json.loads((output / "environment/gateway/config.json").read_text()) assert "real-provider-secret" not in json.dumps(gateway) - assert gateway["scopes"]["producer"]["token_sha256"] + # Nothing per-run is baked: the gateway names the variables it will read once + # at start, and the producer scope can be replaced from the launcher. + assert gateway["scopes"]["producer"]["token_env"] == "VERO_PRODUCER_TOKEN" + assert gateway["scopes"]["producer"]["override_env"] == "VERO_PRODUCER_SCOPE" + assert "token_sha256" not in gateway["scopes"]["producer"] launch = json.loads((output / "environment/gateway/launch.json").read_text()) assert launch["upstream_api_key_source"] == "TEST_UPSTREAM_KEY" assert launch["upstream_api_key_target"] == "VERO_INFERENCE_UPSTREAM_API_KEY" - assert launch["producer_api_key"] == main_environment["OPENAI_API_KEY"] + assert "producer_api_key" not in launch + assert launch["producer_token_env"] == "VERO_PRODUCER_TOKEN" + # The optimizer's key is the compose-time value of that same variable. + assert main_environment["OPENAI_API_KEY"].startswith("${VERO_PRODUCER_TOKEN:?") + # The main container is blanked for the raw variables; only OPENAI_* carries the scope. + for name in ("VERO_PRODUCER_TOKEN", "VERO_EVALUATION_TOKEN", "VERO_FINALIZATION_TOKEN"): + assert main_environment[name] == "" seed = (output / "environment/main/seed.sh").read_text() assert 'model_provider = "vero_gateway"' in seed assert "supports_websockets = false" in seed serve = json.loads((output / "environment/sidecar/serve.json").read_text()) backend = serve["backends"]["harbor-validation"] assert backend["passthrough_environment"] == ["TEST_MODAL_TOKEN"] - assert backend["inference_gateway_token"] - assert backend["inference_gateway_token"] != "real-provider-secret" + assert backend["inference_gateway_token_env"] == "VERO_EVALUATION_TOKEN" + assert backend.get("inference_gateway_token") is None assert "real-provider-secret" not in json.dumps(serve) assert (output / "environment/gateway/Dockerfile").is_file() @@ -1689,3 +1714,58 @@ def test_outer_trial_limits_apply_to_a_command_backend_too(tmp_path): optimizer_sandbox_idle_timeout_seconds=3600, ) assert config.optimizer_agent_timeout_seconds == 72000 +def test_compile_is_deterministic_so_a_task_can_be_frozen(tmp_path): + """Two compiles of one config at one vero commit hash identically. + + This is what `vero harbor build --check` relies on. Tokens are not minted at + compile, the baseline commit carries a fixed date, and the manifest skips the + git index and reflogs, which hold stat data rather than content. + """ + from vero.harbor.cli import _compiled_manifest, _manifest_drift + + config = _config( + tmp_path, + inference_gateway=InferenceGatewaySpec( + producer=InferenceBudgetSpec(allowed_models=["gpt-producer"]), + evaluation=InferenceBudgetSpec(allowed_models=["gpt-target"]), + ), + ) + first = compile_harbor_task(config, tmp_path / "one", vero_root=Path(__file__).parents[1]) + second = compile_harbor_task(config, tmp_path / "two", vero_root=Path(__file__).parents[1]) + a = _compiled_manifest(first, tmp_path / "build.yaml") + b = _compiled_manifest(second, tmp_path / "build.yaml") + assert _manifest_drift(a, b) == [] + assert a["files"] + assert any(p.startswith("environment/agent-baseline/.git/objects/") for p in a["files"]) + assert not any(p.endswith("/.git/index") for p in a["files"]) + + +def test_launcher_mints_run_inputs_matching_the_compiled_names(tmp_path): + from vero.harbor.cli import _runtime_inference_environment + + config = _config( + tmp_path, + inference_gateway=InferenceGatewaySpec( + producer=InferenceBudgetSpec( + allowed_models=["anthropic/claude-opus-5", "claude-opus-5"], + model_aliases={"gpt-5.6-sol": "azure_ai/gpt-5.6-sol"}, + ), + evaluation=InferenceBudgetSpec(allowed_models=["gpt-target"]), + ), + ) + output = compile_harbor_task(config, tmp_path / "compiled", vero_root=Path(__file__).parents[1]) + runtime = _runtime_inference_environment(config, output) + assert set(runtime) == { + "VERO_PRODUCER_TOKEN", "VERO_EVALUATION_TOKEN", "VERO_FINALIZATION_TOKEN", + "VERO_PRODUCER_SCOPE", + } + tokens = {runtime[k] for k in runtime if k.endswith("_TOKEN")} + assert len(tokens) == 3 and all(len(t) > 20 for t in tokens) + assert json.loads(runtime["VERO_PRODUCER_SCOPE"]) == { + "allowed_models": ["anthropic/claude-opus-5", "claude-opus-5"], + "model_aliases": {"gpt-5.6-sol": "azure_ai/gpt-5.6-sol"}, + } + # And the same names appear in the task.toml environment so harbor passes them on. + task = tomllib.loads((output / "task.toml").read_text(encoding="utf-8")) + for name in runtime: + assert task["environment"]["env"][name] == "${" + name + "}" diff --git a/vero/tests/test_v05_harbor_deployment.py b/vero/tests/test_v05_harbor_deployment.py index 25c0a4b2..4daa54cd 100644 --- a/vero/tests/test_v05_harbor_deployment.py +++ b/vero/tests/test_v05_harbor_deployment.py @@ -343,3 +343,47 @@ def _explode(**kwargs): assert recorded["project"] == "vero-tests" assert recorded["error_type"] == "ValueError" assert "valid URL" in recorded["error"] + + +def test_sidecar_resolves_gateway_tokens_from_its_environment_once(tmp_path): + from vero.harbor.backend import HarborBackendConfig + from vero.harbor.deployment import producer_scope_override, resolve_runtime_secrets + + cases = tmp_path / "v.jsonl" + cases.write_text('{"case_id": "a"}\n') + config = HarborBackendConfig( + task_source="org/tasks@1", + agent_import_path="a.b:C", + cases_path=str(cases), + harbor_requirement="harbor==0.1", + evaluation_set_name="v", + partition="validation", + inference_gateway_url="http://inference-gateway:8001", + inference_gateway_token_env="VERO_EVALUATION_TOKEN", + inference_gateway_finalization_token_env="VERO_FINALIZATION_TOKEN", + ) + resolved = resolve_runtime_secrets( + config, {"VERO_EVALUATION_TOKEN": "e", "VERO_FINALIZATION_TOKEN": "f"} + ) + assert resolved.inference_gateway_token == "e" + assert resolved.inference_gateway_finalization_token == "f" + assert config.inference_gateway_token is None # the compiled config is untouched + import pytest as _pytest + + with _pytest.raises(RuntimeError, match="VERO_FINALIZATION_TOKEN"): + resolve_runtime_secrets(config, {"VERO_EVALUATION_TOKEN": "e"}) + # A url with neither a token nor a token variable is still rejected. + with _pytest.raises(ValueError, match="must be set together"): + HarborBackendConfig( + task_source="org/tasks@1", + agent_import_path="a.b:C", + cases_path=str(cases), + harbor_requirement="harbor==0.1", + evaluation_set_name="v", + partition="validation", + inference_gateway_url="http://inference-gateway:8001", + ) + assert producer_scope_override({}) is None + assert producer_scope_override( + {"VERO_PRODUCER_SCOPE": json.dumps({"allowed_models": ["m"]})} + ) == {"allowed_models": ["m"]} diff --git a/vero/tests/test_v05_harbor_inference.py b/vero/tests/test_v05_harbor_inference.py index b8db35e6..06ea3bfd 100644 --- a/vero/tests/test_v05_harbor_inference.py +++ b/vero/tests/test_v05_harbor_inference.py @@ -3,6 +3,8 @@ import asyncio import json +import pytest + import httpx from fastapi.testclient import TestClient @@ -1186,3 +1188,60 @@ def upstream(request: httpx.Request): ("vendor_x/gpt-test", "gpt-test"), ("plain", None), ] + + +def test_resolve_gateway_config_reads_tokens_and_producer_scope_once(): + from vero.gateway.inference import ( + InferenceGatewayConfig, + InferenceScopeConfig, + create_inference_gateway_app, + resolve_gateway_config, + token_digest, + ) + + config = InferenceGatewayConfig( + scopes={ + "producer": InferenceScopeConfig( + token_env="VERO_PRODUCER_TOKEN", + override_env="VERO_PRODUCER_SCOPE", + allowed_models=["openai/gpt-5.4"], + ), + "evaluation": InferenceScopeConfig( + token_env="VERO_EVALUATION_TOKEN", allowed_models=["target"] + ), + } + ) + with pytest.raises(ValueError, match="resolve_gateway_config"): + create_inference_gateway_app(config=config, upstream_api_key="k") + + environ = { + "VERO_PRODUCER_TOKEN": "p-token", + "VERO_EVALUATION_TOKEN": "e-token", + "VERO_PRODUCER_SCOPE": json.dumps( + {"allowed_models": ["claude-opus-5"], "model_aliases": {"a": "b"}} + ), + } + resolved = resolve_gateway_config(config, environ) + assert resolved.scopes["producer"].token_sha256 == token_digest("p-token") + assert resolved.scopes["evaluation"].token_sha256 == token_digest("e-token") + assert resolved.scopes["producer"].allowed_models == ["claude-opus-5"] + assert resolved.scopes["producer"].model_aliases == {"a": "b"} + assert resolved.scopes["evaluation"].allowed_models == ["target"] + # The compiled config is untouched, and a later change to the environment + # does not reach the resolved one. + assert config.scopes["producer"].allowed_models == ["openai/gpt-5.4"] + environ["VERO_PRODUCER_SCOPE"] = json.dumps({"allowed_models": ["other"]}) + assert resolved.scopes["producer"].allowed_models == ["claude-opus-5"] + + environ["VERO_PRODUCER_SCOPE"] = "" + assert resolve_gateway_config(config, environ).scopes["producer"].allowed_models == [ + "openai/gpt-5.4" + ] + with pytest.raises(RuntimeError, match="VERO_EVALUATION_TOKEN"): + resolve_gateway_config(config, {"VERO_PRODUCER_TOKEN": "p"}) + with pytest.raises(RuntimeError, match="may only set"): + resolve_gateway_config( + config, {**environ, "VERO_PRODUCER_SCOPE": json.dumps({"max_tokens": 1})} + ) + with pytest.raises(ValueError, match="token_sha256 or token_env"): + InferenceScopeConfig(allowed_models=["x"]) From 03aa50850e28d1fe81c9719f9c487ef16f3fafd3 Mon Sep 17 00:00:00 2001 From: Varun Ursekar Date: Sat, 12 Sep 2026 15:03:29 -0700 Subject: [PATCH 2/8] Let a build pin a published vero instead of copying the source tree New build field vero_requirement (an exact pin such as scaleapi-vero==0.6.0). When set, the compiler skips the source copy and both images install that requirement with the extras each needs; the compiled task then holds no vero and is small enough to commit (GAIA shell: 1.5 MB against 6.4 MB). The pin must equal the vero doing the compiling, since the images read the configs it wrote; a mismatch is refused at compile time. Unset everywhere for now: 0.5.0 on PyPI predates this branch's gateway and outer-limit changes, so a task compiled against it would not start. The GAIA yamls carry the line commented out with that explanation. Manifests are regenerated after the rebase onto the renamed package. Co-Authored-By: Claude Fable 5.1 --- .../baseline/compiled.manifest.json | 4 +- .../gaia/baseline/build.shell.yaml | 4 ++ harness-opt-bench/gaia/baseline/build.yaml | 4 ++ .../gaia/baseline/compiled.manifest.json | 4 +- .../officeqa/baseline/compiled.manifest.json | 4 +- .../baseline/compiled.manifest.json | 4 +- vero/src/vero/harbor/build/compiler.py | 41 ++++++++++++++++--- vero/src/vero/harbor/build/config.py | 19 +++++++++ vero/tests/test_v05_harbor_build.py | 31 ++++++++++++++ 9 files changed, 101 insertions(+), 14 deletions(-) diff --git a/harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json b/harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json index 288e517b..095c0a8a 100644 --- a/harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json +++ b/harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json @@ -83,11 +83,11 @@ "environment/sidecar/cases/validation.jsonl": "4d46fc533edc1862d9ee04f8b6fcb835c017a9360c8a1a88cb7a6963cd54aa91", "environment/sidecar/serve.json": "92aa6ad2ec2e7967bcf50d694f55a1efc00f911a1979a0d3e297bab6ebf903d3", "environment/sidecar/task-source/": "abfee46a96f0ffa72ad3b4b287227ffb32a42cdc9188e1dddc72685dfcfa7c30", - "environment/vero/": "9ccf5b0d3ae10a57f561f9e92777eccc905ee159396b004498bdbca0d71e9160", + "environment/vero/": "1855222d415aba68140d0576f06c5388d491c17520994c12680402828c95706f", "instruction.md": "2665ef594b9dffcc447e8013ac15c633cd6baf69b7e019fef92382f479c71cb7", "solution/solve.sh": "906e1e8dcd72edd2fb1f11f36bb29f2df7c203269ffd7badf9fa7632bcd98544", "task.toml": "a23272759e2304e32610da127c97098121e980bc38568a602175d039d807c7e5", "tests/test.sh": "b65e90e6d67aacedd559c6b1c9d0f1d5c5071ecc9178a5bd7f9e1066f332bb0d" }, - "vero_commit": "21cb81954019c5db6e54f196d2bf048c2e3d2ba0" + "vero_commit": "64e76571afcb2827ab7bfeddda2f16a9ad6544c9" } diff --git a/harness-opt-bench/gaia/baseline/build.shell.yaml b/harness-opt-bench/gaia/baseline/build.shell.yaml index 1b7cfcef..f08b0dfa 100644 --- a/harness-opt-bench/gaia/baseline/build.shell.yaml +++ b/harness-opt-bench/gaia/baseline/build.shell.yaml @@ -12,6 +12,10 @@ task_source: gaia/gaia@sha256:bbc356f476e0b70ba77da11a9be7d6345918d1e4a2daade0d6 task_manifest: ../partitions/manifest.json agent_import_path: gaia_agent.agent:GaiaAgent harbor_requirement: harbor[modal]==0.20.0 +# Set once the vero on PyPI carries the compiled-task changes this branch relies on +# (env-delivered scope tokens, outer-trial limits); until then the images copy the +# local source. With it set, the compiled task holds no vero and can be committed. +# vero_requirement: scaleapi-vero==0.6.0 partition_files: development: ../partitions/development.json diff --git a/harness-opt-bench/gaia/baseline/build.yaml b/harness-opt-bench/gaia/baseline/build.yaml index 9bd8a9da..a9c705ec 100644 --- a/harness-opt-bench/gaia/baseline/build.yaml +++ b/harness-opt-bench/gaia/baseline/build.yaml @@ -7,6 +7,10 @@ task_source: gaia/gaia@sha256:bbc356f476e0b70ba77da11a9be7d6345918d1e4a2daade0d6 task_manifest: ../partitions/manifest.json agent_import_path: gaia_agent.agent:GaiaAgent harbor_requirement: harbor[modal]==0.20.0 +# Set once the vero on PyPI carries the compiled-task changes this branch relies on +# (env-delivered scope tokens, outer-trial limits); until then the images copy the +# local source. With it set, the compiled task holds no vero and can be committed. +# vero_requirement: scaleapi-vero==0.6.0 partition_files: development: ../partitions/development.json diff --git a/harness-opt-bench/gaia/baseline/compiled.manifest.json b/harness-opt-bench/gaia/baseline/compiled.manifest.json index 5ee80e73..341d1c9a 100644 --- a/harness-opt-bench/gaia/baseline/compiled.manifest.json +++ b/harness-opt-bench/gaia/baseline/compiled.manifest.json @@ -82,11 +82,11 @@ "environment/sidecar/cases/test.jsonl": "9cf01d06d194f4e7de3257189b7629dea6d33c572a913d98147ec3b05c338044", "environment/sidecar/cases/validation.jsonl": "292ddd833d0a584c648349dcea0eca5c8dec170fe596110df6a300935d0e3c1e", "environment/sidecar/serve.json": "0befd4c90605d1aa2d798a89222f62b1da80426c4a2d0c1325ea28a82c72606c", - "environment/vero/": "9ccf5b0d3ae10a57f561f9e92777eccc905ee159396b004498bdbca0d71e9160", + "environment/vero/": "1855222d415aba68140d0576f06c5388d491c17520994c12680402828c95706f", "instruction.md": "ee539ac13ad8bdcaf5c3c2bd7465eb44db72001cb7181793c02fe99d440d88f6", "solution/solve.sh": "10923f62ac228055ab21388df27d9131a457bb51bafbb883cfd425e3c7d0a4ad", "task.toml": "473186cb0a19314df24cf7114469da5c7504de1a91b27830123b5f7e42490d68", "tests/test.sh": "b65e90e6d67aacedd559c6b1c9d0f1d5c5071ecc9178a5bd7f9e1066f332bb0d" }, - "vero_commit": "21cb81954019c5db6e54f196d2bf048c2e3d2ba0" + "vero_commit": "64e76571afcb2827ab7bfeddda2f16a9ad6544c9" } diff --git a/harness-opt-bench/officeqa/baseline/compiled.manifest.json b/harness-opt-bench/officeqa/baseline/compiled.manifest.json index f993f032..f65095c1 100644 --- a/harness-opt-bench/officeqa/baseline/compiled.manifest.json +++ b/harness-opt-bench/officeqa/baseline/compiled.manifest.json @@ -83,11 +83,11 @@ "environment/sidecar/cases/validation.jsonl": "14d86ab3071fb4091efd7b80687b1b8d8176c6b76dc1676d9a72a27f9d99032f", "environment/sidecar/serve.json": "f70daf21febb1f19b3cc63c109d92d85a4ebb862a8375a8ecd992ff28f1386ae", "environment/sidecar/task-source/": "be9f7f935eac438753a93bc1dccb66dc82a47c2911a9befad507d8f1e4e0480b", - "environment/vero/": "9ccf5b0d3ae10a57f561f9e92777eccc905ee159396b004498bdbca0d71e9160", + "environment/vero/": "1855222d415aba68140d0576f06c5388d491c17520994c12680402828c95706f", "instruction.md": "c9f995857987ecbf98ca9d1bb35dd3ecee240fc50c906cda2a464f75d879471b", "solution/solve.sh": "6472c890bec7a80fbbf5a07e232ee0aa33a2636ca34e919bb4083e470bf11630", "task.toml": "82490115a3279e0a53465b7a70b00f9e154cd3999179799e083befd538d010f8", "tests/test.sh": "b65e90e6d67aacedd559c6b1c9d0f1d5c5071ecc9178a5bd7f9e1066f332bb0d" }, - "vero_commit": "21cb81954019c5db6e54f196d2bf048c2e3d2ba0" + "vero_commit": "64e76571afcb2827ab7bfeddda2f16a9ad6544c9" } diff --git a/harness-opt-bench/terminal-bench/baseline/compiled.manifest.json b/harness-opt-bench/terminal-bench/baseline/compiled.manifest.json index 300b3bde..443ffb6a 100644 --- a/harness-opt-bench/terminal-bench/baseline/compiled.manifest.json +++ b/harness-opt-bench/terminal-bench/baseline/compiled.manifest.json @@ -82,11 +82,11 @@ "environment/sidecar/cases/test.jsonl": "f294e63d251b10418e318a392b7a2825e4fed834e65a46dd5d63e1d32b059357", "environment/sidecar/cases/validation.jsonl": "30c87786e8441a2d50c14e0b09d8f2a4caf3b35709acee2bdbbca97c0e121fcb", "environment/sidecar/serve.json": "b9576b9435c5f71a871c5f2a7a9ecc59fc8ba66df81c47a70315f916a4ed8ad8", - "environment/vero/": "9ccf5b0d3ae10a57f561f9e92777eccc905ee159396b004498bdbca0d71e9160", + "environment/vero/": "1855222d415aba68140d0576f06c5388d491c17520994c12680402828c95706f", "instruction.md": "044b5f8e0e5be1cb293898923b97cb3801399f2932dbec4b8a71f960c335576f", "solution/solve.sh": "c53ec7868ab20f6fa5bebc19c3c9a742e0419010be3f06e04e985d299c2de88c", "task.toml": "6832271042299c0131829efea187a85fb7598a2029b0dea3f675d7eebd62dda6", "tests/test.sh": "b65e90e6d67aacedd559c6b1c9d0f1d5c5071ecc9178a5bd7f9e1066f332bb0d" }, - "vero_commit": "21cb81954019c5db6e54f196d2bf048c2e3d2ba0" + "vero_commit": "64e76571afcb2827ab7bfeddda2f16a9ad6544c9" } diff --git a/vero/src/vero/harbor/build/compiler.py b/vero/src/vero/harbor/build/compiler.py index c4f1ddc0..93f9ba70 100644 --- a/vero/src/vero/harbor/build/compiler.py +++ b/vero/src/vero/harbor/build/compiler.py @@ -12,6 +12,7 @@ import subprocess import tarfile import tomllib +from importlib.metadata import PackageNotFoundError from importlib.metadata import version as distribution_version from pathlib import Path, PurePosixPath @@ -129,6 +130,34 @@ def _safe_extract_tar(payload: bytes, destination: Path) -> None: archive.extractall(destination, filter="data") +def _published_vero(requirement: str | None) -> str | None: + """The build's pinned vero requirement, checked against the vero compiling it. + + The compiled serve.json and gateway config are read by the vero inside the + images, so they must be written by the same version. A pin that names another + version would compile cleanly and fail at container start, or worse, run with a + config it half understands. + """ + if requirement is None: + return None + name, _, version = requirement.partition("==") + try: + installed = distribution_version(name) + except PackageNotFoundError: + return requirement # not importable under that name here; trust the pin + if installed != version: + raise ValueError( + f"vero_requirement pins {requirement} but this compiler is {name} " + f"{installed}; the images must run the version that compiled them" + ) + return requirement + + +def _with_extras(requirement: str, extras: str) -> str: + name, _, version = requirement.partition("==") + return f"{name}[{extras}]=={version}" + + #: Every compiled baseline commit is stamped with this instant, not the wall #: clock, so its hash is a function of the seed's contents only. BASELINE_COMMIT_DATE = "2000-01-01T00:00:00Z" @@ -561,8 +590,9 @@ def compile_harbor_task( """Emit a self-contained Harbor task directory from validated config.""" output = Path(output_dir).expanduser().resolve() source_root = (vero_root or Path(__file__).parents[4]).resolve() - use_local_vero = _is_vero_source(source_root) - if vero_root is not None and not use_local_vero: + published = _published_vero(config.vero_requirement) + use_local_vero = published is None and _is_vero_source(source_root) + if vero_root is not None and published is None and not use_local_vero: raise ValueError(f"vero_root {source_root} is not a scaleapi-vero source checkout") protected = [Path(config.agent_repo).resolve()] if use_local_vero: @@ -788,10 +818,9 @@ def compile_harbor_task( "vero_requirement": ( None if use_local_vero - else ( - "scaleapi-vero[" - + ("harbor,wandb" if config.wandb is not None else "harbor") - + f"]=={distribution_version('scaleapi-vero')}" + else _with_extras( + published or f"scaleapi-vero=={distribution_version('scaleapi-vero')}", + "harbor,wandb" if config.wandb is not None else "harbor", ) ), "harbor_requirement": config.harbor_requirement, diff --git a/vero/src/vero/harbor/build/config.py b/vero/src/vero/harbor/build/config.py index d2b2f1f2..2c0a433b 100644 --- a/vero/src/vero/harbor/build/config.py +++ b/vero/src/vero/harbor/build/config.py @@ -68,6 +68,12 @@ class _TaskIdentityFields(StrictModel): while the built-in opening line tells them to *improve* one. base_image_main: Base image for the main container. base_image_sidecar: Base image for the sidecar container. + vero_requirement: Install vero into the task images from this published + requirement (``scaleapi-vero==0.6.0``) instead of copying the source + checkout in. The version must equal the vero doing the compiling, + because the compiled serve and gateway configs are read by the vero + inside the images. None (the default) copies the local source, which + is what a checkout with unreleased changes needs. """ name: str @@ -79,6 +85,19 @@ class _TaskIdentityFields(StrictModel): instruction_template: str | None = None base_image_main: str = "ghcr.io/astral-sh/uv:python3.12-bookworm" base_image_sidecar: str = "ghcr.io/astral-sh/uv:python3.12-bookworm" + vero_requirement: str | None = None + + @field_validator("vero_requirement") + @classmethod + def validate_vero_requirement(cls, value: str | None) -> str | None: + if value is None: + return None + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*==[0-9][A-Za-z0-9.+!-]*", value): + raise ValueError( + "vero_requirement must be an exact pin like scaleapi-vero==0.6.0 " + "(no extras: the compiler adds the ones each image needs)" + ) + return value @field_validator("name", "agent_repo", "base_image_main", "base_image_sidecar") @classmethod diff --git a/vero/tests/test_v05_harbor_build.py b/vero/tests/test_v05_harbor_build.py index 7951a14d..5541787a 100644 --- a/vero/tests/test_v05_harbor_build.py +++ b/vero/tests/test_v05_harbor_build.py @@ -1769,3 +1769,34 @@ def test_launcher_mints_run_inputs_matching_the_compiled_names(tmp_path): task = tomllib.loads((output / "task.toml").read_text(encoding="utf-8")) for name in runtime: assert task["environment"]["env"][name] == "${" + name + "}" + + +def test_vero_requirement_installs_from_pypi_instead_of_copying_source(tmp_path): + """A pinned published vero keeps the source tree out of the compiled task.""" + from importlib.metadata import version + + pin = f"scaleapi-vero=={version('scaleapi-vero')}" + output = compile_harbor_task( + _config(tmp_path, vero_requirement=pin), tmp_path / "compiled", + vero_root=Path(__file__).parents[1], + ) + assert not (output / "environment/vero").exists() + main = (output / "environment/Dockerfile").read_text(encoding="utf-8") + sidecar = (output / "environment/sidecar/Dockerfile").read_text(encoding="utf-8") + assert f"scaleapi-vero[harbor]=={version('scaleapi-vero')}" in main + assert f"scaleapi-vero[harbor]=={version('scaleapi-vero')}" in sidecar + assert "COPY vero " not in main and "COPY vero " not in sidecar + # and the manifest no longer carries a vero subtree + from vero.harbor.cli import _compiled_manifest + + assert not any(p.startswith("environment/vero/") for p in _compiled_manifest(output, tmp_path / "b.yaml")["files"]) + + +def test_vero_requirement_must_match_the_compiling_vero(tmp_path): + with pytest.raises(ValueError, match="must run the version that compiled them"): + compile_harbor_task( + _config(tmp_path / "a", vero_requirement="scaleapi-vero==0.0.1"), + tmp_path / "compiled-a", vero_root=Path(__file__).parents[1], + ) + with pytest.raises(ValueError, match="exact pin"): + _config(tmp_path / "b", vero_requirement="scaleapi-vero[harbor]>=0.5") From f9467754fa730745f568f6b87173db821188376a Mon Sep 17 00:00:00 2001 From: Varun Ursekar Date: Sat, 12 Sep 2026 15:28:53 -0700 Subject: [PATCH 3/8] Run every sandbox in the enterprise Modal environment Harbor hands the Modal SDK no environment name, so sandboxes went to the workspace default (main), where every cell before 2026-09 ran. Declaring MODAL_ENVIRONMENT among each build's secrets carries it to the launcher and the sidecar alike; the env files set it to enterprise. Manifests regenerated for the extra task.toml line. Co-Authored-By: Claude Fable 5.1 --- .../browsecomp-plus/baseline/compiled.manifest.json | 12 ++++++------ .../gaia/baseline/compiled.manifest.json | 12 ++++++------ .../officeqa/baseline/compiled.manifest.json | 12 ++++++------ .../terminal-bench/baseline/compiled.manifest.json | 12 ++++++------ 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json b/harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json index 095c0a8a..1eddddf5 100644 --- a/harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json +++ b/harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json @@ -36,7 +36,7 @@ "environment/agent-baseline/src/browsecomp_plus_agent/agent.py": "137790f566ee7bc8939fb918fa5235c1e254f25598714f96e1eda10dd4d6f35e", "environment/agent-baseline/tests/test_agent.py": "25788ac708cf995f557b195e37bab7153af4291b3ae8f7d29fa4c48d2d02539d", "environment/agent-baseline/uv.lock": "e8f297525100fbd6c660b85970c7f02ec779ee35bbd8d3020108f7fd3146d05e", - "environment/agent-env-blanks.json": "90b8150c7380714e1b169d902c8cc153c58596bc8000dd8adeb49469f9af5699", + "environment/agent-env-blanks.json": "e527d19e8a42180386e30ec7219cf7dfa1127fda0340eb00be871f82fd45399f", "environment/agent-seed/.git/COMMIT_EDITMSG": "4b654bd1437066b13498661f3ca14774daf1066d072036beffaf06f0c014250e", "environment/agent-seed/.git/HEAD": "28d25bf82af4c0e2b72f50959b2beb859e3e60b9630a5e8c603dad4ddb2b6e80", "environment/agent-seed/.git/config": "cae33efdb02cf774435c1ff9cb16bcc1014606908530c6e1dc727615fe3e8cda", @@ -71,7 +71,7 @@ "environment/agent-seed/src/browsecomp_plus_agent/agent.py": "137790f566ee7bc8939fb918fa5235c1e254f25598714f96e1eda10dd4d6f35e", "environment/agent-seed/tests/test_agent.py": "25788ac708cf995f557b195e37bab7153af4291b3ae8f7d29fa4c48d2d02539d", "environment/agent-seed/uv.lock": "e8f297525100fbd6c660b85970c7f02ec779ee35bbd8d3020108f7fd3146d05e", - "environment/docker-compose.yaml": "db36414cd7b7015e76b36da47a7d6c5d93d8a9d1c53ac39fbf31363ef94a9e04", + "environment/docker-compose.yaml": "e29555f903e97c83d00da27df88dd317bf263bda1b2b2675462424837daeb938", "environment/gateway/Dockerfile": "70d149f1928b135b8e11b7bb8d4ef70d81f331a6766bb3947192ff876e2eea26", "environment/gateway/config.json": "650eabf9ff9899771d5d39b0aa6f233f8405bdd98a1261aab9c3bc5beaeb6f97", "environment/gateway/launch.json": "462bcc8d2749f1364d79e14d36be7144a84441b6bb74c985f798115ef26636ae", @@ -81,13 +81,13 @@ "environment/sidecar/cases/development.jsonl": "becd3eef6841ad0174b8ec4bc73088432221f8849c48a5c822b955e62003c17d", "environment/sidecar/cases/test.jsonl": "b20f6750e35cabdfb65c186ff886026f78ed4b37e3cb11e67fe7538531e1dc6d", "environment/sidecar/cases/validation.jsonl": "4d46fc533edc1862d9ee04f8b6fcb835c017a9360c8a1a88cb7a6963cd54aa91", - "environment/sidecar/serve.json": "92aa6ad2ec2e7967bcf50d694f55a1efc00f911a1979a0d3e297bab6ebf903d3", + "environment/sidecar/serve.json": "fcad4e44748f06d8e2d1296dca60a2b8674e94e8618a79539588554992d69bd3", "environment/sidecar/task-source/": "abfee46a96f0ffa72ad3b4b287227ffb32a42cdc9188e1dddc72685dfcfa7c30", - "environment/vero/": "1855222d415aba68140d0576f06c5388d491c17520994c12680402828c95706f", + "environment/vero/": "e12fbae407e3bf6acfaa6445e5debc683592b66180ab69b6e6eff2b395f93f30", "instruction.md": "2665ef594b9dffcc447e8013ac15c633cd6baf69b7e019fef92382f479c71cb7", "solution/solve.sh": "906e1e8dcd72edd2fb1f11f36bb29f2df7c203269ffd7badf9fa7632bcd98544", - "task.toml": "a23272759e2304e32610da127c97098121e980bc38568a602175d039d807c7e5", + "task.toml": "d0bba169e6e962f4fe90d13ebf675f732c8ed80ae3daa2f0b6f6eff8ce211ce8", "tests/test.sh": "b65e90e6d67aacedd559c6b1c9d0f1d5c5071ecc9178a5bd7f9e1066f332bb0d" }, - "vero_commit": "64e76571afcb2827ab7bfeddda2f16a9ad6544c9" + "vero_commit": "96e57cf22e5b7f3a4e8cea9ade61bf5f61c66692" } diff --git a/harness-opt-bench/gaia/baseline/compiled.manifest.json b/harness-opt-bench/gaia/baseline/compiled.manifest.json index 341d1c9a..bb97771d 100644 --- a/harness-opt-bench/gaia/baseline/compiled.manifest.json +++ b/harness-opt-bench/gaia/baseline/compiled.manifest.json @@ -36,7 +36,7 @@ "environment/agent-baseline/src/gaia_agent/agent.py": "54272d5adbad89b254e2a64789b4f37054c398e153e722f0d9f0e452e27c6926", "environment/agent-baseline/tests/test_agent.py": "11a9f3ea6b9a0daae3558f66b1ad044fda886ca86d5011a774ce20c24f57b70f", "environment/agent-baseline/uv.lock": "5e94e388c3050cf90b59b08cbe55030dd102e30475c171415a190da9969cf2c2", - "environment/agent-env-blanks.json": "90b8150c7380714e1b169d902c8cc153c58596bc8000dd8adeb49469f9af5699", + "environment/agent-env-blanks.json": "e527d19e8a42180386e30ec7219cf7dfa1127fda0340eb00be871f82fd45399f", "environment/agent-seed/.git/COMMIT_EDITMSG": "4b654bd1437066b13498661f3ca14774daf1066d072036beffaf06f0c014250e", "environment/agent-seed/.git/HEAD": "28d25bf82af4c0e2b72f50959b2beb859e3e60b9630a5e8c603dad4ddb2b6e80", "environment/agent-seed/.git/config": "cae33efdb02cf774435c1ff9cb16bcc1014606908530c6e1dc727615fe3e8cda", @@ -71,7 +71,7 @@ "environment/agent-seed/src/gaia_agent/agent.py": "54272d5adbad89b254e2a64789b4f37054c398e153e722f0d9f0e452e27c6926", "environment/agent-seed/tests/test_agent.py": "11a9f3ea6b9a0daae3558f66b1ad044fda886ca86d5011a774ce20c24f57b70f", "environment/agent-seed/uv.lock": "5e94e388c3050cf90b59b08cbe55030dd102e30475c171415a190da9969cf2c2", - "environment/docker-compose.yaml": "f67edf521e9b2c5a3983f27a9e941d824039fbd17caac20cc05fcbc317e8c4a6", + "environment/docker-compose.yaml": "1e9d9c8496de0d73ff5bcbaa05a1fcaa4eb753ccff9933388a30cc25c3b1bba1", "environment/gateway/Dockerfile": "70d149f1928b135b8e11b7bb8d4ef70d81f331a6766bb3947192ff876e2eea26", "environment/gateway/config.json": "52f4f8064ce5a310cf4a7cbec7d067233fea68e26432f610f0d59660cb4898dd", "environment/gateway/launch.json": "462bcc8d2749f1364d79e14d36be7144a84441b6bb74c985f798115ef26636ae", @@ -81,12 +81,12 @@ "environment/sidecar/cases/development.jsonl": "ef9a63afb40f7f08d9b95995857c5f81eb2f817006d14176cfb3e8676f688555", "environment/sidecar/cases/test.jsonl": "9cf01d06d194f4e7de3257189b7629dea6d33c572a913d98147ec3b05c338044", "environment/sidecar/cases/validation.jsonl": "292ddd833d0a584c648349dcea0eca5c8dec170fe596110df6a300935d0e3c1e", - "environment/sidecar/serve.json": "0befd4c90605d1aa2d798a89222f62b1da80426c4a2d0c1325ea28a82c72606c", - "environment/vero/": "1855222d415aba68140d0576f06c5388d491c17520994c12680402828c95706f", + "environment/sidecar/serve.json": "150b4d1a683f4f023a679c55514dbff77657d048e850ce7a409985eda75cf424", + "environment/vero/": "e12fbae407e3bf6acfaa6445e5debc683592b66180ab69b6e6eff2b395f93f30", "instruction.md": "ee539ac13ad8bdcaf5c3c2bd7465eb44db72001cb7181793c02fe99d440d88f6", "solution/solve.sh": "10923f62ac228055ab21388df27d9131a457bb51bafbb883cfd425e3c7d0a4ad", - "task.toml": "473186cb0a19314df24cf7114469da5c7504de1a91b27830123b5f7e42490d68", + "task.toml": "664b022bed341b1e41e8e657553ed942cb3611bbdf7e5a278e6fac2093665e38", "tests/test.sh": "b65e90e6d67aacedd559c6b1c9d0f1d5c5071ecc9178a5bd7f9e1066f332bb0d" }, - "vero_commit": "64e76571afcb2827ab7bfeddda2f16a9ad6544c9" + "vero_commit": "96e57cf22e5b7f3a4e8cea9ade61bf5f61c66692" } diff --git a/harness-opt-bench/officeqa/baseline/compiled.manifest.json b/harness-opt-bench/officeqa/baseline/compiled.manifest.json index f65095c1..2012d76e 100644 --- a/harness-opt-bench/officeqa/baseline/compiled.manifest.json +++ b/harness-opt-bench/officeqa/baseline/compiled.manifest.json @@ -36,7 +36,7 @@ "environment/agent-baseline/src/officeqa_agent/agent.py": "290e0c83797589b640ba93635edf2bd5c53f3880008c3c34f76496153b448367", "environment/agent-baseline/tests/test_agent.py": "1a2a6dd660deab8ce5c619756be0a6cad493d488c4217f28981e601d868ce986", "environment/agent-baseline/uv.lock": "256b8b915d8a1b7a8cc19619fa96d2f32869d615e9fe1055686228c02f138658", - "environment/agent-env-blanks.json": "90b8150c7380714e1b169d902c8cc153c58596bc8000dd8adeb49469f9af5699", + "environment/agent-env-blanks.json": "e527d19e8a42180386e30ec7219cf7dfa1127fda0340eb00be871f82fd45399f", "environment/agent-seed/.git/COMMIT_EDITMSG": "4b654bd1437066b13498661f3ca14774daf1066d072036beffaf06f0c014250e", "environment/agent-seed/.git/HEAD": "28d25bf82af4c0e2b72f50959b2beb859e3e60b9630a5e8c603dad4ddb2b6e80", "environment/agent-seed/.git/config": "cae33efdb02cf774435c1ff9cb16bcc1014606908530c6e1dc727615fe3e8cda", @@ -71,7 +71,7 @@ "environment/agent-seed/src/officeqa_agent/agent.py": "290e0c83797589b640ba93635edf2bd5c53f3880008c3c34f76496153b448367", "environment/agent-seed/tests/test_agent.py": "1a2a6dd660deab8ce5c619756be0a6cad493d488c4217f28981e601d868ce986", "environment/agent-seed/uv.lock": "256b8b915d8a1b7a8cc19619fa96d2f32869d615e9fe1055686228c02f138658", - "environment/docker-compose.yaml": "f67edf521e9b2c5a3983f27a9e941d824039fbd17caac20cc05fcbc317e8c4a6", + "environment/docker-compose.yaml": "1e9d9c8496de0d73ff5bcbaa05a1fcaa4eb753ccff9933388a30cc25c3b1bba1", "environment/gateway/Dockerfile": "70d149f1928b135b8e11b7bb8d4ef70d81f331a6766bb3947192ff876e2eea26", "environment/gateway/config.json": "f218803a48ce1f927020df0c7443e5229689985cfc88f5f715b2658bf89e0507", "environment/gateway/launch.json": "462bcc8d2749f1364d79e14d36be7144a84441b6bb74c985f798115ef26636ae", @@ -81,13 +81,13 @@ "environment/sidecar/cases/development.jsonl": "8c4df163f6fa4b495392552e3fa9da8a1f743486783888d5a9150a391dd06e75", "environment/sidecar/cases/test.jsonl": "d96a98c9ae504045f9cb2a8d7fb8fa1815528237bb68855d5e0fde8c6b031b81", "environment/sidecar/cases/validation.jsonl": "14d86ab3071fb4091efd7b80687b1b8d8176c6b76dc1676d9a72a27f9d99032f", - "environment/sidecar/serve.json": "f70daf21febb1f19b3cc63c109d92d85a4ebb862a8375a8ecd992ff28f1386ae", + "environment/sidecar/serve.json": "081ca2451acfb2df010c4b749baa01d4bce4727f5ad7ae511b08a070059059fa", "environment/sidecar/task-source/": "be9f7f935eac438753a93bc1dccb66dc82a47c2911a9befad507d8f1e4e0480b", - "environment/vero/": "1855222d415aba68140d0576f06c5388d491c17520994c12680402828c95706f", + "environment/vero/": "e12fbae407e3bf6acfaa6445e5debc683592b66180ab69b6e6eff2b395f93f30", "instruction.md": "c9f995857987ecbf98ca9d1bb35dd3ecee240fc50c906cda2a464f75d879471b", "solution/solve.sh": "6472c890bec7a80fbbf5a07e232ee0aa33a2636ca34e919bb4083e470bf11630", - "task.toml": "82490115a3279e0a53465b7a70b00f9e154cd3999179799e083befd538d010f8", + "task.toml": "b96923cb61c2d71edd27ce027a00cbe1d535290285c8e5416c1b218ed8eb1ac6", "tests/test.sh": "b65e90e6d67aacedd559c6b1c9d0f1d5c5071ecc9178a5bd7f9e1066f332bb0d" }, - "vero_commit": "64e76571afcb2827ab7bfeddda2f16a9ad6544c9" + "vero_commit": "96e57cf22e5b7f3a4e8cea9ade61bf5f61c66692" } diff --git a/harness-opt-bench/terminal-bench/baseline/compiled.manifest.json b/harness-opt-bench/terminal-bench/baseline/compiled.manifest.json index 443ffb6a..9bc30d3c 100644 --- a/harness-opt-bench/terminal-bench/baseline/compiled.manifest.json +++ b/harness-opt-bench/terminal-bench/baseline/compiled.manifest.json @@ -36,7 +36,7 @@ "environment/agent-baseline/src/terminal_bench_agent/agent.py": "2749fdf6dd72e9e6ac67de22f226df3d09cb5d08d3b5bb2fbb1f8ce6f589d394", "environment/agent-baseline/tests/test_agent.py": "a6cc1afb3be8d4f9d6e484d7e1f5aa4e058806eb8096cd9ebfa08a7addf5d6d2", "environment/agent-baseline/uv.lock": "c7e865882c9ee4ea963fcc0511472286c3c22c8bb35051d5d4d9c8523e8a205a", - "environment/agent-env-blanks.json": "90b8150c7380714e1b169d902c8cc153c58596bc8000dd8adeb49469f9af5699", + "environment/agent-env-blanks.json": "e527d19e8a42180386e30ec7219cf7dfa1127fda0340eb00be871f82fd45399f", "environment/agent-seed/.git/COMMIT_EDITMSG": "4b654bd1437066b13498661f3ca14774daf1066d072036beffaf06f0c014250e", "environment/agent-seed/.git/HEAD": "28d25bf82af4c0e2b72f50959b2beb859e3e60b9630a5e8c603dad4ddb2b6e80", "environment/agent-seed/.git/config": "cae33efdb02cf774435c1ff9cb16bcc1014606908530c6e1dc727615fe3e8cda", @@ -71,7 +71,7 @@ "environment/agent-seed/src/terminal_bench_agent/agent.py": "2749fdf6dd72e9e6ac67de22f226df3d09cb5d08d3b5bb2fbb1f8ce6f589d394", "environment/agent-seed/tests/test_agent.py": "a6cc1afb3be8d4f9d6e484d7e1f5aa4e058806eb8096cd9ebfa08a7addf5d6d2", "environment/agent-seed/uv.lock": "c7e865882c9ee4ea963fcc0511472286c3c22c8bb35051d5d4d9c8523e8a205a", - "environment/docker-compose.yaml": "f67edf521e9b2c5a3983f27a9e941d824039fbd17caac20cc05fcbc317e8c4a6", + "environment/docker-compose.yaml": "1e9d9c8496de0d73ff5bcbaa05a1fcaa4eb753ccff9933388a30cc25c3b1bba1", "environment/gateway/Dockerfile": "70d149f1928b135b8e11b7bb8d4ef70d81f331a6766bb3947192ff876e2eea26", "environment/gateway/config.json": "51047a5c509fca6a38d44d5cd275a779df1e03fb0005b1cd15c90312bb6a157f", "environment/gateway/launch.json": "462bcc8d2749f1364d79e14d36be7144a84441b6bb74c985f798115ef26636ae", @@ -81,12 +81,12 @@ "environment/sidecar/cases/development.jsonl": "57896d3f3d12ceef4650246ae4f7e836879e0d0f40fbf4579156e7567f5bd25f", "environment/sidecar/cases/test.jsonl": "f294e63d251b10418e318a392b7a2825e4fed834e65a46dd5d63e1d32b059357", "environment/sidecar/cases/validation.jsonl": "30c87786e8441a2d50c14e0b09d8f2a4caf3b35709acee2bdbbca97c0e121fcb", - "environment/sidecar/serve.json": "b9576b9435c5f71a871c5f2a7a9ecc59fc8ba66df81c47a70315f916a4ed8ad8", - "environment/vero/": "1855222d415aba68140d0576f06c5388d491c17520994c12680402828c95706f", + "environment/sidecar/serve.json": "e69dfcef0ecc43f0b43e999798e15b8a9139b680c4ceb7d9d2173b28bbb1382c", + "environment/vero/": "e12fbae407e3bf6acfaa6445e5debc683592b66180ab69b6e6eff2b395f93f30", "instruction.md": "044b5f8e0e5be1cb293898923b97cb3801399f2932dbec4b8a71f960c335576f", "solution/solve.sh": "c53ec7868ab20f6fa5bebc19c3c9a742e0419010be3f06e04e985d299c2de88c", - "task.toml": "6832271042299c0131829efea187a85fb7598a2029b0dea3f675d7eebd62dda6", + "task.toml": "437fef0a28b782607ea05b9597725310e6e553c0d55448f173f281e20e032dad", "tests/test.sh": "b65e90e6d67aacedd559c6b1c9d0f1d5c5071ecc9178a5bd7f9e1066f332bb0d" }, - "vero_commit": "64e76571afcb2827ab7bfeddda2f16a9ad6544c9" + "vero_commit": "96e57cf22e5b7f3a4e8cea9ade61bf5f61c66692" } From a653917942a5c0d8a8b494a9ec385a641de339bb Mon Sep 17 00:00:00 2001 From: Varun Ursekar Date: Sat, 12 Sep 2026 15:42:19 -0700 Subject: [PATCH 4/8] GAIA shell e2e smoke variant: eight cases and a 25-minute agent clock build.shell.e2e.yaml is build.shell.yaml with partitions-e2e (2/3/3 cases), six-run budgets, rescore_top_k 1, and optimizer_agent_timeout_seconds 1500, so a trial reaches finalization inside an hour. It exercises compile, tokens, evaluations, the agent clock handing over to the verifier, and the session archive after a vero change. Its score is never reported. Co-Authored-By: Claude Fable 5.1 --- .../gaia/baseline/build.shell.e2e.yaml | 215 ++++++++++++++++++ .../gaia/partitions-e2e/README.md | 7 + .../gaia/partitions-e2e/development.json | 4 + .../gaia/partitions-e2e/test.json | 5 + .../gaia/partitions-e2e/validation.json | 5 + 5 files changed, 236 insertions(+) create mode 100644 harness-opt-bench/gaia/baseline/build.shell.e2e.yaml create mode 100644 harness-opt-bench/gaia/partitions-e2e/README.md create mode 100644 harness-opt-bench/gaia/partitions-e2e/development.json create mode 100644 harness-opt-bench/gaia/partitions-e2e/test.json create mode 100644 harness-opt-bench/gaia/partitions-e2e/validation.json diff --git a/harness-opt-bench/gaia/baseline/build.shell.e2e.yaml b/harness-opt-bench/gaia/baseline/build.shell.e2e.yaml new file mode 100644 index 00000000..50f0dfcc --- /dev/null +++ b/harness-opt-bench/gaia/baseline/build.shell.e2e.yaml @@ -0,0 +1,215 @@ +name: vero/optimize-gaia-shell-e2e +description: >- + The program must answer canonical GAIA tasks: general-knowledge questions that + may need research, computation, or inspection of attached files. How it does + that is yours to decide -- the approach, what the agent can do, how it uses its + environment, how many model calls it takes, and when it stops. Preserve the + Harbor agent interface and the class import path. The grader reads the exact + answer from /app/answer.txt. +instruction_template: instruction.shell.md.j2 +agent_repo: target-shell +task_source: gaia/gaia@sha256:bbc356f476e0b70ba77da11a9be7d6345918d1e4a2daade0d6dfb82ee6f7b761 +task_manifest: ../partitions/manifest.json +agent_import_path: gaia_agent.agent:GaiaAgent +harbor_requirement: harbor[modal]==0.20.0 +# Set once the vero on PyPI carries the compiled-task changes this branch relies on +# (env-delivered scope tokens, outer-trial limits); until then the images copy the +# local source. With it set, the compiled task holds no vero and can be committed. +# vero_requirement: scaleapi-vero==0.6.0 + +# E2E SMOKE VARIANT of build.shell.yaml. Same seed, same target, same gateway +# scopes; eight cases in total, tiny budgets, and a 25-minute agent clock so the +# run reaches finalization quickly. It exists to prove the trial machinery end to +# end (compile, tokens, evaluations, agent timeout -> verifier, session archive) +# after a change to vero, not to measure anything. Never report its score. +partition_files: + development: ../partitions-e2e/development.json + validation: ../partitions-e2e/validation.json + test: ../partitions-e2e/test.json + +# total_cases = 4 full passes per partition (dev 33*4, validation 66*4). +agent_access: + - partition: development + disclosure: full + expose_case_resources: true + total_runs: 6 + total_cases: 12 + - partition: validation + disclosure: aggregate + expose_case_resources: false + min_aggregate_cases: 1 + total_runs: 6 + total_cases: 18 + +selection_partition: validation +targets: + - partition: test + reward_key: reward + # Zero by construction, not by measurement: the skeleton writes an empty + # answer, so every case is scored and wrong. Confirm with one baseline round + # before quoting deltas -- the number is a claim about the seed, not a + # measurement of it. + baseline_reward: 0.0 + failure_value: 0.0 + max_attempts: 1 + # The held-out eval is noisy: score the selected candidate 3x per case and + # average, so the final reward is comparable to the pinned baseline, which was + # itself pooled over 3 rounds. Without this the candidate carries ~sqrt(3) more + # standard error than the floor it is judged against. + # Per-target override - search/validation keep the global n_attempts (1). + n_attempts: 3 + aggregate_attempts: mean + +evaluation_set_name: gaia +objective: + selector: + metric: score + direction: maximize +reward_mode: submit # agent picks; falls back to auto_best, then current version +baseline_floor: false # gates on validation while reward is on test; opt-in only +score_baseline: false +rescore_top_k: 1 +rescore_attempts: 1 + +# GAIA is multimodal: 5 of the 66 held-out tasks send image inputs, and a +# text-only target 400s on them ("This model does not support image inputs"), +# which reads as ordinary agent failure and caps achievable reward near 0.92. +# Unprefixed on purpose -- the agent sends model_name.removeprefix("openai/"), +# so an openai/-prefixed name here would be allow-listed in one form and +# requested in another, and the gateway would deny it. +model: gpt-5.4-mini +environment_name: ${inner_env:-modal} +# inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default +extra_harbor_args: ["--ek", "app_name=harness-opt-bench", "--ek", "sandbox_idle_timeout_secs=3600"] +harbor_python_version: "3.12" +n_attempts: 1 +# Raised from 1 on 2026-07-29. This field exists precisely so "a transient +# upstream rate-limit storm during a mandatory evaluation does not fail +# otherwise-good candidates" (see HarborBuildConfig), and it was set to the +# least aggressive value. Measured that night: the shared Fireworks per-minute +# bucket rejected 78,748 requests across one officeqa pass, sustaining +# ~900-1,000/min for 24 minutes, and SEVEN rejections spread over ten minutes +# were enough to destroy a 37-case evaluation. The retry stack during search is +# only the target agent's own SDK (max_retries=8, ~1 min of jittered backoff, +# and candidate-controlled so it cannot be relied on) plus this one; the +# infrastructure_max_attempts retry below applies to trusted finalization only. +# 4 retries at multiplier 2 from a 4s floor, capped at 120s, gives roughly four +# more minutes of ride-out per case. That covers the bursts, though not a +# 24-minute saturation -- for which the answer is fewer concurrent runs or a +# larger provider quota, not more retrying. +max_retries: 4 +retry_max_wait_seconds: 120 +infrastructure_max_attempts: 3 +infrastructure_retry_delay_seconds: 5 +aggregate_attempts: best +feedback_transcripts: true +feedback_max_bytes: 16000 +expose_attempt_detail: false +# Unreachable: worst case is ceil(198/24) x 600 = 5400s, every +# finalize trial (66 held-out x n_attempts=3) hitting its own cap. Assumes +# max_concurrency=24; recompute if that drops. +timeout_seconds: 1800 +# Exactly the dataset's declared [agent] timeout_sec, so vero's derived +# --agent-timeout-multiplier is 1.0 and the target agent gets precisely the +# clock the benchmark intends. Harbor times agent setup, environment build and +# verification on separate clocks with separate multipliers, so none of them +# eat into this budget and no buffer is warranted. +case_timeout_seconds: 600 +task_agent_timeout_seconds: 600 +max_concurrency: 24 # 8 -> 24; see officeqa for the measured headroom argument +error_rate_threshold: 0.1 +# Unreachable: worst-case finalize (5400) + worst-case rescore_top_k=3 +# validation rescore (5400). A verifier timeout loses the score outright. +verifier_timeout_seconds: 3600 +# Limits on the OUTER optimizer trial: the Modal sandbox the optimizer itself runs +# in, not the nested evaluation. Every value reproduces what ran implicitly before +# 2026-09, now written down so the task states its own bounds. +# sandbox timeout: Harbor's Modal default of 24 h. When it fires the sandbox is +# destroyed with no verifier and no archive, so the trial is lost. +optimizer_sandbox_timeout_seconds: 14400 +# idle timeout: reclaims a sandbox whose optimizer died with no running command. +# The inner evaluation app has used 3600 since July; the outer sandbox had none. +optimizer_sandbox_idle_timeout_seconds: 1800 +# agent clock: Harbor stops the optimizer here and STILL runs the verifier, so a +# runaway is scored and archived instead of lost at the sandbox timeout. Clean +# cells finish in 8-17 h; the longest finalization seen was ~3.5 h, which is +# what the 4 h gap to the sandbox timeout is for. +# 25 minutes: long enough for the optimizer to run a few evaluations, short +# enough that Harbor stops it and hands over to the verifier within the hour, +# which is the path this variant exists to exercise. +optimizer_agent_timeout_seconds: 1500 +# Harbor's default resolves to public internet; stated rather than inherited. +optimizer_allow_internet: true +# optimizer_cpus / optimizer_memory_mb / optimizer_storage_mb are left unset on +# purpose: that is Modal's default (0.125-core / 128 MiB reservation, bursting to +# the host), which is what every August cell ran under. Set them once the first +# rebuttal wave has been read off the Modal dashboard. +secrets: + - MODAL_TOKEN_ID + - MODAL_TOKEN_SECRET + # Which Modal environment the sandboxes land in. Harbor passes none to the SDK, + # so without this every sandbox goes to the workspace default (`main`), which is + # where all pre-2026-09 cells ran. Declared as a secret so it reaches the sidecar + # (inner evaluations) as well as the launcher (outer trial). Set to `enterprise`. + - MODAL_ENVIRONMENT + - WANDB_API_KEY + - WANDB_BASE_URL # self-hosted or cloud W&B + +# candidate harness runs as an unprivileged uid, unable to read held-out state. +harness_user: harness +# Optimizer-agent env (forwarded to the harbor claude-code agent as --ae KEY=VALUE). +# Claude Code's Bash tool caps a single call at BASH_MAX_TIMEOUT_MS (default +# 600000=10min), well under one inner eval, which pushed the officeqa optimizer +# into --detach + background-poll + end-turn -- and a headless --print run is +# never re-woken, so the search died there. Raise the cap so a whole eval fits in +# one blocking call. The background-task vars are defence in depth only: they gate +# *automatic* backgrounding and do NOT remove the Bash tool's run_in_background +# parameter, which the model can still choose. The instruction forbids that. +agent_env: + # Above this benchmark's widest single eval: a full validation pass is + # ceil(66/24) x 600 = 1800s worst case. + BASH_MAX_TIMEOUT_MS: "3600000" + BASH_DEFAULT_TIMEOUT_MS: "3600000" # same as max: an un-timed eval must still block + ENABLE_BACKGROUND_TASKS: "0" + FORCE_AUTO_BACKGROUND_TASKS: "0" + # Harnesses installed with `uv tool install` (mini-swe-agent, swe-agent) + # default to symlinking their entry point into /usr/local/bin, which the + # unprivileged optimizer user cannot write: "Failed to install executable + # ... Permission denied". npm/nvm-based harnesses (claude-code, opencode) + # are unaffected, so this only bites when the harness changes. + UV_TOOL_BIN_DIR: "/home/agent/.local/bin" + + +wandb: + project: harness-engineering-bench # one project for the whole suite + group: gaia # keeps the benchmark distinguishable in the shared project + name: ${wandb_run:-gaia-shell-e2e} # per-launch label, e.g. --param wandb_run=gaia__opencode + tags: [gaia, shell-seed, e2e-smoke] + log_traces: true + +inference_gateway: + upstream_api_key_env: OPENAI_API_KEY + upstream_base_url_env: OPENAI_BASE_URL + # Stamp request-log records with a thread_id so per_trial_tokens.py can attribute + # gateway token usage to individual trials (trusted, vs. content-matching). + # Without it the fallback recovers only each conversation's root turn: measured + # 0-13% coverage unstamped against 90-98% stamped, even with --tasks-dir. + request_log_attribution: true + producer: + allowed_models: ["${optimizer_model:-openai/gpt-5.4}"] + max_concurrency: 8 + # See officeqa/baseline/build.yaml for the sizing rationale: the case budget is + # the spend control, so a token cap only needs to stop a runaway. + evaluation: + allowed_models: [gpt-5.4-mini] + max_requests: 200000 + max_tokens: 200000000 # 30 agent case-runs at most + max_concurrency: 64 + # Reserved so a search-phase overspend can never starve held-out scoring. + finalization: + allowed_models: [gpt-5.4-mini] + max_requests: 200000 + max_tokens: 2000000000 # 66 test cases x3 attempts + rescore headroom + max_concurrency: 64 +instruct_multifidelity: true +instruct_exhaust_budget: true diff --git a/harness-opt-bench/gaia/partitions-e2e/README.md b/harness-opt-bench/gaia/partitions-e2e/README.md new file mode 100644 index 00000000..85873c84 --- /dev/null +++ b/harness-opt-bench/gaia/partitions-e2e/README.md @@ -0,0 +1,7 @@ +# e2e smoke partitions + +Two development, three validation and three test cases, the first entries of the +real GAIA shell partitions. Used only by `baseline/build.shell.e2e.yaml`, whose +purpose is to exercise the whole optimizer trial (compile, gateway, evaluations, +agent clock, finalization, session archive) in well under an hour. Scores from it +mean nothing and are never reported. diff --git a/harness-opt-bench/gaia/partitions-e2e/development.json b/harness-opt-bench/gaia/partitions-e2e/development.json new file mode 100644 index 00000000..a33df613 --- /dev/null +++ b/harness-opt-bench/gaia/partitions-e2e/development.json @@ -0,0 +1,4 @@ +[ + "gaia/00d579ea-0889-4fd9-a771-2c8d79835c8d", + "gaia/0a65cb96-cb6e-4a6a-8aae-c1084f613456" +] \ No newline at end of file diff --git a/harness-opt-bench/gaia/partitions-e2e/test.json b/harness-opt-bench/gaia/partitions-e2e/test.json new file mode 100644 index 00000000..a47f2299 --- /dev/null +++ b/harness-opt-bench/gaia/partitions-e2e/test.json @@ -0,0 +1,5 @@ +[ + "gaia/023e9d44-96ae-4eed-b912-244ee8c3b994", + "gaia/0512426f-4d28-49f0-be77-06d05daec096", + "gaia/076c8171-9b3b-49b9-a477-244d2a532826" +] \ No newline at end of file diff --git a/harness-opt-bench/gaia/partitions-e2e/validation.json b/harness-opt-bench/gaia/partitions-e2e/validation.json new file mode 100644 index 00000000..6f122465 --- /dev/null +++ b/harness-opt-bench/gaia/partitions-e2e/validation.json @@ -0,0 +1,5 @@ +[ + "gaia/0383a3ee-47a7-41a4-b493-519bdefe0488", + "gaia/04a04a9b-226c-43fd-b319-d5e89743676f", + "gaia/05407167-39ec-4d3a-a234-73a9120c325d" +] \ No newline at end of file From ac9034baf48888e01035cd0cfd13be7ebbe0e1c3 Mon Sep 17 00:00:00 2001 From: Varun Ursekar Date: Sat, 12 Sep 2026 18:20:06 -0700 Subject: [PATCH 5/8] Document compiled tasks in the rewritten CONFIGURATION.md; trim the e2e yaml comment A plain-language section covering deterministic compiles, the committed manifests and --check, per-run inputs arriving through the environment, and the vero_requirement pin. Co-Authored-By: Claude Fable 5.1 --- harness-opt-bench/CONFIGURATION.md | 19 +++++++++++++++++++ .../gaia/baseline/build.shell.e2e.yaml | 6 +----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/harness-opt-bench/CONFIGURATION.md b/harness-opt-bench/CONFIGURATION.md index 32f09f17..09bf51f5 100644 --- a/harness-opt-bench/CONFIGURATION.md +++ b/harness-opt-bench/CONFIGURATION.md @@ -194,6 +194,25 @@ Some tasks depend on external services of their own and therefore use a reduced isolation profile. That limitation must be declared in the benchmark configuration and considered before running with an adversarial optimizer. +## Compiled tasks + +`vero harbor build` turns a `build.yaml` into a self-contained task directory, +and two compiles of the same sources produce identical output. Nothing specific +to one run is written into the compiled task: the per-run access tokens and the +optimizer model are supplied by the launcher through the environment and read +once when the services start, so a running evaluation cannot be altered from +outside. + +Each reported benchmark commits `baseline/compiled.manifest.json`, a checksum of +every compiled file. `vero harbor build --check ` recompiles and fails +if a checkout no longer reproduces the recorded task. Regenerate the manifest +whenever the seed, the partitions, or the build configuration changes. + +By default the compiled task carries a copy of the VeRO source tree so its +images can be built without a package index. Setting `vero_requirement` to an +exact published version (for example `scaleapi-vero==0.6.0`) installs that +release instead; the version must match the VeRO doing the compiling. + ## Adding or changing a benchmark 1. Update the benchmark's `baseline/build.yaml`. diff --git a/harness-opt-bench/gaia/baseline/build.shell.e2e.yaml b/harness-opt-bench/gaia/baseline/build.shell.e2e.yaml index 50f0dfcc..fe7519b2 100644 --- a/harness-opt-bench/gaia/baseline/build.shell.e2e.yaml +++ b/harness-opt-bench/gaia/baseline/build.shell.e2e.yaml @@ -147,11 +147,7 @@ optimizer_allow_internet: true secrets: - MODAL_TOKEN_ID - MODAL_TOKEN_SECRET - # Which Modal environment the sandboxes land in. Harbor passes none to the SDK, - # so without this every sandbox goes to the workspace default (`main`), which is - # where all pre-2026-09 cells ran. Declared as a secret so it reaches the sidecar - # (inner evaluations) as well as the launcher (outer trial). Set to `enterprise`. - - MODAL_ENVIRONMENT + - MODAL_ENVIRONMENT # which Modal environment the sandboxes land in; unset = workspace default - WANDB_API_KEY - WANDB_BASE_URL # self-hosted or cloud W&B From 9213e037b240ddea9506bb7753118f4270c1b5a1 Mon Sep 17 00:00:00 2001 From: Varun Ursekar Date: Sat, 12 Sep 2026 19:20:40 -0700 Subject: [PATCH 6/8] Clean up and document the benchmark tree Build files, READMEs and the runbook are rewritten in plain language. The evaluation environment is a required parameter (inner_env), target models are named without a provider prefix and mapped to a deployment through the evaluation scope's model_aliases (target_model_route), and the Terminal-Bench variant becomes build.routed.yaml with the route as a parameter. Kept as they were: the harbor extra the evaluation environment needs (now overridable), the secrets list and optional telemetry block, and the sandbox options that group and reclaim evaluation sandboxes. rescore_candidate.py learns the required-parameter placeholder and applies the same target routing. Manifests regenerated with inner_env=modal. Co-Authored-By: Claude Fable 5.1 --- harness-opt-bench/CONFIGURATION.md | 8 +- harness-opt-bench/README.md | 18 +- harness-opt-bench/archive/README.md | 28 +- .../archive/swe-atlas-qna/README.md | 44 ++- .../archive/swe-atlas-qna/baseline/README.md | 57 ++- .../baseline/build.gpt54mini.yaml | 126 +------ .../archive/swe-atlas-qna/baseline/build.yaml | 126 +------ .../baseline/target/src/atlas_agent/agent.py | 11 +- .../archive/swe-bench-pro/README.md | 205 +++-------- .../swe-bench-pro/baseline/build.sample.yaml | 46 +-- .../archive/swe-bench-pro/baseline/build.yaml | 35 +- .../baseline/secrets.env.example | 23 +- .../baseline/target/pyproject.toml | 2 +- .../target/src/swebench_pro_agent/agent.py | 7 +- harness-opt-bench/archive/tau3/README.md | 40 ++- .../archive/tau3/baseline/README.md | 37 +- .../archive/tau3/baseline/build.yaml | 73 +--- .../baseline/target/src/tau3_agent/agent.py | 24 +- harness-opt-bench/browsecomp-plus/README.md | 94 ++--- .../browsecomp-plus/baseline/README.md | 32 +- .../browsecomp-plus/baseline/build.yaml | 123 +++---- .../baseline/compiled.manifest.json | 8 +- .../baseline/secrets.env.example | 20 +- .../target/src/browsecomp_plus_agent/agent.py | 11 +- harness-opt-bench/gaia/README.md | 88 +++-- harness-opt-bench/gaia/baseline/README.md | 144 ++------ .../gaia/baseline/build.shell.e2e.yaml | 126 ++----- .../gaia/baseline/build.shell.yaml | 128 ++----- harness-opt-bench/gaia/baseline/build.yaml | 128 ++----- .../gaia/baseline/compiled.manifest.json | 8 +- .../gaia/baseline/secrets.env.example | 21 +- .../gaia/partitions-e2e/README.md | 13 +- harness-opt-bench/officeqa/README.md | 69 ++++ .../officeqa/baseline/build.yaml | 146 +++----- .../officeqa/baseline/compiled.manifest.json | 8 +- .../target/src/officeqa_agent/agent.py | 15 +- harness-opt-bench/scripts/per_trial_tokens.py | 4 +- .../scripts/rescore_candidate.py | 47 ++- harness-opt-bench/secrets.env.example | 31 +- .../skills/run-benchmark/SKILL.md | 328 +++++++----------- harness-opt-bench/terminal-bench/README.md | 269 ++++---------- .../terminal-bench/baseline/build.azure.yaml | 212 ----------- .../terminal-bench/baseline/build.routed.yaml | 152 ++++++++ .../terminal-bench/baseline/build.yaml | 132 +++---- .../baseline/compiled.manifest.json | 8 +- .../baseline/secrets.env.example | 21 +- vero/tests/test_v05_benchmark_configs.py | 65 ++-- 47 files changed, 1180 insertions(+), 2181 deletions(-) create mode 100644 harness-opt-bench/officeqa/README.md delete mode 100644 harness-opt-bench/terminal-bench/baseline/build.azure.yaml create mode 100644 harness-opt-bench/terminal-bench/baseline/build.routed.yaml diff --git a/harness-opt-bench/CONFIGURATION.md b/harness-opt-bench/CONFIGURATION.md index 09bf51f5..2035aae7 100644 --- a/harness-opt-bench/CONFIGURATION.md +++ b/harness-opt-bench/CONFIGURATION.md @@ -97,6 +97,10 @@ not affect candidate selection. | [BrowseComp-Plus](browsecomp-plus/baseline/build.yaml) | `deepseek-v4-flash` | 33 / 66 / 66 | 0.462 ± 0.028 | | [Terminal-Bench](terminal-bench/baseline/build.yaml) | `grok-build-0.1` | 17 / 36 / 36 | 0.241 ± 0.013 | +Target models are named without a provider. Each build maps the name to a +deployment under the evaluation scope's `model_aliases`; pass +`--param target_model_route=` to use a different one. + Each baseline is the mean of three independent test rounds; the reported uncertainty is the standard deviation of those round means. GAIA uses a multimodal target because some held-out cases contain images. The reported GAIA @@ -205,7 +209,8 @@ outside. Each reported benchmark commits `baseline/compiled.manifest.json`, a checksum of every compiled file. `vero harbor build --check ` recompiles and fails -if a checkout no longer reproduces the recorded task. Regenerate the manifest +if a checkout no longer reproduces the recorded task. The manifests are written +with `inner_env=modal`; pass the same parameter when checking. Regenerate the manifest whenever the seed, the partitions, or the build configuration changes. By default the compiled task carries a copy of the VeRO source tree so its @@ -234,6 +239,7 @@ uv run pytest tests/test_v05_benchmark_configs.py uv run vero harbor build \ --config ../harness-opt-bench//baseline/build.yaml \ + --param inner_env= \ --output ``` diff --git a/harness-opt-bench/README.md b/harness-opt-bench/README.md index 7b4ad60e..84fa0277 100644 --- a/harness-opt-bench/README.md +++ b/harness-opt-bench/README.md @@ -32,10 +32,10 @@ Each benchmark pins its target, dataset, split, budgets, and scoring protocol in | Benchmark | Editable target | Dataset | Split | | --- | --- | --- | --- | -| [GAIA](gaia/baseline/) | Tool-using multimodal agent | GAIA | 20% / 40% / 40% | -| [OfficeQA](officeqa/baseline/) | Grounded document-QA agent | Treasury Bulletin corpus | 20% / 40% / 40% | -| [BrowseComp-Plus](browsecomp-plus/baseline/) | Fixed-corpus research agent | BrowseComp-Plus | 20% / 40% / 40% | -| [Terminal-Bench](terminal-bench/baseline/) | Shell-based terminal agent | Terminal-Bench 2.1 | 20% / 40% / 40% | +| [GAIA](gaia/) | Tool-using multimodal agent | GAIA | 20% / 40% / 40% | +| [OfficeQA](officeqa/) | Grounded document-QA agent | Treasury Bulletin corpus | 20% / 40% / 40% | +| [BrowseComp-Plus](browsecomp-plus/) | Fixed-corpus research agent | BrowseComp-Plus | 20% / 40% / 40% | +| [Terminal-Bench](terminal-bench/) | Shell-based terminal agent | Terminal-Bench 2.1 | 20% / 40% / 40% | Additional benchmarks that were implemented but are not part of the reported suite live under [`archive/`](archive/). @@ -66,14 +66,15 @@ uv run vero harbor run \ --env-file .env \ --agent \ --model \ + --param inner_env= \ --param optimizer_model= \ -o ../runs//jobs ``` -The env file holds your model-endpoint key, execution-environment tokens and -telemetry credentials; keep it outside the benchmark definition. The two model -arguments differ because some harnesses rewrite the model name before sending -it, and the gateway only accepts the name it was told to expect. The runbook, +The env file holds credentials required by your inference and execution +services; keep it outside the benchmark definition. Some harnesses rewrite the +model name before sending it, so `optimizer_model` must match that final value. +The runbook, [`skills/run-benchmark/SKILL.md`](skills/run-benchmark/SKILL.md), covers that rule, the preflight and the health checks. Before launching a full experiment, use [`vero/examples/harness-conformance/`](../vero/examples/harness-conformance/) @@ -101,6 +102,7 @@ inspect the generated task: cd vero VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ --config ../harness-opt-bench//baseline/build.yaml \ + --param inner_env= \ --output ``` diff --git a/harness-opt-bench/archive/README.md b/harness-opt-bench/archive/README.md index 454e049b..c8e327a0 100644 --- a/harness-opt-bench/archive/README.md +++ b/harness-opt-bench/archive/README.md @@ -1,15 +1,17 @@ # Archived benchmarks -Three optimization tasks that were wired, run, and left out of the paper. They -still compile with `vero harbor build`, but they are not maintained to the -conventions in `../CONFIGURATION.md` and nothing in CI exercises them. - -| benchmark | why it is not in the paper | -|---|---| -| `swe-atlas-qna/` | The binary held-out reward sits at the floor (seed 0.067, sd 0.025): 20 optimizer cells spanned 0.054 to 0.127 and no adjacent pair separated. The continuous `agg_score` the rubric also emits (seed 0.632) is more informative but is not surfaced as a selectable reward. | -| `tau3/` | Its user simulator and grader are both LLMs, so a single unchanged harness scored 0.800 and 0.547 on the same development set. Deltas under about 0.1 are unresolved, and only 4 of 16 grid cells recorded a usable baseline. | -| `swe-bench-pro/` | Never normalized to the shared conventions: 1x case budgets, single-shot held-out scoring, no pinned baseline, an agent clock at 0.6x the declared case timeout, and a seed that edits nothing. Its `build.yaml` is the full 731-instance split; `build.sample.yaml` is a nested 33/66/66 subsample. | - -Runs that were made against these configs are under `runs/swe-atlas-qna/`, -`runs/tau3/` and `runs/conformance/`, and the write-ups that judged them are -`runs/AUDIT-swe-atlas-qna.md` and the tau3 rows of `runs/RESULTS-INDEX.md`. +These benchmarks are retained for reproducibility and future development but are +not part of the reported HarnessOpt-Bench suite. + +| Benchmark | Why it was archived | +| --- | --- | +| [SWE-Atlas-QnA](swe-atlas-qna/) | Binary rewards did not separate optimizer results reliably | +| [tau3](tau3/) | Simulator and grader variability obscured small harness improvements | +| [SWE-bench-Pro](swe-bench-pro/) | The configuration was not normalized to the shared evaluation protocol | + +The configurations remain useful as working examples, but results should not be +compared directly with the reported suite without first reviewing their pinned +baselines, budgets, attempts, and timeouts. + +Each benchmark directory explains its task, split, editable target, and current +limitations. diff --git a/harness-opt-bench/archive/swe-atlas-qna/README.md b/harness-opt-bench/archive/swe-atlas-qna/README.md index 75eed4f0..349f26c9 100644 --- a/harness-opt-bench/archive/swe-atlas-qna/README.md +++ b/harness-opt-bench/archive/swe-atlas-qna/README.md @@ -1,26 +1,36 @@ # SWE-Atlas-QnA -This benchmark optimizes an agent that investigates a checked-out software -repository and writes an evidence-backed answer to a deep codebase question. -Scoring is the canonical rubric judge shipped in each Harbor task. +SWE-Atlas-QnA evaluates an agent that investigates a checked-out repository and +writes an evidence-backed answer to a codebase question. -The pinned 124-task dataset is split 25/49/50 for development, validation, and -test. The split is deterministic and stratified by source repository so that -the ten represented codebases occur across the three partitions. +## At a glance -Regenerate or verify the committed split from an exported dataset: +| Item | Value | +| --- | --- | +| Cases | 124 | +| Development / validation / test | 25 / 49 / 50 | +| Editable harness | `baseline/target/` | +| Split strategy | Source repository | +| Scoring | Canonical rubric judge | +| Status | Archived; not part of the reported suite | -```bash -python harness-opt-bench/scripts/partition_dataset.py swe-atlas-qna \ - --tasks-dir /path/to/exported/dataset \ - --output-dir harness-opt-bench/candidates/swe-atlas-qna/partitions \ - --fetch-registry +Development exposes complete task results and repositories. Validation is +aggregate-only, and test is held out for final scoring. + +## Data and split +The split is deterministic and keeps all ten source repositories represented +across partitions. Verify it against an exported dataset: + +~~~bash python harness-opt-bench/scripts/partition_dataset.py swe-atlas-qna \ - --tasks-dir /path/to/exported/dataset \ - --output-dir harness-opt-bench/candidates/swe-atlas-qna/partitions \ + --tasks-dir \ + --output-dir harness-opt-bench/archive/swe-atlas-qna/partitions \ --check -``` +~~~ + +Use `--fetch-registry` when intentionally refreshing the pinned package. Review +the task manifest and split before accepting new results. -`--fetch-registry` requires the pinned Harbor package and verifies every task -name and content digest against Harbor Hub. +See [the baseline guide](baseline/README.md) for the editable target and the two +target-model builds. diff --git a/harness-opt-bench/archive/swe-atlas-qna/baseline/README.md b/harness-opt-bench/archive/swe-atlas-qna/baseline/README.md index 0c173afc..83fa2468 100644 --- a/harness-opt-bench/archive/swe-atlas-qna/baseline/README.md +++ b/harness-opt-bench/archive/swe-atlas-qna/baseline/README.md @@ -1,39 +1,36 @@ -# SWE-Atlas-QnA codebase agent +# SWE-Atlas-QnA editable target -This leaf benchmark optimizes a small Harbor-native agent that explores the -repository mounted at `/app` and writes its final answer to -`/logs/agent/answer.txt`. The editable program controls its prompt, search -strategy, shell tools, context management, and answer synthesis. +The seed explores the repository mounted at `/app` and writes its answer to +`/logs/agent/answer.txt`. The optimizer may change its prompt, search strategy, +shell tools, context management, answer synthesis, and dependencies. -Two pinned target builds share this seed, split, budgets, and access policy; -they differ only in target model, and each pins the seed floor measured on -**its own** model — a delta against the other build's floor is a model -comparison, not an optimization result: +Two builds share the same target, split, budgets, and access policy: -| build | target model | pinned `baseline_reward` | -| --- | --- | --- | -| `build.yaml` | `fireworks_ai/gpt-oss-120b` | 0.0667 (K=3, n=150) | -| `build.gpt54mini.yaml` | `gpt-5.4-mini` | 0.1216 (K=3, n=148) | +| Build | Target model | Pinned seed baseline | +| --- | --- | ---: | +| `build.yaml` | `gpt-oss-120b` | 0.0667 | +| `build.gpt54mini.yaml` | `gpt-5.4-mini` | 0.1216 | -Both floors were measured with `scripts/rescore_candidate.py --seed`, the same -path that produced every other pinned baseline in the suite. Trials the seed -itself killed score 0 (they are harness headroom, and finalization taxes a -candidate's dead attempts the same way); trials the platform killed are -excluded. For gpt-5.4-mini that prices in the seed's ~8% empty-completion -fail-fast — the largest single piece of fixable headroom in this seed. +Each baseline belongs to its configured model. Comparing a candidate against the +other build's baseline would mix harness improvement with a model change. +Candidate-caused failures score zero; evaluation-system failures are excluded. -The Harbor tasks retain their canonical rubric-based verifier. That verifier -needs `OPENAI_API_BASE`; the target agent uses `OPENAI_BASE_URL`. They may -point to the same OpenAI-compatible endpoint. +## Compile -Compile from the repository root: +From the repository root: -```bash +~~~bash cd vero VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ - --config ../harness-opt-bench/swe-atlas-qna/baseline/build.yaml \ - --output ../harness-opt-bench/swe-atlas-qna/baseline/compiled -``` - -For a real run, provide `OPENAI_API_KEY`, `OPENAI_BASE_URL`, -`OPENAI_API_BASE`, and the Modal credentials declared in `build.yaml`. + --config ../harness-opt-bench/archive/swe-atlas-qna/baseline/build.yaml \ + --param inner_env= \ + --output +~~~ + +Use `build.gpt54mini.yaml` to compile the alternate target-model build. The +`VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time +validation. + +Dataset and split details are in the +[SWE-Atlas-QnA overview](../README.md). For a real optimization run, follow the +shared [`run-benchmark` guide](../../../skills/run-benchmark/SKILL.md). diff --git a/harness-opt-bench/archive/swe-atlas-qna/baseline/build.gpt54mini.yaml b/harness-opt-bench/archive/swe-atlas-qna/baseline/build.gpt54mini.yaml index aa0525d2..b276c422 100644 --- a/harness-opt-bench/archive/swe-atlas-qna/baseline/build.gpt54mini.yaml +++ b/harness-opt-bench/archive/swe-atlas-qna/baseline/build.gpt54mini.yaml @@ -7,7 +7,7 @@ agent_repo: target task_source: scale-ai/swe-atlas-qna@sha256:0e26bc0313ae2fc6f912b67b928e648c7f20d17d91f765f702a93042ce5be0e4 task_manifest: ../partitions/manifest.json agent_import_path: atlas_agent.agent:AtlasAgent -harbor_requirement: harbor[modal]==0.20.0 +harbor_requirement: harbor==0.20.0 partition_files: development: ../partitions/development.json @@ -31,10 +31,9 @@ selection_partition: validation targets: - partition: test reward_key: reward - # Measured on THIS target model, not inherited: build.yaml's floor was - # measured on fireworks_ai/gpt-oss-120b and says nothing about gpt-5.4-mini - # -- a delta against it would be a model comparison dressed up as an - # optimization result. Pinned via + # Measured on this target model rather than inherited from build.yaml; using + # a different model's floor would turn the result into a model comparison. + # Pinned via # rescore_candidate.py --seed --benchmark swe-atlas-qna --model gpt-5.4-mini # (the same path that produced every other pinned baseline). # @@ -67,49 +66,14 @@ rescore_top_k: 3 rescore_attempts: 1 model: gpt-5.4-mini -environment_name: ${inner_env:-modal} -# inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default. -# keepalive: these images set ENTRYPOINT ["/bin/bash"], which Modal prepends to the -# sandbox command; harbor's default keepalive ["sh","-c","sleep infinity"] then dies -# instantly (bash interprets the sh binary as a script) and every trial fails with -# "Sandbox already shut down". Passing bash's own args keeps the sandbox alive. -extra_harbor_args: ["--ek", "app_name=harness-opt-bench", "--ek", "sandbox_idle_timeout_secs=3600", "--ek", 'keepalive=["-c","sleep infinity"]'] -# Concurrent cells each `npm install -g` their harness at once, which saturates the -# link: harbor's agent-setup budget is 360s and FOUR of five cells died on it on -# 2026-07-30 (AgentSetupTimeoutError at 360.0s, mid `npm install -g -# @anthropic-ai/claude-code` / the claude-code bootstrap curl), across all four -# harnesses and all five models. A single atlas cell run alone survives, which is -# why the smoke never caught it. x4 gives 24 minutes to get installed. -# This is the OUTER trial's budget, hence optimizer_harbor_args and not -# extra_harbor_args (which tunes the nested eval). Setup budget only: agent -# execution, case and verifier timeouts are untouched, so measured scores stay -# comparable. swe-bench-pro's grid config has carried this since 07-29; atlas was -# simply never given it. +environment_name: ${inner_env:?set inner_env to a Harbor environment} +# Allow enough setup time when several optimizer cells start together. optimizer_harbor_args: ["--agent-setup-timeout-multiplier", "4"] harbor_python_version: "3.12" n_attempts: 1 -# Retries are kept at the sibling value but are near-idle on this target: -# gpt-5.4-mini resolves to azure_ai/gpt-5.4-mini, NOT the fireworks pool, and -# gaia has run 51 rounds against it at a 0.5% upstream error rate versus 47% -# for gpt-oss-120b on 2026-07-30. Original gpt-oss rationale, kept for history: -# the old target was fireworks_ai/gpt-oss-120b, on the SAME org-wide -# fireworks quota that swe-bench-pro measured as bursty (429 on 11 of 12 -# sequential calls, then 1 of 4 a minute later). That is a provider-level limit, -# not this key's, so it cannot be tuned away from here and has to be ridden out. -# swe-bench-pro went 1 -> 5 after a single retry proved insufficient there; atlas -# was left at 1. Retries only cover infrastructure/rate-limit errors, so this does -# not change any measured score. +# Retry transient request failures without changing candidate-caused outcomes. max_retries: 5 infrastructure_max_attempts: 3 -# 20 -> 5, matching every sibling (gaia, officeqa, tau3, browsecomp-plus, -# swe-bench-pro all pin 5). 20 was this file's own invention and it is pure dead -# wall clock on a 429-heavy path: the live 2026-07-30 sonnet5 run logged 6713 -# rate-limit rejections out of 13697 gateway calls, so a 4x longer backoff is a -# 4x longer wait on roughly half of all traffic. The attempt budget stays at 5 -# (still 5x the siblings' 1), so success probability is unchanged in kind; only -# the gap between attempts shrinks, and these rejections are bursty rather than -# sustained. Retries only cover infrastructure/rate-limit errors, so no measured -# score moves. infrastructure_retry_delay_seconds: 5 aggregate_attempts: best feedback_transcripts: true @@ -131,39 +95,12 @@ error_rate_threshold: 0.1 # Unreachable: worst-case finalize (75600) + worst-case rescore_top_k=3 # validation rescore (75600). A verifier timeout loses the score outright. verifier_timeout_seconds: 176400 -# Limits on the OUTER optimizer trial: the Modal sandbox the optimizer itself runs -# in, not the nested evaluation. Every value reproduces what ran implicitly before -# 2026-09, now written down so the task states its own bounds. -# sandbox timeout: Harbor's Modal default of 24 h. When it fires the sandbox is -# destroyed with no verifier and no archive, so the trial is lost. +# Limits on the outer optimizer trial. The agent timeout leaves time for final +# scoring before the hard sandbox deadline. optimizer_sandbox_timeout_seconds: 86400 -# idle timeout: reclaims a sandbox whose optimizer died with no running command. -# The inner evaluation app has used 3600 since July; the outer sandbox had none. optimizer_sandbox_idle_timeout_seconds: 3600 -# agent clock: Harbor stops the optimizer here and STILL runs the verifier, so a -# runaway is scored and archived instead of lost at the sandbox timeout. Clean -# cells finish in 8-17 h; the longest finalization seen was ~3.5 h, which is -# what the 4 h gap to the sandbox timeout is for. optimizer_agent_timeout_seconds: 72000 -# Harbor's default resolves to public internet; stated rather than inherited. optimizer_allow_internet: true -# optimizer_cpus / optimizer_memory_mb / optimizer_storage_mb are left unset on -# purpose: that is Modal's default (0.125-core / 128 MiB reservation, bursting to -# the host), which is what every August cell ran under. Set them once the first -# rebuttal wave has been read off the Modal dashboard. -secrets: - - MODAL_TOKEN_ID - - MODAL_TOKEN_SECRET - - MODAL_ENVIRONMENT # which Modal environment the sandboxes land in; unset = workspace default - - WANDB_API_KEY - - WANDB_BASE_URL # self-hosted or cloud W&B - -wandb: - project: harness-engineering-bench # one project for the whole suite - group: swe-atlas-qna # keeps the benchmark distinguishable in the shared project - name: ${wandb_run:-swe-atlas-qna} # per-launch label, e.g. --param wandb_run=swe-atlas-qna__claude-sonnet-5 - tags: [swe-atlas-qna] - log_traces: true inference_gateway: upstream_api_key_env: OPENAI_API_KEY @@ -176,26 +113,9 @@ inference_gateway: # this run is a per-case cost number for the mini-vs-nano decision. request_log_attribution: true producer: - # BOTH spellings, prefixed and bare. The gateway compares the requested model - # against this list as an exact string (gateway/inference.py: `model not in - # scope.allowed_models`), and the agent harnesses strip the provider prefix - # before calling it. With only the prefixed form here, every openai- and - # anthropic-routed optimizer gets a 403 "model is not allowed for this scope" - # on its very first call. swe-bench-pro's grid config already carries both - # after that cost a whole run; this one still had a single entry. - # A third entry for the harness's auxiliary model. opencode issues one - # summarisation/title call per session on a small model of the same provider - # family, and it sends the DATED id: the live 2026-07-30 sonnet5 run logged - # exactly one `claude-haiku-4-5-20251001` producer request and it came back - # 403 model_denied, while all 190 `claude-sonnet-5` calls passed. Non-fatal - # (1 call in 191) and not a speed lever, but it is invisible outside the - # gateway request log. CONFIGURATION.md names the bare `claude-haiku-4-5` / - # `gpt-5.4-nano`; the gateway compares exact strings, so the dated spelling - # that actually goes on the wire is what has to be here. Parameterised - # because the name varies by provider family. + # Include an auxiliary model when the optimizer harness uses one. allowed_models: - - "${optimizer_model:-openai/gpt-5.4}" - - "${optimizer_model_bare:-gpt-5.4}" + - "${optimizer_model:-gpt-5.4}" - "${optimizer_aux_model:-gpt-5.4-nano}" max_concurrency: 8 # See officeqa/baseline/build.yaml for the sizing rationale: the case budget is @@ -213,34 +133,18 @@ inference_gateway: max_concurrency: 64 instruct_multifidelity: true instruct_exhaust_budget: true -# The rubric judge runs inside the Modal task container and cannot reach the -# compose-internal gateway; it gets the real upstream on OPENAI_* while the -# candidate agent keeps the metered gateway on VERO_AGENT_INFERENCE_*. +# The rubric judge needs direct endpoint access while the candidate agent stays +# on its metered, allow-listed gateway scope. task_services_use_upstream: true # ACCEPTED EXPOSURE (non-adversarial optimizer): with isolation off, candidate # harness code shares the sidecar uid, so it could read the upstream key and # the held-out session state. Runs using this config get a post-run leakage # audit of every candidate version. Proper fix: per-role egress isolation. harness_user: null -# Optimizer-agent env (forwarded to the harbor claude-code agent as --ae KEY=VALUE). -# Claude Code's Bash tool caps a single call at BASH_MAX_TIMEOUT_MS (default -# 600000=10min), well under one inner eval, which pushed the officeqa optimizer -# into --detach + background-poll + end-turn -- and a headless --print run is -# never re-woken, so the search died there. Raise the cap so a whole eval fits in -# one blocking call. The background-task vars are defence in depth only: they gate -# *automatic* backgrounding and do NOT remove the Bash tool's run_in_background -# parameter, which the model can still choose. The instruction forbids that. +# Allow one full evaluation to complete in a foreground tool call. agent_env: - # Above this benchmark's widest single eval: a full validation pass is - # ceil(49/24) x 10800 = 32400s worst case. BASH_MAX_TIMEOUT_MS: "39600000" - BASH_DEFAULT_TIMEOUT_MS: "39600000" # same as max: an un-timed eval must still block + BASH_DEFAULT_TIMEOUT_MS: "39600000" ENABLE_BACKGROUND_TASKS: "0" FORCE_AUTO_BACKGROUND_TASKS: "0" - # Harnesses installed with `uv tool install` (mini-swe-agent, swe-agent) - # default to symlinking their entry point into /usr/local/bin, which the - # unprivileged optimizer user cannot write: "Failed to install executable - # ... Permission denied". npm/nvm-based harnesses (claude-code, opencode) - # are unaffected, so this only bites when the harness changes. UV_TOOL_BIN_DIR: "/home/agent/.local/bin" - diff --git a/harness-opt-bench/archive/swe-atlas-qna/baseline/build.yaml b/harness-opt-bench/archive/swe-atlas-qna/baseline/build.yaml index bb82f149..e97bc9e0 100644 --- a/harness-opt-bench/archive/swe-atlas-qna/baseline/build.yaml +++ b/harness-opt-bench/archive/swe-atlas-qna/baseline/build.yaml @@ -7,7 +7,7 @@ agent_repo: target task_source: scale-ai/swe-atlas-qna@sha256:0e26bc0313ae2fc6f912b67b928e648c7f20d17d91f765f702a93042ce5be0e4 task_manifest: ../partitions/manifest.json agent_import_path: atlas_agent.agent:AtlasAgent -harbor_requirement: harbor[modal]==0.20.0 +harbor_requirement: harbor==0.20.0 partition_files: development: ../partitions/development.json @@ -56,46 +56,15 @@ score_baseline: false rescore_top_k: 3 rescore_attempts: 1 -model: fireworks_ai/gpt-oss-120b -environment_name: ${inner_env:-modal} -# inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default. -# keepalive: these images set ENTRYPOINT ["/bin/bash"], which Modal prepends to the -# sandbox command; harbor's default keepalive ["sh","-c","sleep infinity"] then dies -# instantly (bash interprets the sh binary as a script) and every trial fails with -# "Sandbox already shut down". Passing bash's own args keeps the sandbox alive. -extra_harbor_args: ["--ek", "app_name=harness-opt-bench", "--ek", "sandbox_idle_timeout_secs=3600", "--ek", 'keepalive=["-c","sleep infinity"]'] -# Concurrent cells each `npm install -g` their harness at once, which saturates the -# link: harbor's agent-setup budget is 360s and FOUR of five cells died on it on -# 2026-07-30 (AgentSetupTimeoutError at 360.0s, mid `npm install -g -# @anthropic-ai/claude-code` / the claude-code bootstrap curl), across all four -# harnesses and all five models. A single atlas cell run alone survives, which is -# why the smoke never caught it. x4 gives 24 minutes to get installed. -# This is the OUTER trial's budget, hence optimizer_harbor_args and not -# extra_harbor_args (which tunes the nested eval). Setup budget only: agent -# execution, case and verifier timeouts are untouched, so measured scores stay -# comparable. swe-bench-pro's grid config has carried this since 07-29; atlas was -# simply never given it. +model: gpt-oss-120b +environment_name: ${inner_env:?set inner_env to a Harbor environment} +# Allow enough setup time when several optimizer cells start together. optimizer_harbor_args: ["--agent-setup-timeout-multiplier", "4"] harbor_python_version: "3.12" n_attempts: 1 -# This benchmark's target is fireworks_ai/gpt-oss-120b, on the SAME org-wide -# fireworks quota that swe-bench-pro measured as bursty (429 on 11 of 12 -# sequential calls, then 1 of 4 a minute later). That is a provider-level limit, -# not this key's, so it cannot be tuned away from here and has to be ridden out. -# swe-bench-pro went 1 -> 5 after a single retry proved insufficient there; atlas -# was left at 1. Retries only cover infrastructure/rate-limit errors, so this does -# not change any measured score. +# Retry transient request failures without changing candidate-caused outcomes. max_retries: 5 infrastructure_max_attempts: 3 -# 20 -> 5, matching every sibling (gaia, officeqa, tau3, browsecomp-plus, -# swe-bench-pro all pin 5). 20 was this file's own invention and it is pure dead -# wall clock on a 429-heavy path: the live 2026-07-30 sonnet5 run logged 6713 -# rate-limit rejections out of 13697 gateway calls, so a 4x longer backoff is a -# 4x longer wait on roughly half of all traffic. The attempt budget stays at 5 -# (still 5x the siblings' 1), so success probability is unchanged in kind; only -# the gap between attempts shrinks, and these rejections are bursty rather than -# sustained. Retries only cover infrastructure/rate-limit errors, so no measured -# score moves. infrastructure_retry_delay_seconds: 5 aggregate_attempts: best feedback_transcripts: true @@ -117,39 +86,12 @@ error_rate_threshold: 0.1 # Unreachable: worst-case finalize (75600) + worst-case rescore_top_k=3 # validation rescore (75600). A verifier timeout loses the score outright. verifier_timeout_seconds: 176400 -# Limits on the OUTER optimizer trial: the Modal sandbox the optimizer itself runs -# in, not the nested evaluation. Every value reproduces what ran implicitly before -# 2026-09, now written down so the task states its own bounds. -# sandbox timeout: Harbor's Modal default of 24 h. When it fires the sandbox is -# destroyed with no verifier and no archive, so the trial is lost. +# Limits on the outer optimizer trial. The agent timeout leaves time for final +# scoring before the hard sandbox deadline. optimizer_sandbox_timeout_seconds: 86400 -# idle timeout: reclaims a sandbox whose optimizer died with no running command. -# The inner evaluation app has used 3600 since July; the outer sandbox had none. optimizer_sandbox_idle_timeout_seconds: 3600 -# agent clock: Harbor stops the optimizer here and STILL runs the verifier, so a -# runaway is scored and archived instead of lost at the sandbox timeout. Clean -# cells finish in 8-17 h; the longest finalization seen was ~3.5 h, which is -# what the 4 h gap to the sandbox timeout is for. optimizer_agent_timeout_seconds: 72000 -# Harbor's default resolves to public internet; stated rather than inherited. optimizer_allow_internet: true -# optimizer_cpus / optimizer_memory_mb / optimizer_storage_mb are left unset on -# purpose: that is Modal's default (0.125-core / 128 MiB reservation, bursting to -# the host), which is what every August cell ran under. Set them once the first -# rebuttal wave has been read off the Modal dashboard. -secrets: - - MODAL_TOKEN_ID - - MODAL_TOKEN_SECRET - - MODAL_ENVIRONMENT # which Modal environment the sandboxes land in; unset = workspace default - - WANDB_API_KEY - - WANDB_BASE_URL # self-hosted or cloud W&B - -wandb: - project: harness-engineering-bench # one project for the whole suite - group: swe-atlas-qna # keeps the benchmark distinguishable in the shared project - name: ${wandb_run:-swe-atlas-qna} # per-launch label, e.g. --param wandb_run=swe-atlas-qna__claude-sonnet-5 - tags: [swe-atlas-qna] - log_traces: true inference_gateway: upstream_api_key_env: OPENAI_API_KEY @@ -160,76 +102,38 @@ inference_gateway: # 0-13% coverage unstamped against 90-98% stamped, even with --tasks-dir. request_log_attribution: true producer: - # BOTH spellings, prefixed and bare. The gateway compares the requested model - # against this list as an exact string (gateway/inference.py: `model not in - # scope.allowed_models`), and the agent harnesses strip the provider prefix - # before calling it. With only the prefixed form here, every openai- and - # anthropic-routed optimizer gets a 403 "model is not allowed for this scope" - # on its very first call. swe-bench-pro's grid config already carries both - # after that cost a whole run; this one still had a single entry. - # A third entry for the harness's auxiliary model. opencode routes its - # summarisation/title work to a small model of the SAME provider family, and - # this is not a rounding error: measured in the gateway request log of the - # live 2026-07-30 claude-opus-5 opencode run, the producer scope served 339 - # `claude-haiku-4-5-20251001` requests against 248 `claude-opus-5` -- the aux - # model was 58% of producer traffic. Leaving it off the list does not cost one - # denied call, it costs most of them. - # The default must match the DEFAULT OPTIMIZER'S FAMILY, hence gpt-5.4-nano - # next to openai/gpt-5.4. Pairing an OpenAI optimizer default with an - # Anthropic aux default is what the terra run hit: one `gpt-5.4-nano` producer - # request, 403 model_denied, because the slot did not exist at all. - # Anthropic optimizers must pass the DATED id explicitly - # (`claude-haiku-4-5-20251001`). CONFIGURATION.md names the bare - # `claude-haiku-4-5`, but the gateway compares exact strings, so the dated - # spelling that actually goes on the wire is what has to be here. + # Include an auxiliary model when the optimizer harness uses one. allowed_models: - - "${optimizer_model:-openai/gpt-5.4}" - - "${optimizer_model_bare:-gpt-5.4}" + - "${optimizer_model:-gpt-5.4}" - "${optimizer_aux_model:-gpt-5.4-nano}" max_concurrency: 8 # See officeqa/baseline/build.yaml for the sizing rationale: the case budget is # the spend control, so a token cap only needs to stop a runaway. evaluation: - allowed_models: [fireworks_ai/gpt-oss-120b] + allowed_models: [gpt-oss-120b] max_requests: 200000 max_tokens: 2000000000 # 296 agent case-runs (100 dev + 196 validation) max_concurrency: 64 # Reserved so a search-phase overspend can never starve held-out scoring. finalization: - allowed_models: [fireworks_ai/gpt-oss-120b] + allowed_models: [gpt-oss-120b] max_requests: 200000 max_tokens: 2000000000 # 50 test cases x3 attempts + rescore headroom max_concurrency: 64 instruct_multifidelity: true instruct_exhaust_budget: true -# The rubric judge runs inside the Modal task container and cannot reach the -# compose-internal gateway; it gets the real upstream on OPENAI_* while the -# candidate agent keeps the metered gateway on VERO_AGENT_INFERENCE_*. +# The rubric judge needs direct endpoint access while the candidate agent stays +# on its metered, allow-listed gateway scope. task_services_use_upstream: true # ACCEPTED EXPOSURE (non-adversarial optimizer): with isolation off, candidate # harness code shares the sidecar uid, so it could read the upstream key and # the held-out session state. Runs using this config get a post-run leakage # audit of every candidate version. Proper fix: per-role egress isolation. harness_user: null -# Optimizer-agent env (forwarded to the harbor claude-code agent as --ae KEY=VALUE). -# Claude Code's Bash tool caps a single call at BASH_MAX_TIMEOUT_MS (default -# 600000=10min), well under one inner eval, which pushed the officeqa optimizer -# into --detach + background-poll + end-turn -- and a headless --print run is -# never re-woken, so the search died there. Raise the cap so a whole eval fits in -# one blocking call. The background-task vars are defence in depth only: they gate -# *automatic* backgrounding and do NOT remove the Bash tool's run_in_background -# parameter, which the model can still choose. The instruction forbids that. +# Allow one full evaluation to complete in a foreground tool call. agent_env: - # Above this benchmark's widest single eval: a full validation pass is - # ceil(49/24) x 10800 = 32400s worst case. BASH_MAX_TIMEOUT_MS: "39600000" - BASH_DEFAULT_TIMEOUT_MS: "39600000" # same as max: an un-timed eval must still block + BASH_DEFAULT_TIMEOUT_MS: "39600000" ENABLE_BACKGROUND_TASKS: "0" FORCE_AUTO_BACKGROUND_TASKS: "0" - # Harnesses installed with `uv tool install` (mini-swe-agent, swe-agent) - # default to symlinking their entry point into /usr/local/bin, which the - # unprivileged optimizer user cannot write: "Failed to install executable - # ... Permission denied". npm/nvm-based harnesses (claude-code, opencode) - # are unaffected, so this only bites when the harness changes. UV_TOOL_BIN_DIR: "/home/agent/.local/bin" - diff --git a/harness-opt-bench/archive/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py b/harness-opt-bench/archive/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py index 471d40db..7d86e50f 100644 --- a/harness-opt-bench/archive/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py +++ b/harness-opt-bench/archive/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py @@ -12,12 +12,7 @@ from openai import AsyncOpenAI def _is_reasoning_model(model: str) -> bool: - """Whether `model` is an OpenAI reasoning model. - - Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects - reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served - open models match none of these prefixes, so they keep the legacy shape. - """ + """Whether `model` uses the reasoning-model request fields.""" name = model.lower() return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name @@ -198,9 +193,7 @@ async def run( environment: BaseEnvironment, context: AgentContext, ) -> None: - # Stateless Chat Completions: the full message history is resent each - # turn (provider prompt-caching handles the repeated prefix), which - # works across every provider, unlike the OpenAI-only Responses API. + # The API is stateless, so resend the full message history each turn. messages: list[dict[str, Any]] = [ {"role": "system", "content": INSTRUCTIONS}, {"role": "user", "content": instruction}, diff --git a/harness-opt-bench/archive/swe-bench-pro/README.md b/harness-opt-bench/archive/swe-bench-pro/README.md index 992b2757..19534844 100644 --- a/harness-opt-bench/archive/swe-bench-pro/README.md +++ b/harness-opt-bench/archive/swe-bench-pro/README.md @@ -1,181 +1,62 @@ # SWE-bench-Pro -This benchmark optimizes a code-editing agent on SWE-bench-Pro tasks. Each task -checks a real project out at `/app` and describes a bug fix or feature. The -editable agent (`baseline/target/src/swebench_pro_agent/agent.py`) edits the -repository in place; the **task-source verifier** applies the resulting repo state -and runs the task's hidden test suite for the reward. The agent does not -self-grade and writes no answer file: reward comes entirely from the suite. +SWE-bench-Pro evaluates a code-editing agent. Each task checks out a project, +describes a change, and grades the edited repository with hidden tests. -## Task source +## At a glance -SWE-bench-Pro ships as the **`swebenchpro` dataset in the default Harbor -registry**, version `1.0`, 731 instances across 11 upstream projects. It is a -*registry* dataset, not an `/@sha256:` package like -swe-atlas-qna or tau3, so the pin is a bare `name@version`: +| Item | Value | +| --- | --- | +| Task source | `swebenchpro@1.0` | +| Cases | 731 across 11 projects | +| Development / validation / test | 146 / 292 / 293 | +| Editable harness | `baseline/target/` | +| Split strategy | Source repository | +| Status | Archived; not part of the reported suite | -``` -task_source: swebenchpro@1.0 -``` +The dataset version and `partitions/manifest.json` pin the task definitions. +Development resources are not exposed through VeRO for this registry-backed +dataset. Each task grades itself without a judge model. -Its tasks resolve to git-backed task ids under -`laude-institute/harbor-datasets` at commit -`c8e8f3fac7097accaacf261d74c3d6f441de45b1`, path -`datasets/swebenchpro/instance___-`. Each task ships an -`environment/Dockerfile` that `FROM`s a public prebuilt DockerHub image -(`jefzda/sweap-images:`, 0.5-4.8 GB), resets the repo to the base -commit, and clears `ENTRYPOINT` (so harbor's default keepalive works and no -`--ek keepalive` override is needed, unlike swe-atlas-qna). +## Configurations -Grading is entirely offline: `tests/test.sh` checks out the gold test files, -runs the official `run_script.sh`, parses results with the official `parser.py`, -and writes `1` to `/logs/verifier/reward.txt` only when every test in -`fail_to_pass` and `pass_to_pass` passed. **No judge model, so no verifier -credentials are needed** (`--ve` is unnecessary for this benchmark). +| Build | Split | Intended use | +| --- | --- | --- | +| `baseline/build.yaml` | 146 / 292 / 293 | Full benchmark | +| `baseline/build.sample.yaml` | 33 / 66 / 66 | Shorter optimizer runs | -Two consequences of being a registry rather than a package dataset: +The sample is nested inside the full split, so no test case moves into +development. It remains an experimental configuration with a provisional +baseline. -- `agent_access[development].expose_case_resources` must be `false`. VeRO - materializes case resources through `PackageDatasetClient`, which only parses - `/` refs and rejects a bare `swebenchpro@1.0`. -- The recorded per-task `ref` in `partitions/manifest.json` is - `:` rather than a `sha256:` content hash. - -## Split design - -The committed split is deterministic and stratified by source repository (so each -represented codebase appears across all three partitions): - -- development: 146 (20%, full result disclosure to the optimizer) -- validation: 292 (40%, aggregate-only; used to select candidates) -- test: 293 (40%, held out until Harbor grades the completed outer task) - -731 does not divide evenly: the exact ratios are 146.2/292.4/292.4, so the one -leftover case is assigned by largest remainder, and the .4/.4 tie between -validation and test breaks towards test. That matches how the sibling benchmarks -resolved the same tie (officeqa 246 -> 49/98/99, swe-atlas-qna 124 -> 25/49/50). - -Regenerate and verify the split with: - -```bash -harbor download dataset swebenchpro@1.0 --output-dir /tmp/swebenchpro # or a -# sparse checkout of laude-institute/harbor-datasets at the pinned commit +## Verify the split +~~~bash uv run --python 3.12 \ - harness-opt-bench/swe-bench-pro/scripts/partition_swe_bench_pro.py \ - --tasks-dir /tmp/swebenchpro --fetch-registry + python harness-opt-bench/archive/swe-bench-pro/scripts/partition_swe_bench_pro.py \ + --tasks-dir \ + --check +~~~ -uv run --python 3.12 \ - harness-opt-bench/swe-bench-pro/scripts/partition_swe_bench_pro.py \ - --tasks-dir /tmp/swebenchpro --check -``` - -## Target model: which models the seed agent can actually run on - -The seed agent drives the **OpenAI Responses API** and chains turns with -`previous_response_id`. Measured against the shared LiteLLM proxy -(`heb-litellm-proxy`), that shape constrains the usable target models: +## Editable target -| model | turn 1 | chained turn (`previous_response_id`) | -|---|---|---| -| gpt-4o | OK (with the reasoning gate) | OK | -| gpt-5.4-mini-2026-03-17 | OK | OK | -| gpt-5.6-sol | OK | OK | -| deepseek-v4-flash | OK | OK | -| gpt-oss-120b | OK | OK | -| **qwen-3.6-27b** | OK | **fails, deterministically** | -| gpt-4.1 | 400 on `reasoning` | n/a | -| gpt-5.3-codex | 404 on the proxy's Responses route | n/a | +The agent in `baseline/target/` inspects and edits the checked-out project. The +optimizer may change its prompt, tools, control flow, context handling, and +dependencies. The task source, partitions, target model, and hidden tests remain +fixed. -`qwen-3.6-27b` answers the first turn but every chained call returns -`litellm.BadRequestError: Fireworks_aiException ... invalid_request_error` -(3/3 reproductions, plus an in-harbor trial). The same conversation works on -`/chat/completions`, so the fault is LiteLLM's Responses-to-ChatCompletions -bridge replaying a shape Fireworks rejects, not the model. Switching the agent -to a stateless full-history `input` list also works on qwen, but that changes -the seed harness being measured, so it has not been done here. - -`reasoning: {"effort": ...}` is gated on `_is_reasoning_model()` (the same -capability predicate the other five benchmark agents use): sending it to gpt-4o -or gpt-4.1 is a hard 400 on the very first turn. - -## Build the outer Harbor task +## Compile From the repository root: -```bash +~~~bash cd vero VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ - --config ../harness-opt-bench/swe-bench-pro/baseline/build.yaml \ - --output ../harness-opt-bench/swe-bench-pro/baseline/compiled -``` - -Omit `VERO_SKIP_SECRET_CHECK=1` for a real build so VeRO verifies that the OpenAI -and Modal credentials declared in `build.yaml` are present. Set `OPENAI_BASE_URL` -to your OpenAI-compatible endpoint. The `compiled/` directory is generated and -intentionally ignored. - -## Measure the held-out baseline - -A K=3 baseline is three independent single-attempt rounds over the 293-case -`test` partition, scored by harbor's own -`stats.evals.*.metrics[0].mean` (an errored trial counts as a zero). Each round -is a plain `harbor run` against the registry dataset, with -`--agent-timeout-multiplier 0.6` (`case_timeout_seconds` 1800 over the task's -declared `[agent] timeout_sec` 3000): - -```bash -harbor run \ - -a swebench_pro_agent.agent:SweBenchProAgent \ - -m -e modal \ - -d swebenchpro@1.0 \ - $(python3 -c "import json;print(' '.join('-i '+t for t in json.load(open('../../partitions/test.json'))))") \ - --n-attempts 1 --max-retries 3 -n 5 --yes \ - --ek app_name=harness-opt-bench --ek sandbox_idle_timeout_secs=3600 \ - --agent-timeout-multiplier 0.6 \ - --env-file baseline/secrets.env \ - -o ../../../runs/baseline-swe-bench-pro/roundN -``` - -Run it three times into `round1/`, `round2/`, `round3/`; the pinned -`baseline_reward` is the mean of the three round means and the `±` is the -population stdev across them. Concurrency matters: `-n 12` against the shared -Azure gpt-4o deployment produced 45 `RateLimitError` retries in 15 minutes, -while `-n 5` produced none. - -## Run the optimization - -`vero harbor run` compiles the `build.yaml` to a temporary task, wires credentials -(relocating the upstream inference key to the gateway and handing the optimizer a -scoped producer token), and invokes Harbor. Copy the secrets template first: - -```bash -cp harness-opt-bench/swe-bench-pro/baseline/secrets.env.example \ - harness-opt-bench/swe-bench-pro/baseline/secrets.env # then edit it - -cd vero -uv run vero harbor run \ - --config ../harness-opt-bench/swe-bench-pro/baseline/build.yaml \ - --env-file ../harness-opt-bench/swe-bench-pro/baseline/secrets.env \ - --environment modal \ - --agent codex \ - --model gpt-5.3-codex \ - --yes \ - -o ../runs/swe-bench-pro-full/jobs -``` - -`--env-file` values override the ambient shell and are passed only through the -subprocess environment, never on the command line. Anything after the known -options (here `--yes -o ...`) is forwarded verbatim to `harbor run`. - -The producer scope's allow-list in `build.yaml` is `gpt-5.3-codex`, so pass -`--model gpt-5.3-codex` (or an aligned model) to keep the optimizer's model and -the gateway allow-list in lockstep and avoid a 403 mismatch. The evaluation scope -is pinned to `gpt-4o` (matching `model: openai/gpt-4o`). - -**Modal endpoint requirement:** `--environment modal` runs eval sandboxes on -Modal, which must be able to reach the model endpoint in `OPENAI_BASE_URL`. Use a -Modal-reachable endpoint (for example the Azure -`https://.openai.azure.com/openai/v1` endpoint), **not** a VPN-internal -host that only resolves on the Scale network. Use `--environment docker` to run -the optimizer locally against Modal eval backends instead. + --config ../harness-opt-bench/archive/swe-bench-pro/baseline/build.yaml \ + --param inner_env= \ + --output +~~~ + +The `VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time +validation. For a real optimization run, follow the shared +[`run-benchmark` guide](../../skills/run-benchmark/SKILL.md). diff --git a/harness-opt-bench/archive/swe-bench-pro/baseline/build.sample.yaml b/harness-opt-bench/archive/swe-bench-pro/baseline/build.sample.yaml index a6d52c58..119edcd6 100644 --- a/harness-opt-bench/archive/swe-bench-pro/baseline/build.sample.yaml +++ b/harness-opt-bench/archive/swe-bench-pro/baseline/build.sample.yaml @@ -32,7 +32,7 @@ task_source: swebenchpro@1.0 # a second thing to keep in sync. task_manifest: ../partitions/manifest.json agent_import_path: swebench_pro_agent.agent:SweBenchProAgent -harbor_requirement: harbor[modal]==0.20.0 +harbor_requirement: harbor==0.20.0 partition_files: development: ../partitions/sample/development.json @@ -92,10 +92,8 @@ score_baseline: false # pinned as baseline_reward above, like every sibling rescore_top_k: 3 rescore_attempts: 1 -model: fireworks_ai/deepseek-v4-flash -environment_name: ${inner_env:-modal} -# inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default -extra_harbor_args: ["--ek", "app_name=harness-opt-bench", "--ek", "sandbox_idle_timeout_secs=3600"] +model: deepseek-v4-flash +environment_name: ${inner_env:?set inner_env to a Harbor environment} harbor_python_version: "3.12" n_attempts: 1 max_retries: 1 @@ -120,55 +118,25 @@ task_agent_timeout_seconds: 3000 max_concurrency: 24 # matching the rest of the suite error_rate_threshold: 0.1 verifier_timeout_seconds: 28800 -# Limits on the OUTER optimizer trial: the Modal sandbox the optimizer itself runs -# in, not the nested evaluation. Every value reproduces what ran implicitly before -# 2026-09, now written down so the task states its own bounds. -# sandbox timeout: Harbor's Modal default of 24 h. When it fires the sandbox is -# destroyed with no verifier and no archive, so the trial is lost. +# Limits on the outer optimizer trial. The agent timeout leaves time for final +# scoring before the hard sandbox deadline. optimizer_sandbox_timeout_seconds: 86400 -# idle timeout: reclaims a sandbox whose optimizer died with no running command. -# The inner evaluation app has used 3600 since July; the outer sandbox had none. optimizer_sandbox_idle_timeout_seconds: 3600 -# agent clock: Harbor stops the optimizer here and STILL runs the verifier, so a -# runaway is scored and archived instead of lost at the sandbox timeout. Clean -# cells finish in 8-17 h; the longest finalization seen was ~3.5 h, which is -# what the 4 h gap to the sandbox timeout is for. optimizer_agent_timeout_seconds: 72000 -# Harbor's default resolves to public internet; stated rather than inherited. optimizer_allow_internet: true -# optimizer_cpus / optimizer_memory_mb / optimizer_storage_mb are left unset on -# purpose: that is Modal's default (0.125-core / 128 MiB reservation, bursting to -# the host), which is what every August cell ran under. Set them once the first -# rebuttal wave has been read off the Modal dashboard. -secrets: - - MODAL_TOKEN_ID - - MODAL_TOKEN_SECRET - - MODAL_ENVIRONMENT # which Modal environment the sandboxes land in; unset = workspace default - - WANDB_API_KEY - - WANDB_BASE_URL # self-hosted W&B (scaleai.wandb.io) # candidate harness runs as an unprivileged uid, unable to read held-out state. harness_user: harness -wandb: - project: vero-swe-bench-pro - group: swe-bench-pro-sample # distinct from the full-split group - log_traces: true - inference_gateway: upstream_api_key_env: OPENAI_API_KEY upstream_base_url_env: OPENAI_BASE_URL producer: - # NOTE: gpt-5.3-codex has no /responses deployment on the LiteLLM proxy - # (/chat/completions is 200, /responses is 404 DeploymentNotFound), and the - # codex agent is Responses-native, so launch with - # `--model gpt-5.6-sol` (or gpt-4.1) until that deployment exists. + # The optimizer model must support the API used by its harness. allowed_models: ["${optimizer_model:-gpt-5.6-sol}"] max_concurrency: 8 evaluation: - # Prefixed form only: the agents strip just `openai/` before calling the - # gateway, which then matches this allow-list as an exact string. - allowed_models: [fireworks_ai/deepseek-v4-flash] + allowed_models: [deepseek-v4-flash] max_requests: 15000 max_tokens: 100000000 max_concurrency: 64 diff --git a/harness-opt-bench/archive/swe-bench-pro/baseline/build.yaml b/harness-opt-bench/archive/swe-bench-pro/baseline/build.yaml index b516feb0..c04c9261 100644 --- a/harness-opt-bench/archive/swe-bench-pro/baseline/build.yaml +++ b/harness-opt-bench/archive/swe-bench-pro/baseline/build.yaml @@ -12,7 +12,7 @@ agent_repo: target task_source: swebenchpro@1.0 task_manifest: ../partitions/manifest.json agent_import_path: swebench_pro_agent.agent:SweBenchProAgent -harbor_requirement: harbor[modal]==0.20.0 +harbor_requirement: harbor==0.20.0 partition_files: development: ../partitions/development.json @@ -54,9 +54,7 @@ rescore_top_k: 3 rescore_attempts: 1 model: gpt-4o -environment_name: ${inner_env:-modal} -# inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default -extra_harbor_args: ["--ek", "app_name=harness-opt-bench", "--ek", "sandbox_idle_timeout_secs=3600"] +environment_name: ${inner_env:?set inner_env to a Harbor environment} harbor_python_version: "3.12" n_attempts: 1 max_retries: 1 @@ -78,41 +76,16 @@ task_agent_timeout_seconds: 3000 max_concurrency: 8 error_rate_threshold: 0.1 verifier_timeout_seconds: 28800 -# Limits on the OUTER optimizer trial: the Modal sandbox the optimizer itself runs -# in, not the nested evaluation. Every value reproduces what ran implicitly before -# 2026-09, now written down so the task states its own bounds. -# sandbox timeout: Harbor's Modal default of 24 h. When it fires the sandbox is -# destroyed with no verifier and no archive, so the trial is lost. +# Limits on the outer optimizer trial. The agent timeout leaves time for final +# scoring before the hard sandbox deadline. optimizer_sandbox_timeout_seconds: 86400 -# idle timeout: reclaims a sandbox whose optimizer died with no running command. -# The inner evaluation app has used 3600 since July; the outer sandbox had none. optimizer_sandbox_idle_timeout_seconds: 3600 -# agent clock: Harbor stops the optimizer here and STILL runs the verifier, so a -# runaway is scored and archived instead of lost at the sandbox timeout. Clean -# cells finish in 8-17 h; the longest finalization seen was ~3.5 h, which is -# what the 4 h gap to the sandbox timeout is for. optimizer_agent_timeout_seconds: 72000 -# Harbor's default resolves to public internet; stated rather than inherited. optimizer_allow_internet: true -# optimizer_cpus / optimizer_memory_mb / optimizer_storage_mb are left unset on -# purpose: that is Modal's default (0.125-core / 128 MiB reservation, bursting to -# the host), which is what every August cell ran under. Set them once the first -# rebuttal wave has been read off the Modal dashboard. -secrets: - - MODAL_TOKEN_ID - - MODAL_TOKEN_SECRET - - MODAL_ENVIRONMENT # which Modal environment the sandboxes land in; unset = workspace default - - WANDB_API_KEY - - WANDB_BASE_URL # self-hosted or cloud W&B # candidate harness runs as an unprivileged uid, unable to read held-out state. harness_user: harness -wandb: - project: vero-swe-bench-pro - group: swe-bench-pro - log_traces: true - inference_gateway: upstream_api_key_env: OPENAI_API_KEY upstream_base_url_env: OPENAI_BASE_URL diff --git a/harness-opt-bench/archive/swe-bench-pro/baseline/secrets.env.example b/harness-opt-bench/archive/swe-bench-pro/baseline/secrets.env.example index 294da122..34ad0f43 100644 --- a/harness-opt-bench/archive/swe-bench-pro/baseline/secrets.env.example +++ b/harness-opt-bench/archive/swe-bench-pro/baseline/secrets.env.example @@ -2,22 +2,7 @@ # Copy to `secrets.env` (gitignored) and fill in. File values override the # ambient shell and never appear on the command line. -# Upstream inference credentials. VeRO reads these as the real provider key/URL -# (per the compiled gateway's *_source fields), relocates them to the -# VERO_INFERENCE_UPSTREAM_* target vars, and hands the optimizer a scoped -# producer token instead — so the raw key never reaches the coding agent. -# -# For `--environment modal`, OPENAI_BASE_URL must be reachable from Modal's -# network (for example the Azure `https://.openai.azure.com/openai/v1` -# endpoint), NOT a VPN-internal host. See the README. -OPENAI_API_KEY=sk-your-upstream-inference-key -OPENAI_BASE_URL=https://your-openai-compatible-endpoint/v1 - -# Modal credentials for the eval sidecar backends (declared required by the -# compiled task.toml). Copy from your ~/.modal.toml profile. -MODAL_TOKEN_ID=ak-... -MODAL_TOKEN_SECRET=as-... - -# Weights & Biases (self-hosted or cloud) for trace/metric logging. -WANDB_API_KEY=your-wandb-key -WANDB_BASE_URL= +# Inference endpoint used by the gateway. Add any execution-environment +# credentials required by your Harbor installation. +OPENAI_API_KEY= +OPENAI_BASE_URL= diff --git a/harness-opt-bench/archive/swe-bench-pro/baseline/target/pyproject.toml b/harness-opt-bench/archive/swe-bench-pro/baseline/target/pyproject.toml index 5c14d886..4281c2b7 100644 --- a/harness-opt-bench/archive/swe-bench-pro/baseline/target/pyproject.toml +++ b/harness-opt-bench/archive/swe-bench-pro/baseline/target/pyproject.toml @@ -4,7 +4,7 @@ version = "0.1.0" description = "Editable Harbor-native SWE-bench-Pro baseline for VeRO" # Floor at 3.12 (harbor 0.20.0 requires it; ">=3.11" made the package's own # `uv run pytest` unresolvable). Cap below 3.14: an uncapped range lets uv -# resolve to 3.14, whose litellm/pyo3 build fails. +# resolve to 3.14, which is not supported by all transitive dependencies. requires-python = ">=3.12,<3.14" dependencies = [ "harbor==0.20.0", diff --git a/harness-opt-bench/archive/swe-bench-pro/baseline/target/src/swebench_pro_agent/agent.py b/harness-opt-bench/archive/swe-bench-pro/baseline/target/src/swebench_pro_agent/agent.py index 4679e15b..77a64446 100644 --- a/harness-opt-bench/archive/swe-bench-pro/baseline/target/src/swebench_pro_agent/agent.py +++ b/harness-opt-bench/archive/swe-bench-pro/baseline/target/src/swebench_pro_agent/agent.py @@ -31,12 +31,7 @@ class _BinaryOutputError(Exception): def _is_reasoning_model(model: str) -> bool: - """Whether `model` is an OpenAI reasoning model. - - Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects - `reasoning`, and every gpt-5 model accepts it. Fireworks-served open models - match none of these prefixes, so they keep the legacy shape. - """ + """Whether `model` uses the reasoning-model request fields.""" name = model.lower() return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name diff --git a/harness-opt-bench/archive/tau3/README.md b/harness-opt-bench/archive/tau3/README.md index 1bcc8999..bef53829 100644 --- a/harness-opt-bench/archive/tau3/README.md +++ b/harness-opt-bench/archive/tau3/README.md @@ -1,22 +1,34 @@ # tau3-bench -This benchmark optimizes a customer-service agent that talks to the canonical -tau3 simulated user and domain environment through the task's `tau3-runtime` -MCP server. It spans airline, retail, telecom, and banking-knowledge domains. +tau3 evaluates a customer-service agent that talks with a simulated user and a +domain environment through each task's MCP server. -The pinned 375-task dataset is split 75/150/150 for development, validation, -and test. The deterministic split is stratified by domain. +## At a glance -Regenerate or verify the committed split from an exported dataset: +| Item | Value | +| --- | --- | +| Domains | Airline, retail, telecom, and banking knowledge | +| Cases | 375 | +| Development / validation / test | 75 / 150 / 150 | +| Editable harness | `baseline/target/` | +| Split strategy | Domain | +| Status | Archived; not part of the reported suite | -```bash -python harness-opt-bench/scripts/partition_dataset.py tau3 \ - --tasks-dir /path/to/exported/dataset \ - --output-dir harness-opt-bench/candidates/tau3/partitions \ - --fetch-registry +Development exposes complete task results. Validation is aggregate-only, and +test is held out for final scoring. + +## Data and split + +Verify the deterministic split against an exported dataset: +~~~bash python harness-opt-bench/scripts/partition_dataset.py tau3 \ - --tasks-dir /path/to/exported/dataset \ - --output-dir harness-opt-bench/candidates/tau3/partitions \ + --tasks-dir \ + --output-dir harness-opt-bench/archive/tau3/partitions \ --check -``` +~~~ + +Use `--fetch-registry` when intentionally refreshing the pinned package. + +See [the baseline guide](baseline/README.md) for the editable target and build +command. diff --git a/harness-opt-bench/archive/tau3/baseline/README.md b/harness-opt-bench/archive/tau3/baseline/README.md index 2ea10702..757ac392 100644 --- a/harness-opt-bench/archive/tau3/baseline/README.md +++ b/harness-opt-bench/archive/tau3/baseline/README.md @@ -1,25 +1,26 @@ -# tau3 MCP customer-service agent +# tau3 editable target -This leaf benchmark optimizes a Harbor-native agent that connects to the MCP -server declared by each tau3 task, obtains the first simulated-user message, -and carries the conversation through domain and communication tool calls. The -editable program controls its prompt, tool selection, context management, and -conversation policy; the task-owned MCP runtime and canonical evaluator remain -outside the candidate repository. +The seed connects to the MCP server declared by each task, obtains the first +simulated-user message, and carries the conversation through domain and +communication tool calls. -The trusted build pins the target model to `gpt-5.4-mini-2026-03-17` and pins -the Harbor dataset, split, budgets, access policy, and final test partition. -The simulated user and natural-language assertion grader use the canonical -model defaults encoded in the pinned tau3 tasks. +The optimizer may change the prompt, tool selection, context management, +conversation policy, and dependencies in `target/`. The task-owned runtime, +simulated user, evaluator, dataset, partitions, and target model remain fixed. -Compile from the repository root: +## Compile -```bash +From the repository root: + +~~~bash cd vero VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ - --config ../harness-opt-bench/candidates/tau3/baseline/build.yaml \ - --output ../harness-opt-bench/candidates/tau3/baseline/compiled -``` + --config ../harness-opt-bench/archive/tau3/baseline/build.yaml \ + --param inner_env= \ + --output +~~~ -For a real run, provide the OpenAI and Modal credentials declared in -`build.yaml`. +The `VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time +validation. Dataset and split details are in the +[tau3 overview](../README.md). For a real optimization run, follow the shared +[`run-benchmark` guide](../../../skills/run-benchmark/SKILL.md). diff --git a/harness-opt-bench/archive/tau3/baseline/build.yaml b/harness-opt-bench/archive/tau3/baseline/build.yaml index 561674fb..5cb00a20 100644 --- a/harness-opt-bench/archive/tau3/baseline/build.yaml +++ b/harness-opt-bench/archive/tau3/baseline/build.yaml @@ -6,7 +6,7 @@ agent_repo: target task_source: sierra-research/tau3-bench@sha256:a57304f682894ac061090769af771a3617664f3ff6e5417d4eadf8e30433e4d9 task_manifest: ../partitions/manifest.json agent_import_path: tau3_agent.agent:Tau3Agent -harbor_requirement: harbor[modal]==0.20.0 +harbor_requirement: harbor==0.20.0 partition_files: development: ../partitions/development.json @@ -52,16 +52,8 @@ score_baseline: false rescore_top_k: 3 rescore_attempts: 1 -model: fireworks_ai/deepseek-v4-flash -environment_name: ${inner_env:-modal} -# inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default -extra_harbor_args: ["--ek", "app_name=harness-opt-bench", "--ek", "sandbox_idle_timeout_secs=3600"] -# NOTE: tau3's optimizer trial reliably dies at ~38-40 min with grpclib -# StreamTerminatedError ("Connection lost"). Four live experiments (default -# DinD, --ek modal_vm_runtime=true, --ek modal_sandbox_v2=true, and -# --max-retries 1) all failed identically, so no `optimizer_harbor_args` value -# is set here: none of them is a fix, and a config value that claims to be one -# is worse than nothing. Tracked upstream; see the PR that added this field. +model: deepseek-v4-flash +environment_name: ${inner_env:?set inner_env to a Harbor environment} harbor_python_version: "3.12" n_attempts: 1 max_retries: 1 @@ -87,39 +79,12 @@ error_rate_threshold: 0.1 # Unreachable: worst-case finalize (68400) + worst-case rescore_top_k=3 # validation rescore (68400). A verifier timeout loses the score outright. verifier_timeout_seconds: 158400 -# Limits on the OUTER optimizer trial: the Modal sandbox the optimizer itself runs -# in, not the nested evaluation. Every value reproduces what ran implicitly before -# 2026-09, now written down so the task states its own bounds. -# sandbox timeout: Harbor's Modal default of 24 h. When it fires the sandbox is -# destroyed with no verifier and no archive, so the trial is lost. +# Limits on the outer optimizer trial. The agent timeout leaves time for final +# scoring before the hard sandbox deadline. optimizer_sandbox_timeout_seconds: 86400 -# idle timeout: reclaims a sandbox whose optimizer died with no running command. -# The inner evaluation app has used 3600 since July; the outer sandbox had none. optimizer_sandbox_idle_timeout_seconds: 3600 -# agent clock: Harbor stops the optimizer here and STILL runs the verifier, so a -# runaway is scored and archived instead of lost at the sandbox timeout. Clean -# cells finish in 8-17 h; the longest finalization seen was ~3.5 h, which is -# what the 4 h gap to the sandbox timeout is for. optimizer_agent_timeout_seconds: 72000 -# Harbor's default resolves to public internet; stated rather than inherited. optimizer_allow_internet: true -# optimizer_cpus / optimizer_memory_mb / optimizer_storage_mb are left unset on -# purpose: that is Modal's default (0.125-core / 128 MiB reservation, bursting to -# the host), which is what every August cell ran under. Set them once the first -# rebuttal wave has been read off the Modal dashboard. -secrets: - - MODAL_TOKEN_ID - - MODAL_TOKEN_SECRET - - MODAL_ENVIRONMENT # which Modal environment the sandboxes land in; unset = workspace default - - WANDB_API_KEY - - WANDB_BASE_URL # self-hosted or cloud W&B - -wandb: - project: harness-engineering-bench # one project for the whole suite - group: tau3 # keeps the benchmark distinguishable in the shared project - name: ${wandb_run:-tau3} # per-launch label, e.g. --param wandb_run=tau3__claude-sonnet-5 - tags: [tau3] - log_traces: true inference_gateway: upstream_api_key_env: OPENAI_API_KEY @@ -130,19 +95,19 @@ inference_gateway: # 0-13% coverage unstamped against 90-98% stamped, even with --tasks-dir. request_log_attribution: true producer: - allowed_models: ["${optimizer_model:-openai/gpt-5.4}"] + allowed_models: ["${optimizer_model:-gpt-5.4}"] max_concurrency: 8 # See officeqa/baseline/build.yaml for the sizing rationale: the case budget is # the spend control, so a token cap only needs to stop a runaway. tau3 is the # largest set here, hence the bigger ceiling. evaluation: - allowed_models: [fireworks_ai/deepseek-v4-flash] + allowed_models: [deepseek-v4-flash] max_requests: 200000 max_tokens: 4000000000 # 900 agent case-runs (300 dev + 600 validation) max_concurrency: 64 # Reserved so a search-phase overspend can never starve held-out scoring. finalization: - allowed_models: [fireworks_ai/deepseek-v4-flash] + allowed_models: [deepseek-v4-flash] max_requests: 200000 max_tokens: 4000000000 # 150 test cases x3 attempts + rescore headroom max_concurrency: 64 @@ -158,28 +123,14 @@ task_services_use_upstream: true # upstream credential, which uid isolation cannot hide from the harness env. # Re-enable once task-service credentials are delivered off the harness env. harness_user: null -# Optimizer-agent env (forwarded to the harbor claude-code agent as --ae KEY=VALUE). -# Claude Code's Bash tool caps a single call at BASH_MAX_TIMEOUT_MS (default -# 600000=10min), well under one inner eval, which pushed the officeqa optimizer -# into --detach + background-poll + end-turn -- and a headless --print run is -# never re-woken, so the search died there. Raise the cap so a whole eval fits in -# one blocking call. The background-task vars are defence in depth only: they gate -# *automatic* backgrounding and do NOT remove the Bash tool's run_in_background -# parameter, which the model can still choose. The instruction forbids that. +# Allow one full evaluation to complete in a foreground tool call. agent_env: - # Above this benchmark's widest single eval: a full validation pass is - # ceil(150/24) x 3600 = 25200s worst case. BASH_MAX_TIMEOUT_MS: "32400000" - BASH_DEFAULT_TIMEOUT_MS: "32400000" # same as max: an un-timed eval must still block + BASH_DEFAULT_TIMEOUT_MS: "32400000" ENABLE_BACKGROUND_TASKS: "0" FORCE_AUTO_BACKGROUND_TASKS: "0" - # Harnesses installed with `uv tool install` (mini-swe-agent, swe-agent) - # default to symlinking their entry point into /usr/local/bin, which the - # unprivileged optimizer user cannot write: "Failed to install executable - # ... Permission denied". npm/nvm-based harnesses (claude-code, opencode) - # are unaffected, so this only bites when the harness changes. UV_TOOL_BIN_DIR: "/home/agent/.local/bin" task_environment: - TAU2_USER_MODEL: openai/gpt-5.4-mini-2026-03-17 - TAU2_NL_ASSERTIONS_MODEL: openai/gpt-5.4-mini-2026-03-17 + TAU2_USER_MODEL: gpt-5.4-mini-2026-03-17 + TAU2_NL_ASSERTIONS_MODEL: gpt-5.4-mini-2026-03-17 diff --git a/harness-opt-bench/archive/tau3/baseline/target/src/tau3_agent/agent.py b/harness-opt-bench/archive/tau3/baseline/target/src/tau3_agent/agent.py index 1cac3594..795a34e8 100644 --- a/harness-opt-bench/archive/tau3/baseline/target/src/tau3_agent/agent.py +++ b/harness-opt-bench/archive/tau3/baseline/target/src/tau3_agent/agent.py @@ -29,12 +29,7 @@ from openai import AsyncOpenAI def _is_reasoning_model(model: str) -> bool: - """Whether `model` is an OpenAI reasoning model. - - Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects - reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served - open models match none of these prefixes, so they keep the legacy shape. - """ + """Whether `model` uses the reasoning-model request fields.""" name = model.lower() return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name @@ -307,10 +302,8 @@ async def run( environment, session_id, "start_conversation", {} ) self._trace({"turn": 0, "tool": "start_conversation", "result": first_text}) - # Stateless Chat Completions: the whole conversation lives in `messages` - # and is resent each turn. This works across every provider (unlike the - # OpenAI-only Responses API) and keeps full history across plain-text - # customer turns (the previous_response_id reset used to drop it). + # Keep the whole conversation in `messages` and resend it each turn so + # plain-text customer turns remain in context. messages: list[dict[str, Any]] = [ {"role": "system", "content": INSTRUCTIONS}, { @@ -364,15 +357,8 @@ async def run( ) if calls: - # Every call the model made is executed, in order. The target - # model does return more than one per turn, and the usual guard - # -- parallel_tool_calls: False -- is not available here: - # litellm rejects it for fireworks_ai with UnsupportedParamsError. - # Acting on only the first would silently skip the rest, which - # for a customer-service agent means a verified identity with no - # message sent. Chat Completions requires exactly one tool - # message per tool_call id, so the assistant turn lists them all - # and each gets its reply below. + # Execute every requested call in order and return one tool + # message for each id so the conversation remains valid. assistant: dict[str, Any] = { "role": "assistant", "tool_calls": [ diff --git a/harness-opt-bench/browsecomp-plus/README.md b/harness-opt-bench/browsecomp-plus/README.md index 4ddd8de4..4482b19a 100644 --- a/harness-opt-bench/browsecomp-plus/README.md +++ b/harness-opt-bench/browsecomp-plus/README.md @@ -1,53 +1,67 @@ # BrowseComp-Plus -This benchmark turns all 830 queries from -[BrowseComp-Plus](https://github.com/texttron/BrowseComp-Plus) into local Harbor -tasks. It evaluates a deep-research agent against the benchmark's fixed corpus -and canonical BM25 index rather than the live web. +BrowseComp-Plus evaluates a research agent against a fixed document corpus and +BM25 index. The agent searches local evidence rather than the live web. -## Pinned sources +## At a glance -- Upstream repository submodule: `046949032b0328319cc9a02663a759ec601d9402` -- Query dataset `Tevatron/browsecomp-plus`: - `144cff8e35b5eaef7e526346aa60774a9deb941f` -- BM25 index `Tevatron/browsecomp-plus-indexes`: - `b3f37f70c33829eb09d04784a54277a31871fd63` +| Item | Value | +| --- | --- | +| Available queries | 830 | +| Evaluated cases | 165 | +| Development / validation / test | 33 / 66 / 66 | +| Editable harness | `baseline/target/` | +| Target model | `deepseek-v4-flash` | +| Pinned seed baseline | 0.4619 ± 0.0283 | +| Search data | Pinned corpus and BM25 index | +| Scoring | Canonical semantic answer evaluator | -The submodule is the Git pointer requested for the integration. The generator -refuses to run if it is checked out at another commit. Hugging Face revisions -are full immutable commit ids as well. +Development exposes complete task results. Validation is aggregate-only, and +test is held out for final scoring. -## Generate Harbor tasks +## Pinned inputs -Initialize the submodule and run the builder from this directory: +| Input | Revision | +| --- | --- | +| BrowseComp-Plus repository | `046949032b0328319cc9a02663a759ec601d9402` | +| `Tevatron/browsecomp-plus` | `144cff8e35b5eaef7e526346aa60774a9deb941f` | +| `Tevatron/browsecomp-plus-indexes` | `b3f37f70c33829eb09d04784a54277a31871fd63` | -```bash +The task builder rejects a different repository revision. The dataset and index +revisions are immutable inputs to the benchmark. + +## Prepare the tasks + +~~~bash git submodule update --init harness-opt-bench/browsecomp-plus/upstream cd harness-opt-bench/browsecomp-plus uv run --no-project --python 3.12 --with datasets==4.0.0 -- \ python scripts/build_tasks.py -``` - -The builder downloads and decrypts the pinned query dataset using the pinned -upstream implementation, then writes one complete Harbor task per query under -the ignored `tasks/` directory. It also regenerates the committed deterministic -166/332/332 development, validation, and test split. Use `--force` to replace -an existing generated tree or `--check` to verify it byte-for-byte. - -The first Harbor image build downloads the pinned BM25 index (about 2.2 GB). -Every task has an identical environment, so subsequent tasks reuse that image. - -## Scoring and trust boundary - -Answers use BrowseComp-Plus's required Explanation / Exact Answer / Confidence -format. The verifier follows the official upstream OpenAI evaluator and its -default `gpt-4.1` judge. The upstream primary leaderboard instead runs the same -judge prompt with Qwen3-32B; this Harbor integration deliberately uses the -repository's supported API evaluator so tasks do not require a local GPU judge. - -The judge runs inside the task environment and therefore receives the real -upstream credential. As with the SWE-Atlas-QnA rubric judge and tau3 task-owned -LLM services, this disables uid isolation and assumes a non-adversarial -optimizer. The editable baseline itself exposes no live-web or shell tool and -uses only the fixed local index. +~~~ + +The command writes all 830 generated tasks to the ignored `tasks/` directory. +It assigns a deterministic 165-case subset to the committed partitions and +leaves the other tasks unassigned. Use `--check` to verify an existing tree or +`--force` to replace one. + +The first task-image build also retrieves the pinned search index. Later tasks +reuse the same image. + +## Evaluation + +Answers contain an explanation, exact answer, and confidence value. The +task-owned evaluator applies the benchmark's pinned judge prompt and model. The +editable target has access only to the local search index and document reader. + +See [the baseline guide](baseline/README.md) for the editable surface and build +command. + +## Where to look + +| Path | Contents | +| --- | --- | +| `baseline/` | Build configuration and editable target | +| `partitions/` | Committed split and task manifest | +| `scripts/build_tasks.py` | Deterministic task generator | +| `task-template/` | Shared task environment and verifier | diff --git a/harness-opt-bench/browsecomp-plus/baseline/README.md b/harness-opt-bench/browsecomp-plus/baseline/README.md index 38af9e9a..9915d6b5 100644 --- a/harness-opt-bench/browsecomp-plus/baseline/README.md +++ b/harness-opt-bench/browsecomp-plus/baseline/README.md @@ -1,20 +1,26 @@ -# BrowseComp-Plus baseline +# BrowseComp-Plus editable target -This editable target is a Responses API deep-research agent with three tools: -search the pinned BM25 index, open a document, and submit a formatted response. -The optimization agent may change its prompts, control flow, tool use, or -dependencies, but not the dataset, index, split, evaluated model, or verifier. +The seed is a research agent with three tools: -Build the generated tasks first as described in the parent -[`README.md`](../README.md), then compile from the repository root: +- search the pinned BM25 index; +- open a document; +- submit a response in the benchmark's required format. -```bash +The optimizer may change prompts, tool use, control flow, and dependencies in +`target/`. It cannot change the corpus, index, partitions, target model, or +verifier. + +Prepare the generated tasks as described in the +[benchmark overview](../README.md), then compile from the repository root: + +~~~bash cd vero VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ --config ../harness-opt-bench/browsecomp-plus/baseline/build.yaml \ - --output ../harness-opt-bench/browsecomp-plus/baseline/compiled -``` + --param inner_env= \ + --output +~~~ -For a real run, copy `secrets.env.example` to the ignored `secrets.env`, fill it -in, and use `vero harbor run` in the same way as the other harness-engineering -benchmarks. +The `VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time +validation. For a real optimization run, use a local credential file and follow +the shared [`run-benchmark` guide](../../skills/run-benchmark/SKILL.md). diff --git a/harness-opt-bench/browsecomp-plus/baseline/build.yaml b/harness-opt-bench/browsecomp-plus/baseline/build.yaml index 877414d8..97d036da 100644 --- a/harness-opt-bench/browsecomp-plus/baseline/build.yaml +++ b/harness-opt-bench/browsecomp-plus/baseline/build.yaml @@ -6,7 +6,8 @@ agent_repo: target task_source: ../tasks task_manifest: ../partitions/manifest.json agent_import_path: browsecomp_plus_agent.agent:BrowseCompPlusAgent -harbor_requirement: harbor[modal]==0.20.0 +# Include the extra your evaluation environment needs. +harbor_requirement: ${harbor_requirement:-harbor[modal]==0.20.0} partition_files: development: ../partitions/development.json @@ -34,11 +35,7 @@ targets: baseline_reward: 0.4619 # re-pinned 0.424 / 0.470 / 0.492 (sd 0.0283); was 0.4495. See runs/BASELINES.md failure_value: 0.0 max_attempts: 1 - # The held-out eval is noisy: score the selected candidate 3x per case and - # average, so the final reward is comparable to the pinned baseline, which was - # itself pooled over 3 rounds. Without this the candidate carries ~sqrt(3) more - # standard error than the floor it is judged against. - # Per-target override - search/validation keep the global n_attempts (1). + # Match the three-round procedure used to estimate the pinned baseline. n_attempts: 3 aggregate_attempts: mean @@ -53,25 +50,15 @@ score_baseline: false rescore_top_k: 3 rescore_attempts: 1 -model: fireworks_ai/deepseek-v4-flash -environment_name: ${inner_env:-modal} +model: deepseek-v4-flash +environment_name: ${inner_env:?set inner_env to a Harbor environment} +# Group evaluation sandboxes under one app and reclaim idle ones; drop these +# for an environment that does not accept them. extra_harbor_args: ["--ek", "app_name=harness-opt-bench", "--ek", "sandbox_idle_timeout_secs=3600"] harbor_python_version: "3.12" n_attempts: 1 -# Raised from 1 on 2026-07-29. This field exists precisely so "a transient -# upstream rate-limit storm during a mandatory evaluation does not fail -# otherwise-good candidates" (see HarborBuildConfig), and it was set to the -# least aggressive value. Measured that night: the shared Fireworks per-minute -# bucket rejected 78,748 requests across one officeqa pass, sustaining -# ~900-1,000/min for 24 minutes, and SEVEN rejections spread over ten minutes -# were enough to destroy a 37-case evaluation. The retry stack during search is -# only the target agent's own SDK (max_retries=8, ~1 min of jittered backoff, -# and candidate-controlled so it cannot be relied on) plus this one; the -# infrastructure_max_attempts retry below applies to trusted finalization only. -# 4 retries at multiplier 2 from a 4s floor, capped at 120s, gives roughly four -# more minutes of ride-out per case. That covers the bursts, though not a -# 24-minute saturation -- for which the answer is fewer concurrent runs or a -# larger provider quota, not more retrying. +# Retry transient request failures during an evaluation. Whole-evaluation +# retries remain limited to trusted final scoring. max_retries: 4 retry_max_wait_seconds: 120 infrastructure_max_attempts: 3 @@ -80,77 +67,64 @@ aggregate_attempts: best feedback_transcripts: true feedback_max_bytes: 16000 expose_attempt_detail: false -# Unreachable: worst case is ceil(198/24) x 3600 = 32400s, every -# finalize trial (66 held-out x n_attempts=3) hitting its own cap. Assumes -# max_concurrency=24; recompute if that drops. +# Above the 32,400-second worst case at the configured concurrency. timeout_seconds: 39600 -# Exactly the dataset's declared [agent] timeout_sec, so vero's derived -# --agent-timeout-multiplier is 1.0 and the target agent gets precisely the -# clock the benchmark intends. Harbor times agent setup, environment build and -# verification on separate clocks with separate multipliers, so none of them -# eat into this budget and no buffer is warranted. +# Match the dataset's declared agent limit. case_timeout_seconds: 3600 task_agent_timeout_seconds: 3600 -max_concurrency: 24 # 8 -> 24; see officeqa for the measured headroom argument +max_concurrency: 24 error_rate_threshold: 0.1 -# Unreachable: worst-case finalize (32400) + worst-case rescore_top_k=3 -# validation rescore (32400). A verifier timeout loses the score outright. +# Covers final scoring plus validation rescoring. verifier_timeout_seconds: 75600 -# Limits on the OUTER optimizer trial: the Modal sandbox the optimizer itself runs -# in, not the nested evaluation. Every value reproduces what ran implicitly before -# 2026-09, now written down so the task states its own bounds. -# sandbox timeout: Harbor's Modal default of 24 h. When it fires the sandbox is -# destroyed with no verifier and no archive, so the trial is lost. +# Limits on the outer optimizer trial. The agent timeout leaves time for final +# scoring before the hard sandbox deadline. optimizer_sandbox_timeout_seconds: 86400 -# idle timeout: reclaims a sandbox whose optimizer died with no running command. -# The inner evaluation app has used 3600 since July; the outer sandbox had none. optimizer_sandbox_idle_timeout_seconds: 3600 -# agent clock: Harbor stops the optimizer here and STILL runs the verifier, so a -# runaway is scored and archived instead of lost at the sandbox timeout. Clean -# cells finish in 8-17 h; the longest finalization seen was ~3.5 h, which is -# what the 4 h gap to the sandbox timeout is for. optimizer_agent_timeout_seconds: 72000 -# Harbor's default resolves to public internet; stated rather than inherited. optimizer_allow_internet: true -# optimizer_cpus / optimizer_memory_mb / optimizer_storage_mb are left unset on -# purpose: that is Modal's default (0.125-core / 128 MiB reservation, bursting to -# the host), which is what every August cell ran under. Set them once the first -# rebuttal wave has been read off the Modal dashboard. + +# Credentials your execution and telemetry services need. Edit to match your +# deployment; each name must be set in the env file. secrets: - MODAL_TOKEN_ID - MODAL_TOKEN_SECRET - - MODAL_ENVIRONMENT # which Modal environment the sandboxes land in; unset = workspace default + - MODAL_ENVIRONMENT - WANDB_API_KEY - WANDB_BASE_URL +# Telemetry (optional). Remove this block to run without it. wandb: - project: harness-engineering-bench # one project for the whole suite - group: browsecomp-plus # keeps the benchmark distinguishable in the shared project - name: ${wandb_run:-browsecomp-plus} # per-launch label, e.g. --param wandb_run=browsecomp-plus__claude-sonnet-5 + project: harness-engineering-bench + group: browsecomp-plus + name: ${wandb_run:-browsecomp-plus} tags: [browsecomp-plus] log_traces: true inference_gateway: upstream_api_key_env: OPENAI_API_KEY upstream_base_url_env: OPENAI_BASE_URL - # Stamp request-log records with a thread_id so per_trial_tokens.py can attribute - # gateway token usage to individual trials (trusted, vs. content-matching). - # Without it the fallback recovers only each conversation's root turn: measured - # 0-13% coverage unstamped against 90-98% stamped, even with --tasks-dir. + # Enables reliable per-trial usage attribution. request_log_attribution: true producer: - allowed_models: ["${optimizer_model:-openai/gpt-5.4}"] + allowed_models: ["${optimizer_model:-gpt-5.4}"] max_concurrency: 8 - # See officeqa/baseline/build.yaml for the sizing rationale: the case budget is - # the spend control, so a token cap only needs to stop a runaway. + # Case budgets control normal use; token caps stop runaway requests. evaluation: - allowed_models: [fireworks_ai/deepseek-v4-flash] + allowed_models: [deepseek-v4-flash] + # Map the target model to a deployment; override per launch with + # --param target_model_route=. + model_aliases: + deepseek-v4-flash: ${target_model_route:-fireworks_ai/deepseek-v4-flash-0731} max_requests: 200000 max_tokens: 2000000000 # 396 agent case-runs (132 dev + 264 validation) max_concurrency: 64 - # Reserved so a search-phase overspend can never starve held-out scoring. + # Keep final scoring capacity separate from search. finalization: - allowed_models: [fireworks_ai/deepseek-v4-flash] + allowed_models: [deepseek-v4-flash] + # Map the target model to a deployment; override per launch with + # --param target_model_route=. + model_aliases: + deepseek-v4-flash: ${target_model_route:-fireworks_ai/deepseek-v4-flash-0731} max_requests: 200000 max_tokens: 2000000000 # 66 test cases x3 attempts + rescore headroom max_concurrency: 64 @@ -158,31 +132,16 @@ inference_gateway: instruct_multifidelity: true instruct_exhaust_budget: true -# The official upstream OpenAI evaluator runs inside the task container. It -# uses the raw upstream credential while the editable target agent continues to -# receive only the evaluation-scoped gateway credential. +# The task-owned evaluator needs direct endpoint access; the editable target +# continues to receive only an evaluation-scoped gateway credential. task_services_use_upstream: true harness_user: null -# Optimizer-agent env (forwarded to the harbor claude-code agent as --ae KEY=VALUE). -# Claude Code's Bash tool caps a single call at BASH_MAX_TIMEOUT_MS (default -# 600000=10min), well under one inner eval, which pushed the officeqa optimizer -# into --detach + background-poll + end-turn -- and a headless --print run is -# never re-woken, so the search died there. Raise the cap so a whole eval fits in -# one blocking call. The background-task vars are defence in depth only: they gate -# *automatic* backgrounding and do NOT remove the Bash tool's run_in_background -# parameter, which the model can still choose. The instruction forbids that. +# Allow one full evaluation to complete in a foreground tool call. agent_env: - # Above this benchmark's widest single eval: a full validation pass is - # ceil(66/24) x 3600 = 10800s worst case. BASH_MAX_TIMEOUT_MS: "14400000" - BASH_DEFAULT_TIMEOUT_MS: "14400000" # same as max: an un-timed eval must still block + BASH_DEFAULT_TIMEOUT_MS: "14400000" ENABLE_BACKGROUND_TASKS: "0" FORCE_AUTO_BACKGROUND_TASKS: "0" - # Harnesses installed with `uv tool install` (mini-swe-agent, swe-agent) - # default to symlinking their entry point into /usr/local/bin, which the - # unprivileged optimizer user cannot write: "Failed to install executable - # ... Permission denied". npm/nvm-based harnesses (claude-code, opencode) - # are unaffected, so this only bites when the harness changes. UV_TOOL_BIN_DIR: "/home/agent/.local/bin" task_environment: diff --git a/harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json b/harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json index 1eddddf5..68e68914 100644 --- a/harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json +++ b/harness-opt-bench/browsecomp-plus/baseline/compiled.manifest.json @@ -73,7 +73,7 @@ "environment/agent-seed/uv.lock": "e8f297525100fbd6c660b85970c7f02ec779ee35bbd8d3020108f7fd3146d05e", "environment/docker-compose.yaml": "e29555f903e97c83d00da27df88dd317bf263bda1b2b2675462424837daeb938", "environment/gateway/Dockerfile": "70d149f1928b135b8e11b7bb8d4ef70d81f331a6766bb3947192ff876e2eea26", - "environment/gateway/config.json": "650eabf9ff9899771d5d39b0aa6f233f8405bdd98a1261aab9c3bc5beaeb6f97", + "environment/gateway/config.json": "5f4591138c708db33676c429548132aaa26b1d25c0f5a7330e40f9f452bc3943", "environment/gateway/launch.json": "462bcc8d2749f1364d79e14d36be7144a84441b6bb74c985f798115ef26636ae", "environment/main/seed.sh": "f51b70ef8fbc646ab6f129d2d7f8f3a72a1c37a9331691dcbd4f9b899dc7bc90", "environment/overlay/skills/evals/SKILL.md": "32cd544a37264d478b01985f11178f6476b093f6c6b2f682d4495b43a14e28cc", @@ -81,13 +81,13 @@ "environment/sidecar/cases/development.jsonl": "becd3eef6841ad0174b8ec4bc73088432221f8849c48a5c822b955e62003c17d", "environment/sidecar/cases/test.jsonl": "b20f6750e35cabdfb65c186ff886026f78ed4b37e3cb11e67fe7538531e1dc6d", "environment/sidecar/cases/validation.jsonl": "4d46fc533edc1862d9ee04f8b6fcb835c017a9360c8a1a88cb7a6963cd54aa91", - "environment/sidecar/serve.json": "fcad4e44748f06d8e2d1296dca60a2b8674e94e8618a79539588554992d69bd3", + "environment/sidecar/serve.json": "a8703c93a2d55cf36c96a55cc215d12fc8b4c16b9aaa86196f4e159b99ebb08f", "environment/sidecar/task-source/": "abfee46a96f0ffa72ad3b4b287227ffb32a42cdc9188e1dddc72685dfcfa7c30", - "environment/vero/": "e12fbae407e3bf6acfaa6445e5debc683592b66180ab69b6e6eff2b395f93f30", + "environment/vero/": "b4b959797b70e9ae9583effc256dedfc5219e3b165aabfbd51a0c979f09e8818", "instruction.md": "2665ef594b9dffcc447e8013ac15c633cd6baf69b7e019fef92382f479c71cb7", "solution/solve.sh": "906e1e8dcd72edd2fb1f11f36bb29f2df7c203269ffd7badf9fa7632bcd98544", "task.toml": "d0bba169e6e962f4fe90d13ebf675f732c8ed80ae3daa2f0b6f6eff8ce211ce8", "tests/test.sh": "b65e90e6d67aacedd559c6b1c9d0f1d5c5071ecc9178a5bd7f9e1066f332bb0d" }, - "vero_commit": "96e57cf22e5b7f3a4e8cea9ade61bf5f61c66692" + "vero_commit": "09deba52bd57913f86795423dd54c9ad5970d8ea" } diff --git a/harness-opt-bench/browsecomp-plus/baseline/secrets.env.example b/harness-opt-bench/browsecomp-plus/baseline/secrets.env.example index a048be71..9b74ff64 100644 --- a/harness-opt-bench/browsecomp-plus/baseline/secrets.env.example +++ b/harness-opt-bench/browsecomp-plus/baseline/secrets.env.example @@ -1,10 +1,12 @@ -# Optimizer and evaluated-agent inference. OPENAI_BASE_URL may point to an -# OpenAI-compatible upstream; use https://api.openai.com/v1 for OpenAI. -OPENAI_API_KEY=sk-your-upstream-inference-key -OPENAI_BASE_URL=https://api.openai.com/v1 +# Inference endpoint used by the gateway. Add any execution-environment +# credentials required by your Harbor installation. +OPENAI_API_KEY= +OPENAI_BASE_URL= -MODAL_TOKEN_ID=your-modal-token-id -MODAL_TOKEN_SECRET=your-modal-token-secret - -WANDB_API_KEY=your-wandb-api-key -WANDB_BASE_URL=https://api.wandb.ai +# Execution environment and telemetry, as declared under `secrets` in build.yaml. +# Edit both together if your deployment uses different services. +MODAL_TOKEN_ID= +MODAL_TOKEN_SECRET= +MODAL_ENVIRONMENT= +WANDB_API_KEY= +WANDB_BASE_URL= diff --git a/harness-opt-bench/browsecomp-plus/baseline/target/src/browsecomp_plus_agent/agent.py b/harness-opt-bench/browsecomp-plus/baseline/target/src/browsecomp_plus_agent/agent.py index 6189527d..1fd17e8a 100644 --- a/harness-opt-bench/browsecomp-plus/baseline/target/src/browsecomp_plus_agent/agent.py +++ b/harness-opt-bench/browsecomp-plus/baseline/target/src/browsecomp_plus_agent/agent.py @@ -13,12 +13,7 @@ from openai import AsyncOpenAI def _is_reasoning_model(model: str) -> bool: - """Whether `model` is an OpenAI reasoning model. - - Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects - reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served - open models match none of these prefixes, so they keep the legacy shape. - """ + """Whether `model` uses the reasoning-model request fields.""" name = model.lower() return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name @@ -225,9 +220,7 @@ async def run( environment: BaseEnvironment, context: AgentContext, ) -> None: - # Stateless Chat Completions: the full message history is resent each - # turn (provider prompt-caching handles the repeated prefix), which - # works across every provider, unlike the OpenAI-only Responses API. + # The API is stateless, so resend the full message history each turn. messages: list[dict[str, Any]] = [ {"role": "system", "content": INSTRUCTIONS}, {"role": "user", "content": instruction}, diff --git a/harness-opt-bench/gaia/README.md b/harness-opt-bench/gaia/README.md index b79f53ad..50f78e72 100644 --- a/harness-opt-bench/gaia/README.md +++ b/harness-opt-bench/gaia/README.md @@ -1,39 +1,57 @@ # GAIA -The GAIA benchmark uses the canonical Harbor task packages and verifier. It -does not reproduce the paper's pure-language subset or its original split. - -The committed split is deterministic and stratified by GAIA level and whether -the task has an attached file: - -- development: 33 cases (20%) -- validation: 66 cases (40%) -- test: 66 cases (40%) - -All 165 cases come from the immutable dataset reference recorded in -[`partitions/manifest.json`](partitions/manifest.json). The development set is -available to the optimization agent with full result disclosure. Validation is -aggregate-only and is used to select candidates. Test is held out until Harbor -grades the completed outer task. The complete development task packages and -attachments are mounted read-only under `.evals/tasks/`; successful and failed -development evaluations place their complete Harbor trial records—including -exact failures and target-agent logs—under -`.evals/results/`. Neither validation nor test resources are mounted. - -The pinned GAIA tasks declare a 600-second Harbor agent timeout. The build -config sets `case_timeout_seconds: 180` and -`task_agent_timeout_seconds: 600`, so VeRO invokes Harbor with an agent-timeout -multiplier of `0.3`. The 180-second limit reported to the optimizer is therefore -the timeout enforced by the inner Harbor trial. - -To verify or regenerate the split after downloading the pinned Harbor dataset: - -```bash -uv run --python 3.12 scripts/partition_gaia.py \ - --tasks-dir /path/to/downloaded/gaia \ +GAIA evaluates a multimodal research agent that can search the web, inspect +files and images, run shell commands, and submit an exact answer. + +## At a glance + +| Item | Value | +| --- | --- | +| Task source | Pinned GAIA Harbor package | +| Editable harness | `baseline/target/` | +| Development / validation / test | 33 / 66 / 66 | +| Split strategy | GAIA level and attachment presence | +| Target model | `gpt-5.4-mini` | +| Pinned seed baselines | 0.6205 working target; 0.0 shell target | +| Scoring | Canonical task verifier | + +Development cases expose full results and task resources. Validation exposes +aggregate scores, and test remains hidden until final scoring. + +## Variants + +| Build | Starting point | Purpose | +| --- | --- | --- | +| `baseline/build.yaml` | Working tool-using agent | Measure harness improvement | +| `baseline/build.shell.yaml` | Minimal non-solving skeleton | Measure harness construction from scratch | +| `baseline/build.shell.e2e.yaml` | Skeleton with eight cases | Exercise the complete pipeline quickly | + +The full and shell builds use the same tasks, model, budgets, and evaluation +policy. See [the baseline guide](baseline/README.md) for their editable surfaces +and build commands. + +## Data and split + +The 165 task references are pinned in +[`partitions/manifest.json`](partitions/manifest.json). To verify the committed +split against downloaded tasks, run from the repository root: + +~~~bash +uv run --python 3.12 harness-opt-bench/gaia/scripts/partition_gaia.py \ + --tasks-dir \ --check -``` +~~~ + +Changing the dataset revision is a benchmark change. Update the source pin, +regenerate the partitions, and review the manifest before using new results. + +## Where to look -Refreshing from a different registry revision is an explicit operation: update -the dataset constant in the script and `build.yaml`, then run with -`--fetch-registry`. Review the manifest diff before committing it. +| Path | Contents | +| --- | --- | +| `baseline/build*.yaml` | Benchmark variants and evaluation settings | +| `baseline/target/` | Working editable agent | +| `baseline/target-shell/` | Minimal editable skeleton | +| `partitions/` | Reported development, validation, and test split | +| `partitions-e2e/` | Small smoke-test split | +| `scripts/partition_gaia.py` | Split verification and regeneration | diff --git a/harness-opt-bench/gaia/baseline/README.md b/harness-opt-bench/gaia/baseline/README.md index b84ef3fd..5f91ae38 100644 --- a/harness-opt-bench/gaia/baseline/README.md +++ b/harness-opt-bench/gaia/baseline/README.md @@ -1,131 +1,43 @@ -# GAIA tool-using baseline +# GAIA editable targets -This leaf benchmark optimizes a small Harbor-native GAIA agent. The editable -program uses the OpenAI Responses API with three capabilities: +This directory contains two starting points for the same GAIA evaluation. -- web search; -- shell commands inside the GAIA task environment; -- image inspection and final-answer submission. +| Target | Build file | Starting behavior | +| --- | --- | --- | +| `target/` | `build.yaml` | Searches, uses the task shell, inspects images, and submits an answer | +| `target-shell/` | `build.shell.yaml` | Loads successfully but submits an empty answer | -The trusted build fixes the evaluated model to -`gpt-5.4-mini-2026-03-17`. A candidate may change the agent's prompts, control -flow, tool definitions, file handling, or dependencies, but it cannot change -the dataset, verifier, split, model, access policy, or final test target. +The optimizer may change prompts, tools, control flow, file handling, and +dependencies inside the selected target. The dataset, partitions, target model, +verifier, and evaluation policy remain fixed. -## The shell-seed variant (`build.shell.yaml`) +## Why keep a shell target? -`build.shell.yaml` runs the same benchmark against `target-shell/` instead of -`target/`. The skeleton there satisfies the Harbor interface, resolves the model -and constructs a client, but its `run` writes an empty answer and returns. It -scores zero. +The working seed measures improvement to an existing harness. The shell target +measures whether an optimizer can build the harness itself. It deliberately +submits an empty answer so every case receives a valid zero rather than becoming +an evaluation error. -**Why.** Every other optimization task in the suite starts from a seed that -already works, so what they measure is *tuning*: how much an optimizer can add -to a competent program. This variant asks a different question -- what an -optimizer does when there is nothing to tune and it has to write the program -first. The two runs are directly comparable because everything except the seed -is held fixed: same cases, same partitions, same target model, same gateway -scoping. The invariant test in `vero/tests/test_v05_benchmark_configs.py` -enforces that, and also asserts the skeleton never calls a model -- a shell that -scores above zero is not a shell. +The shell-specific instruction extends the shared task instructions and changes +only the opening frame from “improve” to “build, then optimize.” Design notes stay +outside the editable target so they do not become hints. -**Why the plumbing is present.** The skeleton keeps the model resolution and the -client construction, including the `removeprefix("openai/")` that the gateway -allow-list requires. Those are properties of this harness, not of GAIA, and -making an optimizer rediscover them by trial and error would spend budget on the -wrong thing and add variance unrelated to the question being asked. - -**Why it writes an empty answer rather than doing nothing.** A case that scores -zero and a case that errors are different events here. Errors count against -`error_rate_threshold`, and a wholly erroring evaluation comes back `invalid`. -Writing the file keeps every case scoreable, so the floor is a real 0.0 and a -half-built candidate gets a number rather than nothing. - -**`baseline_reward: 0.0` is a claim, not a measurement.** It follows from the -skeleton writing an empty answer. Confirm it with one baseline round before -quoting deltas against it. - -**The framing lives in a template, not in `description`.** The built-in -instruction opens with "Improve the program in ...", which is the first thing the -optimizer reads and is false here. `instruction_template: -instruction.shell.md.j2` replaces that opening with "Build the program, then -optimize it". The template `{% extends "instruction.md.j2" %}` and overrides only -the `framing` block, so the workflow, budget, inspection and rules sections are -inherited verbatim and cannot drift as the shared instruction evolves -- the -invariant test asserts both the extends and the byte-identical remainder. -`description` is left saying only what the program must *do*, which keeps task -shape and task content in separate places. - -**Deliberately not in the target.** The rationale above lives in this README -because the optimizer mounts `target-shell/` and would read anything placed -there. Notes about what the seed lacks, or what a good implementation would -contain, are an answer key. - -## Compile the outer Harbor task +## Compile a build From the repository root: -```bash +~~~bash cd vero VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ --config ../harness-opt-bench/gaia/baseline/build.yaml \ - --output ../harness-opt-bench/gaia/baseline/compiled -``` - -Omit `VERO_SKIP_SECRET_CHECK=1` for a real build so VeRO verifies that the -OpenAI and Modal credentials declared in `build.yaml` are present. Set -`OPENAI_BASE_URL` to your OpenAI-compatible endpoint; use -`https://api.openai.com/v1` when calling OpenAI directly. The `compiled/` -directory is generated and intentionally ignored. - -The compiled task does not expose the upstream OpenAI key to either editable -program. A separate inference-gateway container holds it. The outer coding -agent uses a producer-scoped token for `gpt-5.4` or `gpt-5.5`; GAIA candidates use an -evaluation-scoped token restricted to `gpt-5.4-mini-2026-03-17`. Their request -and token budgets are recorded separately and are visible through -`evals status`. - -Run the outer optimization through VeRO so it can give Harbor's coding-agent -adapter the scoped credential while forwarding the upstream credential only to -the gateway. `vero harbor run` is the whole pipeline in one command: it compiles -the `build.yaml` to a temporary task, wires credentials (relocating the upstream -inference key to the gateway and handing the optimizer a scoped producer token), -and invokes Harbor. Put run secrets in a dotenv file rather than exporting them -by hand — copy `secrets.env.example` to `secrets.env` (gitignored) and fill it -in: - -```bash -cp harness-opt-bench/gaia/baseline/secrets.env.example \ - harness-opt-bench/gaia/baseline/secrets.env # then edit it - -cd vero -uv run vero harbor run \ - --config ../harness-opt-bench/gaia/baseline/build.yaml \ - --env-file ../harness-opt-bench/gaia/baseline/secrets.env \ - --environment docker \ - --agent codex \ - --model gpt-5.6-sol \ - --yes \ - -o ../runs/gaia-full/jobs -``` - -`--env-file` values override the ambient shell and are passed only through the -subprocess environment, never on the command line. Anything after the known -options (here `--yes -o ...`) is forwarded verbatim to `harbor run`. The -`--model` value is fed to `${optimizer_model}` so the producer scope's -allow-list and the optimizer agent's model stay in lockstep (no 403 mismatch). -Use `--environment modal` to run the optimizer itself remotely, or `docker` to -run it locally against Modal eval backends. + --param inner_env= \ + --output +~~~ -The coding agent edits only `target/`; inner evaluations run the candidate -against the pinned GAIA tasks on Modal. Complete development tasks and -attachments are mounted read-only under `.evals/tasks/`. After each development -evaluation, complete Harbor trial records for every case—including exact -failures and target-agent logs—are available under -`.evals/results/`. Validation remains aggregate-only, and test remains -verifier-only. +Use `build.shell.yaml` in place of `build.yaml` to compile the shell variant. +The `VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time +validation. -Before Modal teardown, the shared verifier exports the complete VeRO session -and a self-contained `experiment.html` into Harbor's verifier artifacts. Keep -the session archive alongside the HTML: it is the canonical candidate and -evaluation history and can be rendered again with `vero report`. +For a real optimization run, follow the shared +[`run-benchmark` guide](../../skills/run-benchmark/SKILL.md). Dataset and split +details are in the [GAIA overview](../README.md). diff --git a/harness-opt-bench/gaia/baseline/build.shell.e2e.yaml b/harness-opt-bench/gaia/baseline/build.shell.e2e.yaml index fe7519b2..e78ebd72 100644 --- a/harness-opt-bench/gaia/baseline/build.shell.e2e.yaml +++ b/harness-opt-bench/gaia/baseline/build.shell.e2e.yaml @@ -11,11 +11,8 @@ agent_repo: target-shell task_source: gaia/gaia@sha256:bbc356f476e0b70ba77da11a9be7d6345918d1e4a2daade0d6dfb82ee6f7b761 task_manifest: ../partitions/manifest.json agent_import_path: gaia_agent.agent:GaiaAgent -harbor_requirement: harbor[modal]==0.20.0 -# Set once the vero on PyPI carries the compiled-task changes this branch relies on -# (env-delivered scope tokens, outer-trial limits); until then the images copy the -# local source. With it set, the compiled task holds no vero and can be committed. -# vero_requirement: scaleapi-vero==0.6.0 +# Include the extra your evaluation environment needs. +harbor_requirement: ${harbor_requirement:-harbor[modal]==0.20.0} # E2E SMOKE VARIANT of build.shell.yaml. Same seed, same target, same gateway # scopes; eight cases in total, tiny budgets, and a 25-minute agent clock so the @@ -27,7 +24,6 @@ partition_files: validation: ../partitions-e2e/validation.json test: ../partitions-e2e/test.json -# total_cases = 4 full passes per partition (dev 33*4, validation 66*4). agent_access: - partition: development disclosure: full @@ -52,11 +48,7 @@ targets: baseline_reward: 0.0 failure_value: 0.0 max_attempts: 1 - # The held-out eval is noisy: score the selected candidate 3x per case and - # average, so the final reward is comparable to the pinned baseline, which was - # itself pooled over 3 rounds. Without this the candidate carries ~sqrt(3) more - # standard error than the floor it is judged against. - # Per-target override - search/validation keep the global n_attempts (1). + # Match the three-round procedure used to estimate the pinned baseline. n_attempts: 3 aggregate_attempts: mean @@ -71,32 +63,17 @@ score_baseline: false rescore_top_k: 1 rescore_attempts: 1 -# GAIA is multimodal: 5 of the 66 held-out tasks send image inputs, and a -# text-only target 400s on them ("This model does not support image inputs"), -# which reads as ordinary agent failure and caps achievable reward near 0.92. -# Unprefixed on purpose -- the agent sends model_name.removeprefix("openai/"), -# so an openai/-prefixed name here would be allow-listed in one form and -# requested in another, and the gateway would deny it. +# GAIA includes image inputs, so the target must be multimodal. Keep this name +# aligned with the exact value sent by the agent. model: gpt-5.4-mini -environment_name: ${inner_env:-modal} -# inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default +environment_name: ${inner_env:?set inner_env to a Harbor environment} +# Group evaluation sandboxes under one app and reclaim idle ones; drop these +# for an environment that does not accept them. extra_harbor_args: ["--ek", "app_name=harness-opt-bench", "--ek", "sandbox_idle_timeout_secs=3600"] harbor_python_version: "3.12" n_attempts: 1 -# Raised from 1 on 2026-07-29. This field exists precisely so "a transient -# upstream rate-limit storm during a mandatory evaluation does not fail -# otherwise-good candidates" (see HarborBuildConfig), and it was set to the -# least aggressive value. Measured that night: the shared Fireworks per-minute -# bucket rejected 78,748 requests across one officeqa pass, sustaining -# ~900-1,000/min for 24 minutes, and SEVEN rejections spread over ten minutes -# were enough to destroy a 37-case evaluation. The retry stack during search is -# only the target agent's own SDK (max_retries=8, ~1 min of jittered backoff, -# and candidate-controlled so it cannot be relied on) plus this one; the -# infrastructure_max_attempts retry below applies to trusted finalization only. -# 4 retries at multiplier 2 from a 4s floor, capped at 120s, gives roughly four -# more minutes of ride-out per case. That covers the bursts, though not a -# 24-minute saturation -- for which the answer is fewer concurrent runs or a -# larger provider quota, not more retrying. +# Retry transient request failures during an evaluation. Whole-evaluation +# retries remain limited to trusted final scoring. max_retries: 4 retry_max_wait_seconds: 120 infrastructure_max_attempts: 3 @@ -105,97 +82,60 @@ aggregate_attempts: best feedback_transcripts: true feedback_max_bytes: 16000 expose_attempt_detail: false -# Unreachable: worst case is ceil(198/24) x 600 = 5400s, every -# finalize trial (66 held-out x n_attempts=3) hitting its own cap. Assumes -# max_concurrency=24; recompute if that drops. +# This smoke variant intentionally uses shorter overall limits. timeout_seconds: 1800 -# Exactly the dataset's declared [agent] timeout_sec, so vero's derived -# --agent-timeout-multiplier is 1.0 and the target agent gets precisely the -# clock the benchmark intends. Harbor times agent setup, environment build and -# verification on separate clocks with separate multipliers, so none of them -# eat into this budget and no buffer is warranted. case_timeout_seconds: 600 task_agent_timeout_seconds: 600 -max_concurrency: 24 # 8 -> 24; see officeqa for the measured headroom argument +max_concurrency: 24 error_rate_threshold: 0.1 -# Unreachable: worst-case finalize (5400) + worst-case rescore_top_k=3 -# validation rescore (5400). A verifier timeout loses the score outright. verifier_timeout_seconds: 3600 -# Limits on the OUTER optimizer trial: the Modal sandbox the optimizer itself runs -# in, not the nested evaluation. Every value reproduces what ran implicitly before -# 2026-09, now written down so the task states its own bounds. -# sandbox timeout: Harbor's Modal default of 24 h. When it fires the sandbox is -# destroyed with no verifier and no archive, so the trial is lost. +# Limits on the outer optimizer trial. The agent timeout leaves time for final +# scoring before the hard sandbox deadline. optimizer_sandbox_timeout_seconds: 14400 -# idle timeout: reclaims a sandbox whose optimizer died with no running command. -# The inner evaluation app has used 3600 since July; the outer sandbox had none. optimizer_sandbox_idle_timeout_seconds: 1800 -# agent clock: Harbor stops the optimizer here and STILL runs the verifier, so a -# runaway is scored and archived instead of lost at the sandbox timeout. Clean -# cells finish in 8-17 h; the longest finalization seen was ~3.5 h, which is -# what the 4 h gap to the sandbox timeout is for. # 25 minutes: long enough for the optimizer to run a few evaluations, short # enough that Harbor stops it and hands over to the verifier within the hour, # which is the path this variant exists to exercise. optimizer_agent_timeout_seconds: 1500 -# Harbor's default resolves to public internet; stated rather than inherited. optimizer_allow_internet: true -# optimizer_cpus / optimizer_memory_mb / optimizer_storage_mb are left unset on -# purpose: that is Modal's default (0.125-core / 128 MiB reservation, bursting to -# the host), which is what every August cell ran under. Set them once the first -# rebuttal wave has been read off the Modal dashboard. + +# Credentials your execution and telemetry services need. Edit to match your +# deployment; each name must be set in the env file. secrets: - MODAL_TOKEN_ID - MODAL_TOKEN_SECRET - - MODAL_ENVIRONMENT # which Modal environment the sandboxes land in; unset = workspace default + - MODAL_ENVIRONMENT - WANDB_API_KEY - - WANDB_BASE_URL # self-hosted or cloud W&B + - WANDB_BASE_URL + +# Telemetry (optional). Remove this block to run without it. +wandb: + project: harness-engineering-bench + group: gaia + name: ${wandb_run:-gaia-shell-e2e} + tags: [gaia, shell-seed, e2e-smoke] + log_traces: true -# candidate harness runs as an unprivileged uid, unable to read held-out state. +# Keep the candidate isolated from held-out state. harness_user: harness -# Optimizer-agent env (forwarded to the harbor claude-code agent as --ae KEY=VALUE). -# Claude Code's Bash tool caps a single call at BASH_MAX_TIMEOUT_MS (default -# 600000=10min), well under one inner eval, which pushed the officeqa optimizer -# into --detach + background-poll + end-turn -- and a headless --print run is -# never re-woken, so the search died there. Raise the cap so a whole eval fits in -# one blocking call. The background-task vars are defence in depth only: they gate -# *automatic* backgrounding and do NOT remove the Bash tool's run_in_background -# parameter, which the model can still choose. The instruction forbids that. +# Allow one full evaluation to complete in a foreground tool call. agent_env: - # Above this benchmark's widest single eval: a full validation pass is - # ceil(66/24) x 600 = 1800s worst case. BASH_MAX_TIMEOUT_MS: "3600000" - BASH_DEFAULT_TIMEOUT_MS: "3600000" # same as max: an un-timed eval must still block + BASH_DEFAULT_TIMEOUT_MS: "3600000" ENABLE_BACKGROUND_TASKS: "0" FORCE_AUTO_BACKGROUND_TASKS: "0" - # Harnesses installed with `uv tool install` (mini-swe-agent, swe-agent) - # default to symlinking their entry point into /usr/local/bin, which the - # unprivileged optimizer user cannot write: "Failed to install executable - # ... Permission denied". npm/nvm-based harnesses (claude-code, opencode) - # are unaffected, so this only bites when the harness changes. UV_TOOL_BIN_DIR: "/home/agent/.local/bin" -wandb: - project: harness-engineering-bench # one project for the whole suite - group: gaia # keeps the benchmark distinguishable in the shared project - name: ${wandb_run:-gaia-shell-e2e} # per-launch label, e.g. --param wandb_run=gaia__opencode - tags: [gaia, shell-seed, e2e-smoke] - log_traces: true - inference_gateway: upstream_api_key_env: OPENAI_API_KEY upstream_base_url_env: OPENAI_BASE_URL - # Stamp request-log records with a thread_id so per_trial_tokens.py can attribute - # gateway token usage to individual trials (trusted, vs. content-matching). - # Without it the fallback recovers only each conversation's root turn: measured - # 0-13% coverage unstamped against 90-98% stamped, even with --tasks-dir. + # Enables reliable per-trial usage attribution. request_log_attribution: true producer: - allowed_models: ["${optimizer_model:-openai/gpt-5.4}"] + allowed_models: ["${optimizer_model:-gpt-5.4}"] max_concurrency: 8 - # See officeqa/baseline/build.yaml for the sizing rationale: the case budget is - # the spend control, so a token cap only needs to stop a runaway. + # Case budgets control normal use; token caps stop runaway requests. evaluation: allowed_models: [gpt-5.4-mini] max_requests: 200000 diff --git a/harness-opt-bench/gaia/baseline/build.shell.yaml b/harness-opt-bench/gaia/baseline/build.shell.yaml index f08b0dfa..c0d12890 100644 --- a/harness-opt-bench/gaia/baseline/build.shell.yaml +++ b/harness-opt-bench/gaia/baseline/build.shell.yaml @@ -11,18 +11,14 @@ agent_repo: target-shell task_source: gaia/gaia@sha256:bbc356f476e0b70ba77da11a9be7d6345918d1e4a2daade0d6dfb82ee6f7b761 task_manifest: ../partitions/manifest.json agent_import_path: gaia_agent.agent:GaiaAgent -harbor_requirement: harbor[modal]==0.20.0 -# Set once the vero on PyPI carries the compiled-task changes this branch relies on -# (env-delivered scope tokens, outer-trial limits); until then the images copy the -# local source. With it set, the compiled task holds no vero and can be committed. -# vero_requirement: scaleapi-vero==0.6.0 +# Include the extra your evaluation environment needs. +harbor_requirement: ${harbor_requirement:-harbor[modal]==0.20.0} partition_files: development: ../partitions/development.json validation: ../partitions/validation.json test: ../partitions/test.json -# total_cases = 4 full passes per partition (dev 33*4, validation 66*4). agent_access: - partition: development disclosure: full @@ -47,11 +43,7 @@ targets: baseline_reward: 0.0 failure_value: 0.0 max_attempts: 1 - # The held-out eval is noisy: score the selected candidate 3x per case and - # average, so the final reward is comparable to the pinned baseline, which was - # itself pooled over 3 rounds. Without this the candidate carries ~sqrt(3) more - # standard error than the floor it is judged against. - # Per-target override - search/validation keep the global n_attempts (1). + # Match the three-round procedure used to estimate the pinned baseline. n_attempts: 3 aggregate_attempts: mean @@ -66,32 +58,17 @@ score_baseline: false rescore_top_k: 3 rescore_attempts: 1 -# GAIA is multimodal: 5 of the 66 held-out tasks send image inputs, and a -# text-only target 400s on them ("This model does not support image inputs"), -# which reads as ordinary agent failure and caps achievable reward near 0.92. -# Unprefixed on purpose -- the agent sends model_name.removeprefix("openai/"), -# so an openai/-prefixed name here would be allow-listed in one form and -# requested in another, and the gateway would deny it. +# GAIA includes image inputs, so the target must be multimodal. Keep this name +# aligned with the exact value sent by the agent. model: gpt-5.4-mini -environment_name: ${inner_env:-modal} -# inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default +environment_name: ${inner_env:?set inner_env to a Harbor environment} +# Group evaluation sandboxes under one app and reclaim idle ones; drop these +# for an environment that does not accept them. extra_harbor_args: ["--ek", "app_name=harness-opt-bench", "--ek", "sandbox_idle_timeout_secs=3600"] harbor_python_version: "3.12" n_attempts: 1 -# Raised from 1 on 2026-07-29. This field exists precisely so "a transient -# upstream rate-limit storm during a mandatory evaluation does not fail -# otherwise-good candidates" (see HarborBuildConfig), and it was set to the -# least aggressive value. Measured that night: the shared Fireworks per-minute -# bucket rejected 78,748 requests across one officeqa pass, sustaining -# ~900-1,000/min for 24 minutes, and SEVEN rejections spread over ten minutes -# were enough to destroy a 37-case evaluation. The retry stack during search is -# only the target agent's own SDK (max_retries=8, ~1 min of jittered backoff, -# and candidate-controlled so it cannot be relied on) plus this one; the -# infrastructure_max_attempts retry below applies to trusted finalization only. -# 4 retries at multiplier 2 from a 4s floor, capped at 120s, gives roughly four -# more minutes of ride-out per case. That covers the bursts, though not a -# 24-minute saturation -- for which the answer is fewer concurrent runs or a -# larger provider quota, not more retrying. +# Retry transient request failures during an evaluation. Whole-evaluation +# retries remain limited to trusted final scoring. max_retries: 4 retry_max_wait_seconds: 120 infrastructure_max_attempts: 3 @@ -100,94 +77,59 @@ aggregate_attempts: best feedback_transcripts: true feedback_max_bytes: 16000 expose_attempt_detail: false -# Unreachable: worst case is ceil(198/24) x 600 = 5400s, every -# finalize trial (66 held-out x n_attempts=3) hitting its own cap. Assumes -# max_concurrency=24; recompute if that drops. +# Above the 5,400-second worst case at the configured concurrency. timeout_seconds: 7200 -# Exactly the dataset's declared [agent] timeout_sec, so vero's derived -# --agent-timeout-multiplier is 1.0 and the target agent gets precisely the -# clock the benchmark intends. Harbor times agent setup, environment build and -# verification on separate clocks with separate multipliers, so none of them -# eat into this budget and no buffer is warranted. +# Match the dataset's declared agent limit. case_timeout_seconds: 600 task_agent_timeout_seconds: 600 -max_concurrency: 24 # 8 -> 24; see officeqa for the measured headroom argument +max_concurrency: 24 error_rate_threshold: 0.1 -# Unreachable: worst-case finalize (5400) + worst-case rescore_top_k=3 -# validation rescore (5400). A verifier timeout loses the score outright. +# Covers final scoring plus validation rescoring. verifier_timeout_seconds: 14400 -# Limits on the OUTER optimizer trial: the Modal sandbox the optimizer itself runs -# in, not the nested evaluation. Every value reproduces what ran implicitly before -# 2026-09, now written down so the task states its own bounds. -# sandbox timeout: Harbor's Modal default of 24 h. When it fires the sandbox is -# destroyed with no verifier and no archive, so the trial is lost. +# Limits on the outer optimizer trial. The agent timeout leaves time for final +# scoring before the hard sandbox deadline. optimizer_sandbox_timeout_seconds: 86400 -# idle timeout: reclaims a sandbox whose optimizer died with no running command. -# The inner evaluation app has used 3600 since July; the outer sandbox had none. optimizer_sandbox_idle_timeout_seconds: 3600 -# agent clock: Harbor stops the optimizer here and STILL runs the verifier, so a -# runaway is scored and archived instead of lost at the sandbox timeout. Clean -# cells finish in 8-17 h; the longest finalization seen was ~3.5 h, which is -# what the 4 h gap to the sandbox timeout is for. optimizer_agent_timeout_seconds: 72000 -# Harbor's default resolves to public internet; stated rather than inherited. optimizer_allow_internet: true -# optimizer_cpus / optimizer_memory_mb / optimizer_storage_mb are left unset on -# purpose: that is Modal's default (0.125-core / 128 MiB reservation, bursting to -# the host), which is what every August cell ran under. Set them once the first -# rebuttal wave has been read off the Modal dashboard. + +# Credentials your execution and telemetry services need. Edit to match your +# deployment; each name must be set in the env file. secrets: - MODAL_TOKEN_ID - MODAL_TOKEN_SECRET - - MODAL_ENVIRONMENT # which Modal environment the sandboxes land in; unset = workspace default + - MODAL_ENVIRONMENT - WANDB_API_KEY - - WANDB_BASE_URL # self-hosted or cloud W&B + - WANDB_BASE_URL + +# Telemetry (optional). Remove this block to run without it. +wandb: + project: harness-engineering-bench + group: gaia + name: ${wandb_run:-gaia-shell} + tags: [gaia, shell-seed] + log_traces: true -# candidate harness runs as an unprivileged uid, unable to read held-out state. +# Keep the candidate isolated from held-out state. harness_user: harness -# Optimizer-agent env (forwarded to the harbor claude-code agent as --ae KEY=VALUE). -# Claude Code's Bash tool caps a single call at BASH_MAX_TIMEOUT_MS (default -# 600000=10min), well under one inner eval, which pushed the officeqa optimizer -# into --detach + background-poll + end-turn -- and a headless --print run is -# never re-woken, so the search died there. Raise the cap so a whole eval fits in -# one blocking call. The background-task vars are defence in depth only: they gate -# *automatic* backgrounding and do NOT remove the Bash tool's run_in_background -# parameter, which the model can still choose. The instruction forbids that. +# Allow one full evaluation to complete in a foreground tool call. agent_env: - # Above this benchmark's widest single eval: a full validation pass is - # ceil(66/24) x 600 = 1800s worst case. BASH_MAX_TIMEOUT_MS: "3600000" - BASH_DEFAULT_TIMEOUT_MS: "3600000" # same as max: an un-timed eval must still block + BASH_DEFAULT_TIMEOUT_MS: "3600000" ENABLE_BACKGROUND_TASKS: "0" FORCE_AUTO_BACKGROUND_TASKS: "0" - # Harnesses installed with `uv tool install` (mini-swe-agent, swe-agent) - # default to symlinking their entry point into /usr/local/bin, which the - # unprivileged optimizer user cannot write: "Failed to install executable - # ... Permission denied". npm/nvm-based harnesses (claude-code, opencode) - # are unaffected, so this only bites when the harness changes. UV_TOOL_BIN_DIR: "/home/agent/.local/bin" -wandb: - project: harness-engineering-bench # one project for the whole suite - group: gaia # keeps the benchmark distinguishable in the shared project - name: ${wandb_run:-gaia-shell} # per-launch label, e.g. --param wandb_run=gaia__opencode - tags: [gaia, shell-seed] - log_traces: true - inference_gateway: upstream_api_key_env: OPENAI_API_KEY upstream_base_url_env: OPENAI_BASE_URL - # Stamp request-log records with a thread_id so per_trial_tokens.py can attribute - # gateway token usage to individual trials (trusted, vs. content-matching). - # Without it the fallback recovers only each conversation's root turn: measured - # 0-13% coverage unstamped against 90-98% stamped, even with --tasks-dir. + # Enables reliable per-trial usage attribution. request_log_attribution: true producer: - allowed_models: ["${optimizer_model:-openai/gpt-5.4}"] + allowed_models: ["${optimizer_model:-gpt-5.4}"] max_concurrency: 8 - # See officeqa/baseline/build.yaml for the sizing rationale: the case budget is - # the spend control, so a token cap only needs to stop a runaway. + # Case budgets control normal use; token caps stop runaway requests. evaluation: allowed_models: [gpt-5.4-mini] max_requests: 200000 diff --git a/harness-opt-bench/gaia/baseline/build.yaml b/harness-opt-bench/gaia/baseline/build.yaml index a9c705ec..d9dac327 100644 --- a/harness-opt-bench/gaia/baseline/build.yaml +++ b/harness-opt-bench/gaia/baseline/build.yaml @@ -6,18 +6,14 @@ agent_repo: target task_source: gaia/gaia@sha256:bbc356f476e0b70ba77da11a9be7d6345918d1e4a2daade0d6dfb82ee6f7b761 task_manifest: ../partitions/manifest.json agent_import_path: gaia_agent.agent:GaiaAgent -harbor_requirement: harbor[modal]==0.20.0 -# Set once the vero on PyPI carries the compiled-task changes this branch relies on -# (env-delivered scope tokens, outer-trial limits); until then the images copy the -# local source. With it set, the compiled task holds no vero and can be committed. -# vero_requirement: scaleapi-vero==0.6.0 +# Include the extra your evaluation environment needs. +harbor_requirement: ${harbor_requirement:-harbor[modal]==0.20.0} partition_files: development: ../partitions/development.json validation: ../partitions/validation.json test: ../partitions/test.json -# total_cases = 4 full passes per partition (dev 33*4, validation 66*4). agent_access: - partition: development disclosure: full @@ -38,11 +34,7 @@ targets: baseline_reward: 0.6205 # re-pinned 0.554 / 0.625 / 0.682 (sd 0.0524); was 0.5736. See runs/BASELINES.md failure_value: 0.0 max_attempts: 1 - # The held-out eval is noisy: score the selected candidate 3x per case and - # average, so the final reward is comparable to the pinned baseline, which was - # itself pooled over 3 rounds. Without this the candidate carries ~sqrt(3) more - # standard error than the floor it is judged against. - # Per-target override - search/validation keep the global n_attempts (1). + # Match the three-round procedure used to estimate the pinned baseline. n_attempts: 3 aggregate_attempts: mean @@ -57,32 +49,17 @@ score_baseline: false rescore_top_k: 3 rescore_attempts: 1 -# GAIA is multimodal: 5 of the 66 held-out tasks send image inputs, and a -# text-only target 400s on them ("This model does not support image inputs"), -# which reads as ordinary agent failure and caps achievable reward near 0.92. -# Unprefixed on purpose -- the agent sends model_name.removeprefix("openai/"), -# so an openai/-prefixed name here would be allow-listed in one form and -# requested in another, and the gateway would deny it. +# GAIA includes image inputs, so the target must be multimodal. Keep this name +# aligned with the exact value sent by the agent. model: gpt-5.4-mini -environment_name: ${inner_env:-modal} -# inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default +environment_name: ${inner_env:?set inner_env to a Harbor environment} +# Group evaluation sandboxes under one app and reclaim idle ones; drop these +# for an environment that does not accept them. extra_harbor_args: ["--ek", "app_name=harness-opt-bench", "--ek", "sandbox_idle_timeout_secs=3600"] harbor_python_version: "3.12" n_attempts: 1 -# Raised from 1 on 2026-07-29. This field exists precisely so "a transient -# upstream rate-limit storm during a mandatory evaluation does not fail -# otherwise-good candidates" (see HarborBuildConfig), and it was set to the -# least aggressive value. Measured that night: the shared Fireworks per-minute -# bucket rejected 78,748 requests across one officeqa pass, sustaining -# ~900-1,000/min for 24 minutes, and SEVEN rejections spread over ten minutes -# were enough to destroy a 37-case evaluation. The retry stack during search is -# only the target agent's own SDK (max_retries=8, ~1 min of jittered backoff, -# and candidate-controlled so it cannot be relied on) plus this one; the -# infrastructure_max_attempts retry below applies to trusted finalization only. -# 4 retries at multiplier 2 from a 4s floor, capped at 120s, gives roughly four -# more minutes of ride-out per case. That covers the bursts, though not a -# 24-minute saturation -- for which the answer is fewer concurrent runs or a -# larger provider quota, not more retrying. +# Retry transient request failures during an evaluation. Whole-evaluation +# retries remain limited to trusted final scoring. max_retries: 4 retry_max_wait_seconds: 120 infrastructure_max_attempts: 3 @@ -91,94 +68,59 @@ aggregate_attempts: best feedback_transcripts: true feedback_max_bytes: 16000 expose_attempt_detail: false -# Unreachable: worst case is ceil(198/24) x 600 = 5400s, every -# finalize trial (66 held-out x n_attempts=3) hitting its own cap. Assumes -# max_concurrency=24; recompute if that drops. +# Above the 5,400-second worst case at the configured concurrency. timeout_seconds: 7200 -# Exactly the dataset's declared [agent] timeout_sec, so vero's derived -# --agent-timeout-multiplier is 1.0 and the target agent gets precisely the -# clock the benchmark intends. Harbor times agent setup, environment build and -# verification on separate clocks with separate multipliers, so none of them -# eat into this budget and no buffer is warranted. +# Match the dataset's declared agent limit. case_timeout_seconds: 600 task_agent_timeout_seconds: 600 -max_concurrency: 24 # 8 -> 24; see officeqa for the measured headroom argument +max_concurrency: 24 error_rate_threshold: 0.1 -# Unreachable: worst-case finalize (5400) + worst-case rescore_top_k=3 -# validation rescore (5400). A verifier timeout loses the score outright. +# Covers final scoring plus validation rescoring. verifier_timeout_seconds: 14400 -# Limits on the OUTER optimizer trial: the Modal sandbox the optimizer itself runs -# in, not the nested evaluation. Every value reproduces what ran implicitly before -# 2026-09, now written down so the task states its own bounds. -# sandbox timeout: Harbor's Modal default of 24 h. When it fires the sandbox is -# destroyed with no verifier and no archive, so the trial is lost. +# Limits on the outer optimizer trial. The agent timeout leaves time for final +# scoring before the hard sandbox deadline. optimizer_sandbox_timeout_seconds: 86400 -# idle timeout: reclaims a sandbox whose optimizer died with no running command. -# The inner evaluation app has used 3600 since July; the outer sandbox had none. optimizer_sandbox_idle_timeout_seconds: 3600 -# agent clock: Harbor stops the optimizer here and STILL runs the verifier, so a -# runaway is scored and archived instead of lost at the sandbox timeout. Clean -# cells finish in 8-17 h; the longest finalization seen was ~3.5 h, which is -# what the 4 h gap to the sandbox timeout is for. optimizer_agent_timeout_seconds: 72000 -# Harbor's default resolves to public internet; stated rather than inherited. optimizer_allow_internet: true -# optimizer_cpus / optimizer_memory_mb / optimizer_storage_mb are left unset on -# purpose: that is Modal's default (0.125-core / 128 MiB reservation, bursting to -# the host), which is what every August cell ran under. Set them once the first -# rebuttal wave has been read off the Modal dashboard. + +# Credentials your execution and telemetry services need. Edit to match your +# deployment; each name must be set in the env file. secrets: - MODAL_TOKEN_ID - MODAL_TOKEN_SECRET - - MODAL_ENVIRONMENT # which Modal environment the sandboxes land in; unset = workspace default + - MODAL_ENVIRONMENT - WANDB_API_KEY - - WANDB_BASE_URL # self-hosted or cloud W&B + - WANDB_BASE_URL + +# Telemetry (optional). Remove this block to run without it. +wandb: + project: harness-engineering-bench + group: gaia + name: ${wandb_run:-gaia} + tags: [gaia] + log_traces: true -# candidate harness runs as an unprivileged uid, unable to read held-out state. +# Keep the candidate isolated from held-out state. harness_user: harness -# Optimizer-agent env (forwarded to the harbor claude-code agent as --ae KEY=VALUE). -# Claude Code's Bash tool caps a single call at BASH_MAX_TIMEOUT_MS (default -# 600000=10min), well under one inner eval, which pushed the officeqa optimizer -# into --detach + background-poll + end-turn -- and a headless --print run is -# never re-woken, so the search died there. Raise the cap so a whole eval fits in -# one blocking call. The background-task vars are defence in depth only: they gate -# *automatic* backgrounding and do NOT remove the Bash tool's run_in_background -# parameter, which the model can still choose. The instruction forbids that. +# Allow one full evaluation to complete in a foreground tool call. agent_env: - # Above this benchmark's widest single eval: a full validation pass is - # ceil(66/24) x 600 = 1800s worst case. BASH_MAX_TIMEOUT_MS: "3600000" - BASH_DEFAULT_TIMEOUT_MS: "3600000" # same as max: an un-timed eval must still block + BASH_DEFAULT_TIMEOUT_MS: "3600000" ENABLE_BACKGROUND_TASKS: "0" FORCE_AUTO_BACKGROUND_TASKS: "0" - # Harnesses installed with `uv tool install` (mini-swe-agent, swe-agent) - # default to symlinking their entry point into /usr/local/bin, which the - # unprivileged optimizer user cannot write: "Failed to install executable - # ... Permission denied". npm/nvm-based harnesses (claude-code, opencode) - # are unaffected, so this only bites when the harness changes. UV_TOOL_BIN_DIR: "/home/agent/.local/bin" -wandb: - project: harness-engineering-bench # one project for the whole suite - group: gaia # keeps the benchmark distinguishable in the shared project - name: ${wandb_run:-gaia} # per-launch label, e.g. --param wandb_run=gaia__opencode - tags: [gaia] - log_traces: true - inference_gateway: upstream_api_key_env: OPENAI_API_KEY upstream_base_url_env: OPENAI_BASE_URL - # Stamp request-log records with a thread_id so per_trial_tokens.py can attribute - # gateway token usage to individual trials (trusted, vs. content-matching). - # Without it the fallback recovers only each conversation's root turn: measured - # 0-13% coverage unstamped against 90-98% stamped, even with --tasks-dir. + # Enables reliable per-trial usage attribution. request_log_attribution: true producer: - allowed_models: ["${optimizer_model:-openai/gpt-5.4}"] + allowed_models: ["${optimizer_model:-gpt-5.4}"] max_concurrency: 8 - # See officeqa/baseline/build.yaml for the sizing rationale: the case budget is - # the spend control, so a token cap only needs to stop a runaway. + # Case budgets control normal use; token caps stop runaway requests. evaluation: allowed_models: [gpt-5.4-mini] max_requests: 200000 diff --git a/harness-opt-bench/gaia/baseline/compiled.manifest.json b/harness-opt-bench/gaia/baseline/compiled.manifest.json index bb97771d..c3ab2e86 100644 --- a/harness-opt-bench/gaia/baseline/compiled.manifest.json +++ b/harness-opt-bench/gaia/baseline/compiled.manifest.json @@ -73,7 +73,7 @@ "environment/agent-seed/uv.lock": "5e94e388c3050cf90b59b08cbe55030dd102e30475c171415a190da9969cf2c2", "environment/docker-compose.yaml": "1e9d9c8496de0d73ff5bcbaa05a1fcaa4eb753ccff9933388a30cc25c3b1bba1", "environment/gateway/Dockerfile": "70d149f1928b135b8e11b7bb8d4ef70d81f331a6766bb3947192ff876e2eea26", - "environment/gateway/config.json": "52f4f8064ce5a310cf4a7cbec7d067233fea68e26432f610f0d59660cb4898dd", + "environment/gateway/config.json": "cc94dbbf9429344e1383573117b8e2c68f8a58c9451a841fe696478568b1a168", "environment/gateway/launch.json": "462bcc8d2749f1364d79e14d36be7144a84441b6bb74c985f798115ef26636ae", "environment/main/seed.sh": "f51b70ef8fbc646ab6f129d2d7f8f3a72a1c37a9331691dcbd4f9b899dc7bc90", "environment/overlay/skills/evals/SKILL.md": "32cd544a37264d478b01985f11178f6476b093f6c6b2f682d4495b43a14e28cc", @@ -81,12 +81,12 @@ "environment/sidecar/cases/development.jsonl": "ef9a63afb40f7f08d9b95995857c5f81eb2f817006d14176cfb3e8676f688555", "environment/sidecar/cases/test.jsonl": "9cf01d06d194f4e7de3257189b7629dea6d33c572a913d98147ec3b05c338044", "environment/sidecar/cases/validation.jsonl": "292ddd833d0a584c648349dcea0eca5c8dec170fe596110df6a300935d0e3c1e", - "environment/sidecar/serve.json": "150b4d1a683f4f023a679c55514dbff77657d048e850ce7a409985eda75cf424", - "environment/vero/": "e12fbae407e3bf6acfaa6445e5debc683592b66180ab69b6e6eff2b395f93f30", + "environment/sidecar/serve.json": "7b4847bfed41ba938fdef05d93365a7cdfd9f7864333b9712b1def207b7d0c00", + "environment/vero/": "b4b959797b70e9ae9583effc256dedfc5219e3b165aabfbd51a0c979f09e8818", "instruction.md": "ee539ac13ad8bdcaf5c3c2bd7465eb44db72001cb7181793c02fe99d440d88f6", "solution/solve.sh": "10923f62ac228055ab21388df27d9131a457bb51bafbb883cfd425e3c7d0a4ad", "task.toml": "664b022bed341b1e41e8e657553ed942cb3611bbdf7e5a278e6fac2093665e38", "tests/test.sh": "b65e90e6d67aacedd559c6b1c9d0f1d5c5071ecc9178a5bd7f9e1066f332bb0d" }, - "vero_commit": "96e57cf22e5b7f3a4e8cea9ade61bf5f61c66692" + "vero_commit": "09deba52bd57913f86795423dd54c9ad5970d8ea" } diff --git a/harness-opt-bench/gaia/baseline/secrets.env.example b/harness-opt-bench/gaia/baseline/secrets.env.example index 33bc5a52..2f0fb1b6 100644 --- a/harness-opt-bench/gaia/baseline/secrets.env.example +++ b/harness-opt-bench/gaia/baseline/secrets.env.example @@ -2,14 +2,15 @@ # Copy to `secrets.env` (gitignored) and fill in. File values override the # ambient shell and never appear on the command line. -# Upstream inference credentials. VeRO reads these as the real provider key/URL -# (per the compiled gateway's *_source fields), relocates them to the -# VERO_INFERENCE_UPSTREAM_* target vars, and hands the optimizer a scoped -# producer token instead — so the raw key never reaches the coding agent. -OPENAI_API_KEY=sk-your-upstream-inference-key -OPENAI_BASE_URL=https://your-openai-compatible-endpoint/v1 +# Inference endpoint used by the gateway. Add any execution-environment +# credentials required by your Harbor installation. +OPENAI_API_KEY= +OPENAI_BASE_URL= -# Modal credentials for the eval sidecar backends (declared required by the -# compiled task.toml). Copy from your ~/.modal.toml profile. -MODAL_TOKEN_ID=ak-... -MODAL_TOKEN_SECRET=as-... +# Execution environment and telemetry, as declared under `secrets` in build.yaml. +# Edit both together if your deployment uses different services. +MODAL_TOKEN_ID= +MODAL_TOKEN_SECRET= +MODAL_ENVIRONMENT= +WANDB_API_KEY= +WANDB_BASE_URL= diff --git a/harness-opt-bench/gaia/partitions-e2e/README.md b/harness-opt-bench/gaia/partitions-e2e/README.md index 85873c84..2c43b115 100644 --- a/harness-opt-bench/gaia/partitions-e2e/README.md +++ b/harness-opt-bench/gaia/partitions-e2e/README.md @@ -1,7 +1,8 @@ -# e2e smoke partitions +# GAIA smoke-test split -Two development, three validation and three test cases, the first entries of the -real GAIA shell partitions. Used only by `baseline/build.shell.e2e.yaml`, whose -purpose is to exercise the whole optimizer trial (compile, gateway, evaluations, -agent clock, finalization, session archive) in well under an hour. Scores from it -mean nothing and are never reported. +This split contains two development, three validation, and three test cases from +the full GAIA shell split. + +It is used only by `baseline/build.shell.e2e.yaml` to exercise compilation, +evaluation, final scoring, and artifact creation quickly. Its scores are not +benchmark results and must not be reported. diff --git a/harness-opt-bench/officeqa/README.md b/harness-opt-bench/officeqa/README.md new file mode 100644 index 00000000..30545c55 --- /dev/null +++ b/harness-opt-bench/officeqa/README.md @@ -0,0 +1,69 @@ +# OfficeQA + +OfficeQA evaluates a grounded question-answering agent over the Treasury +Bulletin corpus. The agent searches local documents and writes one exact answer. + +## At a glance + +| Item | Value | +| --- | --- | +| Cases | 246 | +| Development / validation / test | 49 / 98 / 99 | +| Editable harness | `baseline/target/` | +| Target model | `deepseek-v4-flash` | +| Pinned seed baseline | 0.3412 ± 0.0330 | +| Scoring | Numeric or text match, with 1% numeric tolerance | + +Development exposes complete task results and corpus resources. Validation is +aggregate-only, and test is held out for final scoring. + +## Prepare the tasks + +OfficeQA task directories are too large to store in this repository. Fetch the +pinned dataset into the ignored `tasks/` directory: + +~~~bash +cd harness-opt-bench/officeqa +bash scripts/vendor_tasks.sh +~~~ + +The script fetches only the OfficeQA subset, verifies that all 246 tasks are +present, and adds the canonical task names required by local staging. + +Check task availability from the repository root: + +~~~bash +python3 harness-opt-bench/scripts/task_data.py --check +~~~ + +## Editable target + +`OfficeQaAgent` can search the corpus with shell commands, inspect images, and +submit an answer. The optimizer may change its prompts, tools, control flow, +context handling, and dependencies. The corpus, partitions, target model, +verifier, and scoring policy remain fixed. + +## Compile + +From the repository root: + +~~~bash +cd vero +VERO_SKIP_SECRET_CHECK=1 uv run vero harbor build \ + --config ../harness-opt-bench/officeqa/baseline/build.yaml \ + --param inner_env= \ + --output +~~~ + +The `VERO_SKIP_SECRET_CHECK` setting is appropriate only for compile-time +validation. For a real optimization run, follow the shared +[`run-benchmark` guide](../skills/run-benchmark/SKILL.md). + +## Where to look + +| Path | Contents | +| --- | --- | +| `baseline/build.yaml` | Evaluation configuration | +| `baseline/target/` | Editable seed agent | +| `partitions/` | Committed split and task manifest | +| `scripts/vendor_tasks.sh` | Dataset preparation | diff --git a/harness-opt-bench/officeqa/baseline/build.yaml b/harness-opt-bench/officeqa/baseline/build.yaml index f66a1a67..d24e0b8c 100644 --- a/harness-opt-bench/officeqa/baseline/build.yaml +++ b/harness-opt-bench/officeqa/baseline/build.yaml @@ -3,16 +3,13 @@ description: >- Improve a grounded-reasoning agent on OfficeQA (Treasury Bulletin corpus QA). The agent must search /app/corpus/ and write the exact answer to /app/answer.txt; scoring is rule-based (numeric/text match with 1% tolerance). -# Local (vendored) dataset — OfficeQA is not in the Harbor hub, only the -# harbor-datasets git repo. task_source points at the vendored task dirs; the -# per-task task.toml carries docker_image=officeqa-corpus (full-corpus mode), so -# extra_harbor_args uses --no-force-build to pull that prebuilt image instead of -# building the guard Dockerfile. +# The task source is vendored locally and uses a prebuilt corpus image. agent_repo: target task_source: ../tasks task_manifest: ../partitions/manifest.json agent_import_path: officeqa_agent.agent:OfficeQaAgent -harbor_requirement: harbor[modal]==0.20.0 +# Include the extra your evaluation environment needs. +harbor_requirement: ${harbor_requirement:-harbor[modal]==0.20.0} partition_files: development: ../partitions/development.json @@ -39,9 +36,7 @@ targets: baseline_reward: 0.3412 # re-pinned 0.354 / 0.296 / 0.374 (sd 0.0330); was 0.3603. See runs/BASELINES.md failure_value: 0.0 max_attempts: 1 - # The held-out eval is noisy: score the selected candidate 3x per case and - # average, so the final reward is comparable to the K=3 pinned baseline. - # Per-target override — search/validation keep the global n_attempts (1). + # Match the three-round procedure used to estimate the pinned baseline. n_attempts: 3 aggregate_attempts: mean @@ -56,27 +51,16 @@ score_baseline: false rescore_top_k: 3 rescore_attempts: 1 -model: fireworks_ai/deepseek-v4-flash -environment_name: ${inner_env:-modal} -# --no-force-build pulls the prebuilt officeqa-corpus image (see task_source -# note above); the --ek groups inner sandboxes under one dedicated Modal app. +model: deepseek-v4-flash +environment_name: ${inner_env:?set inner_env to a Harbor environment} +# Pull the prebuilt corpus image described above. The remaining options group +# evaluation sandboxes under one app and reclaim idle ones; drop them for an +# environment that does not accept them. extra_harbor_args: ["--no-force-build", "--ek", "app_name=harness-opt-bench", "--ek", "sandbox_idle_timeout_secs=3600"] harbor_python_version: "3.12" n_attempts: 1 -# Raised from 1 on 2026-07-29. This field exists precisely so "a transient -# upstream rate-limit storm during a mandatory evaluation does not fail -# otherwise-good candidates" (see HarborBuildConfig), and it was set to the -# least aggressive value. Measured that night: the shared Fireworks per-minute -# bucket rejected 78,748 requests across one officeqa pass, sustaining -# ~900-1,000/min for 24 minutes, and SEVEN rejections spread over ten minutes -# were enough to destroy a 37-case evaluation. The retry stack during search is -# only the target agent's own SDK (max_retries=8, ~1 min of jittered backoff, -# and candidate-controlled so it cannot be relied on) plus this one; the -# infrastructure_max_attempts retry below applies to trusted finalization only. -# 4 retries at multiplier 2 from a 4s floor, capped at 120s, gives roughly four -# more minutes of ride-out per case. That covers the bursts, though not a -# 24-minute saturation -- for which the answer is fewer concurrent runs or a -# larger provider quota, not more retrying. +# Retry transient request failures during an evaluation. Whole-evaluation +# retries remain limited to trusted final scoring. max_retries: 4 retry_max_wait_seconds: 120 infrastructure_max_attempts: 3 @@ -85,111 +69,75 @@ aggregate_attempts: best feedback_transcripts: true feedback_max_bytes: 16000 expose_attempt_detail: false -# Unreachable: worst case is ceil(297/24) x 1800 = 23400s, every -# finalize trial (99 held-out x n_attempts=3) hitting its own cap. Assumes -# max_concurrency=24; recompute if that drops. +# Above the 23,400-second worst case at the configured concurrency. timeout_seconds: 28800 -# Exactly the dataset's declared [agent] timeout_sec, so vero's derived -# --agent-timeout-multiplier is 1.0 and the target agent gets precisely the -# clock the benchmark intends. Harbor times agent setup, environment build and -# verification on separate clocks with separate multipliers, so none of them -# eat into this budget and no buffer is warranted. +# Match the dataset's declared agent limit. case_timeout_seconds: 1800 task_agent_timeout_seconds: 1800 -# Raised 8 -> 24 to cut finalize wall time ~3x (297 trials / concurrency x the -# ~400s measured mean case wall: ~4.1h at 8, ~1.4h at 24 -- the mean, not the -# 1800s cap above). Derived headroom: run #2 sustained 16 case slots -# (2 detached evals x 8) at ~181k metered TPM/slot with upstream_errors=0, so 24 -# slots is ~1.5x a proven configuration. Watch upstream_errors in -# artifacts/inference/usage.json; if it stays 0, there is room to go further. max_concurrency: 24 error_rate_threshold: 0.1 -# Unreachable: worst-case finalize (23400) + worst-case rescore_top_k=3 -# validation rescore (23400). A verifier timeout loses the score outright. +# Covers final scoring plus validation rescoring. verifier_timeout_seconds: 54000 -# Limits on the OUTER optimizer trial: the Modal sandbox the optimizer itself runs -# in, not the nested evaluation. Every value reproduces what ran implicitly before -# 2026-09, now written down so the task states its own bounds. -# sandbox timeout: Harbor's Modal default of 24 h. When it fires the sandbox is -# destroyed with no verifier and no archive, so the trial is lost. +# Limits on the outer optimizer trial. The agent timeout leaves time for final +# scoring before the hard sandbox deadline. optimizer_sandbox_timeout_seconds: 86400 -# idle timeout: reclaims a sandbox whose optimizer died with no running command. -# The inner evaluation app has used 3600 since July; the outer sandbox had none. optimizer_sandbox_idle_timeout_seconds: 3600 -# agent clock: Harbor stops the optimizer here and STILL runs the verifier, so a -# runaway is scored and archived instead of lost at the sandbox timeout. Clean -# cells finish in 8-17 h; the longest finalization seen was ~3.5 h, which is -# what the 4 h gap to the sandbox timeout is for. optimizer_agent_timeout_seconds: 72000 -# Harbor's default resolves to public internet; stated rather than inherited. optimizer_allow_internet: true -# optimizer_cpus / optimizer_memory_mb / optimizer_storage_mb are left unset on -# purpose: that is Modal's default (0.125-core / 128 MiB reservation, bursting to -# the host), which is what every August cell ran under. Set them once the first -# rebuttal wave has been read off the Modal dashboard. + +# Credentials your execution and telemetry services need. Edit to match your +# deployment; each name must be set in the env file. secrets: - MODAL_TOKEN_ID - MODAL_TOKEN_SECRET - - MODAL_ENVIRONMENT # which Modal environment the sandboxes land in; unset = workspace default + - MODAL_ENVIRONMENT - WANDB_API_KEY - - WANDB_BASE_URL # self-hosted or cloud W&B + - WANDB_BASE_URL + +# Telemetry (optional). Remove this block to run without it. +wandb: + project: harness-engineering-bench + group: officeqa + name: ${wandb_run:-officeqa} + tags: [officeqa] + log_traces: true -# candidate harness runs as an unprivileged uid, unable to read held-out state. +# Keep the candidate isolated from held-out state. harness_user: harness -# Optimizer-agent env (forwarded to the harbor claude-code agent as --ae KEY=VALUE). -# Inner evals take 15-30 min; Claude Code's Bash tool caps a single call at -# BASH_MAX_TIMEOUT_MS (default 600000=10min), which forced the agent into -# --detach + background-poll + end-turn (it then ended its turn waiting for a -# notification that never re-wakes a headless --print run). Raise the cap so the -# agent can block on a whole eval in one Bash call. -# The background-task vars are kept for defence in depth but do NOT actually -# disable the Bash tool's run_in_background parameter -- the model can still -# choose it, and run #2 did. The instruction is what forbids it. +# Allow one full evaluation to complete in a foreground tool call. agent_env: - # Above this benchmark's widest single eval: a full validation pass is - # ceil(98/24) x 1800 = 9000s worst case. BASH_MAX_TIMEOUT_MS: "10800000" - BASH_DEFAULT_TIMEOUT_MS: "10800000" # same as max: an un-timed eval must still block - ENABLE_BACKGROUND_TASKS: "0" # gates auto-backgrounding only (see above) + BASH_DEFAULT_TIMEOUT_MS: "10800000" + ENABLE_BACKGROUND_TASKS: "0" FORCE_AUTO_BACKGROUND_TASKS: "0" - # Harnesses installed with `uv tool install` (mini-swe-agent, swe-agent) - # default to symlinking their entry point into /usr/local/bin, which the - # unprivileged optimizer user cannot write: "Failed to install executable - # ... Permission denied". npm/nvm-based harnesses (claude-code, opencode) - # are unaffected, so this only bites when the harness changes. UV_TOOL_BIN_DIR: "/home/agent/.local/bin" -wandb: - project: harness-engineering-bench - group: officeqa # keeps the benchmark distinguishable in the shared project - name: ${wandb_run:-officeqa} # per-launch label, e.g. --param wandb_run=officeqa__claude-sonnet-5 - tags: [officeqa] - log_traces: true - inference_gateway: upstream_api_key_env: OPENAI_API_KEY upstream_base_url_env: OPENAI_BASE_URL - # Stamp request-log records with a thread_id so per_trial_tokens.py can attribute - # gateway token usage to individual trials (trusted, vs. content-matching). + # Enables reliable per-trial usage attribution. request_log_attribution: true producer: - allowed_models: ["${optimizer_model:-openai/gpt-5.4}"] + allowed_models: ["${optimizer_model:-gpt-5.4}"] max_concurrency: 8 - # Token caps are a runaway backstop, not the spend control: the work is already - # bounded by the agent case budget above and by the fixed held-out set, so a cap - # that bites first only aborts authorized work. Sized at ~3M tokens per - # case-run, ~2.3x the worst measured cost (1.33M/case for an optimized - # candidate). ~90% of these tokens are cache reads, counted at full weight. + # Case budgets control normal use; token caps stop runaway requests. evaluation: - allowed_models: [fireworks_ai/deepseek-v4-flash] + allowed_models: [deepseek-v4-flash] + # Map the target model to a deployment; override per launch with + # --param target_model_route=. + model_aliases: + deepseek-v4-flash: ${target_model_route:-fireworks_ai/deepseek-v4-flash-0731} max_requests: 200000 max_tokens: 3000000000 # 588 agent case-runs (196 dev + 392 validation) max_concurrency: 64 - # Reserved so a search-phase overspend can never starve held-out scoring. This - # inherited evaluation's 100M, exhausted mid-finalize, and shipped reward 0.0. + # Keep final scoring capacity separate from search. finalization: - allowed_models: [fireworks_ai/deepseek-v4-flash] + allowed_models: [deepseek-v4-flash] + # Map the target model to a deployment; override per launch with + # --param target_model_route=. + model_aliases: + deepseek-v4-flash: ${target_model_route:-fireworks_ai/deepseek-v4-flash-0731} max_requests: 200000 max_tokens: 3000000000 # 297 finalize trials (99 test x3) + 294 rescore max_concurrency: 64 diff --git a/harness-opt-bench/officeqa/baseline/compiled.manifest.json b/harness-opt-bench/officeqa/baseline/compiled.manifest.json index 2012d76e..37e10082 100644 --- a/harness-opt-bench/officeqa/baseline/compiled.manifest.json +++ b/harness-opt-bench/officeqa/baseline/compiled.manifest.json @@ -73,7 +73,7 @@ "environment/agent-seed/uv.lock": "256b8b915d8a1b7a8cc19619fa96d2f32869d615e9fe1055686228c02f138658", "environment/docker-compose.yaml": "1e9d9c8496de0d73ff5bcbaa05a1fcaa4eb753ccff9933388a30cc25c3b1bba1", "environment/gateway/Dockerfile": "70d149f1928b135b8e11b7bb8d4ef70d81f331a6766bb3947192ff876e2eea26", - "environment/gateway/config.json": "f218803a48ce1f927020df0c7443e5229689985cfc88f5f715b2658bf89e0507", + "environment/gateway/config.json": "fc3b41461cbb0bb982a0f50489a459a3555f2ba0c60f1d7b4723d939777cf35c", "environment/gateway/launch.json": "462bcc8d2749f1364d79e14d36be7144a84441b6bb74c985f798115ef26636ae", "environment/main/seed.sh": "f51b70ef8fbc646ab6f129d2d7f8f3a72a1c37a9331691dcbd4f9b899dc7bc90", "environment/overlay/skills/evals/SKILL.md": "32cd544a37264d478b01985f11178f6476b093f6c6b2f682d4495b43a14e28cc", @@ -81,13 +81,13 @@ "environment/sidecar/cases/development.jsonl": "8c4df163f6fa4b495392552e3fa9da8a1f743486783888d5a9150a391dd06e75", "environment/sidecar/cases/test.jsonl": "d96a98c9ae504045f9cb2a8d7fb8fa1815528237bb68855d5e0fde8c6b031b81", "environment/sidecar/cases/validation.jsonl": "14d86ab3071fb4091efd7b80687b1b8d8176c6b76dc1676d9a72a27f9d99032f", - "environment/sidecar/serve.json": "081ca2451acfb2df010c4b749baa01d4bce4727f5ad7ae511b08a070059059fa", + "environment/sidecar/serve.json": "1b104eaae49398fd45f1a56c6bc4c0915bb1de680895ff30594ca8df93bc0ae6", "environment/sidecar/task-source/": "be9f7f935eac438753a93bc1dccb66dc82a47c2911a9befad507d8f1e4e0480b", - "environment/vero/": "e12fbae407e3bf6acfaa6445e5debc683592b66180ab69b6e6eff2b395f93f30", + "environment/vero/": "b4b959797b70e9ae9583effc256dedfc5219e3b165aabfbd51a0c979f09e8818", "instruction.md": "c9f995857987ecbf98ca9d1bb35dd3ecee240fc50c906cda2a464f75d879471b", "solution/solve.sh": "6472c890bec7a80fbbf5a07e232ee0aa33a2636ca34e919bb4083e470bf11630", "task.toml": "b96923cb61c2d71edd27ce027a00cbe1d535290285c8e5416c1b218ed8eb1ac6", "tests/test.sh": "b65e90e6d67aacedd559c6b1c9d0f1d5c5071ecc9178a5bd7f9e1066f332bb0d" }, - "vero_commit": "96e57cf22e5b7f3a4e8cea9ade61bf5f61c66692" + "vero_commit": "09deba52bd57913f86795423dd54c9ad5970d8ea" } diff --git a/harness-opt-bench/officeqa/baseline/target/src/officeqa_agent/agent.py b/harness-opt-bench/officeqa/baseline/target/src/officeqa_agent/agent.py index ce7a3af1..2cafb43c 100644 --- a/harness-opt-bench/officeqa/baseline/target/src/officeqa_agent/agent.py +++ b/harness-opt-bench/officeqa/baseline/target/src/officeqa_agent/agent.py @@ -14,12 +14,7 @@ from openai import AsyncOpenAI def _is_reasoning_model(model: str) -> bool: - """Whether `model` is an OpenAI reasoning model. - - Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects - reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served - open models match none of these prefixes, so they keep the legacy shape. - """ + """Whether `model` uses the reasoning-model request fields.""" name = model.lower() return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name @@ -200,10 +195,6 @@ def _completion_kwargs( kwargs["tools"] = TOOLS if _is_reasoning_model(self._api_model): kwargs["reasoning_effort"] = "medium" - # parallel_tool_calls is a separate axis: Fireworks-served models reject - # it, but gpt-4o supports it, so this one stays a provider check. - if "fireworks" not in self._api_model: - kwargs["parallel_tool_calls"] = False return kwargs def _account(self, usage: Any, totals: dict[str, int]) -> None: @@ -220,9 +211,7 @@ async def run( environment: BaseEnvironment, context: AgentContext, ) -> None: - # Stateless Chat Completions: the full message history is resent each - # turn (provider prompt-caching handles the repeated prefix), which - # works across every provider, unlike the OpenAI-only Responses API. + # The API is stateless, so resend the full message history each turn. messages: list[dict[str, Any]] = [ {"role": "system", "content": INSTRUCTIONS}, {"role": "user", "content": instruction}, diff --git a/harness-opt-bench/scripts/per_trial_tokens.py b/harness-opt-bench/scripts/per_trial_tokens.py index 8841c92a..95d9b814 100755 --- a/harness-opt-bench/scripts/per_trial_tokens.py +++ b/harness-opt-bench/scripts/per_trial_tokens.py @@ -18,8 +18,8 @@ the envelope. Coverage note: on **stamped** logs (``request_log.attribution: true``) every -turn inherits its conversation's thread_id, so stateful APIs (OpenAI -responses' ``previous_response_id``) attribute fully. On **legacy** logs the +turn inherits its conversation's thread_id, so stateful APIs that use a +``previous_response_id`` attribute fully. On **legacy** logs the fallback recovers only each conversation's root turn — chained follow-ups with empty bodies are unrecoverable and land in the residual. Enable attribution at build time for complete coverage. diff --git a/harness-opt-bench/scripts/rescore_candidate.py b/harness-opt-bench/scripts/rescore_candidate.py index 057abde7..71372153 100644 --- a/harness-opt-bench/scripts/rescore_candidate.py +++ b/harness-opt-bench/scripts/rescore_candidate.py @@ -19,7 +19,7 @@ cd vero && uv run python \\ ../harness-opt-bench/scripts/rescore_candidate.py \\ - --session ../runs/officeqa/claude-sonnet-5-run2/jobs/*/task__*/verifier/session.tar.gz \\ + --session ../runs/officeqa/example-run/jobs/*/task__*/verifier/session.tar.gz \\ --benchmark officeqa --cases 2 --rounds 1 Aggregation matches runs/recompute.py exactly: pooled mean over every scored @@ -57,10 +57,21 @@ def load_build(benchmark: str) -> tuple[dict, Path]: return yaml.safe_load(path.read_text()), path -def resolve_param(value: str) -> str: - """Resolve a `${name:-default}` placeholder to its default.""" - match = re.fullmatch(r"\$\{[^:}]+:-([^}]*)\}", str(value)) - return match.group(1) if match else str(value) +def resolve_param(value: str, params: dict[str, str] | None = None) -> str: + """Resolve a `${name:-default}` or `${name:?message}` placeholder. + + `--param name=value` supplies the value; a `:?` placeholder with no value is + an error, as it is for `vero harbor run`. + """ + match = re.fullmatch(r"\$\{([^:}]+)(?::([-?])([^}]*))?\}", str(value)) + if not match: + return str(value) + name, kind, rest = match.groups() + if params and name in params: + return params[name] + if kind == "-": + return rest + sys.exit(f"build parameter {name!r} is unset: pass --param {name}=") def open_session(session: str, workdir: Path) -> Path: @@ -129,6 +140,17 @@ def extract_candidate(session_dir: Path, version: str, dest: Path) -> None: log(f"extracted candidate {version[:12]} -> {dest}") +def routed_model(build: dict, model: str, params: dict[str, str]) -> str: + """The deployment the gateway would route `model` to, or `model` itself. + + A build names the target model without a provider and maps it to a + deployment under the evaluation scope's `model_aliases`. This runs harbor + against the upstream directly, so apply the same mapping here. + """ + aliases = (build.get("inference_gateway") or {}).get("evaluation", {}).get("model_aliases") or {} + return resolve_param(aliases.get(model, model), params) + + def harbor_command( *, build: dict, @@ -139,6 +161,7 @@ def harbor_command( attempts: int, concurrency: int, model: str, + params: dict[str, str], ) -> list[str]: """Mirror vero/src/vero/harbor/backend.py::_command and the baseline runs. @@ -158,11 +181,11 @@ def harbor_command( "--python", str(build.get("harbor_python_version", "3.12")), "--no-config", "--no-env-file", "--project", str(workspace), - "--with", build.get("harbor_requirement", "harbor[modal]==0.20.0"), + "--with", build.get("harbor_requirement", "harbor==0.20.0"), "harbor", "run", "--yes", *source_args, "--agent-import-path", build["agent_import_path"], - "-e", resolve_param(build.get("environment_name", "modal")), + "-e", resolve_param(build["environment_name"], params), "-m", str(model), "-n", str(concurrency), "--n-attempts", str(attempts), @@ -223,9 +246,12 @@ def main() -> int: parser.add_argument("--attempts", type=int, default=1, help="attempts per case within a round (default 1)") parser.add_argument("--concurrency", type=int, default=24) + parser.add_argument("--param", action="append", default=[], metavar="NAME=VALUE", + help="build parameter, e.g. inner_env=modal (repeatable)") parser.add_argument("--output", help="output dir (default: a temp dir)") parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() + params = dict(item.split("=", 1) for item in args.param) build, build_path = load_build(args.benchmark) outdir = Path(args.output).resolve() if args.output else Path( @@ -262,8 +288,8 @@ def main() -> int: log(f"{len(tasks)} {args.partition} case(s), {args.rounds} round(s), " f"{args.attempts} attempt(s)/case, concurrency {args.concurrency}") - # Some tasks reference the base URL under litellm's alias rather than - # OPENAI_BASE_URL -- swe-atlas-qna's rubric judge declares + # Some tasks reference the base URL under an alternate compatibility name. + # swe-atlas-qna's rubric judge declares # `EVAL_BASE_URL = "${OPENAI_API_BASE}"` in [verifier.env], and harbor aborts # the whole job with "Missing Environment Variables" before running a single # trial if it is unset. Mirror it so a benchmark's own judge can start. @@ -284,7 +310,8 @@ def main() -> int: command = harbor_command( build=build, build_path=build_path, workspace=workspace, tasks=tasks, jobs_dir=jobs_dir, attempts=args.attempts, concurrency=args.concurrency, - model=args.model or build["model"], + model=routed_model(build, args.model or build["model"], params), + params=params, ) if args.dry_run: print(" ".join(command)) diff --git a/harness-opt-bench/secrets.env.example b/harness-opt-bench/secrets.env.example index 534664f2..1a4a66e4 100644 --- a/harness-opt-bench/secrets.env.example +++ b/harness-opt-bench/secrets.env.example @@ -1,31 +1,16 @@ -# Template for a run's credential file. Copy to a name matching *.secrets.env -# (root .gitignore covers `secrets.env` and `*.secrets.env`; a name like -# `secrets.2.env` would NOT be ignored) and fill in real values: -# -# cp harness-engineering-bench/secrets.env.example vero/fireworks-a.secrets.env -# vero harbor run ... --env-file fireworks-a.secrets.env -# -# Why more than one: the binding rate limit is per LiteLLM key (10M TPM, 5000 -# RPM), and one benchmark run peaks around 2.9M TPM at max_concurrency 24. So a -# key carries roughly three concurrent runs, and adding keys scales the suite's -# parallelism proportionally. A distinct file per run also keeps spend -# attributable. See the concurrency note (§) in CONFIGURATION.md. -# -# Never commit a filled-in copy. Every value here is a live credential. +# Copy this file to a git-ignored location and fill in the values required by +# your deployment. Never commit a populated credential file. -# Upstream inference the gateway proxies to. The optimizer and the target agent -# never see this key; the gateway mints per-scope tokens instead. +# Inference endpoint used by the gateway. OPENAI_API_KEY= OPENAI_BASE_URL= -# Modal runs the inner evaluation sandboxes. Parallel runs share one Modal -# workspace unless these differ, and sandbox capacity is its own ceiling -# separate from any inference quota. +# Add execution-environment credentials required by your Harbor installation. + +# Execution environment and telemetry, as declared under `secrets` in build.yaml. +# Edit both together if your deployment uses different services. MODAL_TOKEN_ID= MODAL_TOKEN_SECRET= - -# Telemetry. One key is fine across concurrent runs — they are distinguished by -# `--param wandb_run=`, all landing in the shared harness-engineering-bench -# project. +MODAL_ENVIRONMENT= WANDB_API_KEY= WANDB_BASE_URL= diff --git a/harness-opt-bench/skills/run-benchmark/SKILL.md b/harness-opt-bench/skills/run-benchmark/SKILL.md index 433b774f..fffd39be 100644 --- a/harness-opt-bench/skills/run-benchmark/SKILL.md +++ b/harness-opt-bench/skills/run-benchmark/SKILL.md @@ -1,255 +1,163 @@ --- name: run-benchmark description: >- - Run one HarnessOpt-Bench optimization end-to-end (compile, inference gateway, - optimizer agent, sandboxed evaluations, finalization on the held-out test - split), including the preflight to confirm before launching and the health - checks to verify after. Use when launching, reproducing, or debugging a run. + Run one HarnessOpt-Bench optimization from preflight through held-out scoring, + and verify that the resulting measurement is complete. --- -# Running a HarnessOpt-Bench optimization - -A runbook for launching one benchmark's optimization run and confirming it is -healthy. Fill in your own inference endpoint, Modal account and W&B account; -treat every `` as something you supply. Never commit real keys, -tokens or endpoints. They belong in a local, git-ignored env file. - -## What a run is - -Each benchmark compiles from `harness-opt-bench//baseline/build.yaml`, -the single source of truth. `vero harbor run` compiles it into a Harbor task and -stands up three services: an **inference gateway** that holds the real upstream -key and enforces per-scope model allow-lists and token budgets, an **evaluation -sidecar** that owns the cases, the scoring and the final candidate selection, -and the **optimizer agent**, a coding agent that edits only `target/`, commits -candidates, and scores them through the `evals` CLI. When the optimizer finishes -or its clock runs out, the trusted verifier scores the selected candidate on the -held-out `test` partition and writes the final reward. - -The optimizer holds a scoped producer token for the gateway, never the upstream -key. The per-scope allow-list confines the target model in the normal case but -is not a hard guarantee against an adversarial optimizer, which authors the -candidate and holds the producer token. Benchmarks whose tasks run an -in-container judge set `task_services_use_upstream`, which puts the raw upstream -credential into the evaluation sub-run. Both are recorded in the affected -`build.yaml` files and in `vero/src/vero/gateway/inference.py`. +# Run a HarnessOpt-Bench optimization + +This runbook covers the benchmark workflow without assuming a particular model +provider, execution backend, or observability service. Keep deployment settings +and credentials in local, ignored files. + +## How a run is structured + +Each benchmark is defined by +`harness-opt-bench//baseline/build.yaml`. A run contains three trusted +components: + +- an inference gateway that enforces model and usage limits; +- an evaluation service that owns cases, scores, and candidate selection; +- an optimizer that edits `target/` and submits candidates through `evals`. + +After optimization, the evaluator scores the selected candidate on the held-out +test partition and writes the final result. The optimizer receives only scoped +access and the disclosures allowed for each partition. ## Prerequisites -- A checkout containing both `vero/` and `harness-opt-bench/`. Run commands from - `vero/`. -- `uv`. The CLI is invoked as `uv run vero ...`. -- A **Modal** account and tokens. Inner evaluation sandboxes run there - (`environment_name: ${inner_env:-modal}`), and the outer optimizer trial can too. -- **Docker**, if you run the outer trial locally with `--environment docker`. -- A **Weights & Biases** account and API key, cloud or self-hosted. -- An **OpenAI-compatible inference endpoint** and key that serves both your - optimizer model and the benchmark's target model. The gateway proxies to it. -- A local env file (start from `secrets.env.example`) with `OPENAI_API_KEY`, - `OPENAI_BASE_URL`, `MODAL_TOKEN_ID`, `MODAL_TOKEN_SECRET`, `WANDB_API_KEY`, - `WANDB_BASE_URL`, and optionally `MODAL_ENVIRONMENT` to pick a Modal - environment other than your workspace default. Every build declares these under - `secrets`; the compiler refuses to compile if one is missing. -- The task data for officeqa and browsecomp-plus, which is not in git. Run - `python3 scripts/task_data.py` and follow what it prints. - -## Launch command +- A checkout containing both `vero/` and `harness-opt-bench/`. +- `uv`, with commands run from `vero/`. +- An inference endpoint that supports the models named by the run. +- A Harbor environment capable of running the nested evaluations. +- Any credentials required by those services, stored in a local env file. +- Vendored task data where required. Check it with: + +```bash +python3 ../harness-opt-bench/scripts/task_data.py --check +``` + +Provider-specific packages, credentials, capacity settings, and environment +arguments belong in the launch environment, not in the benchmark definition. + +## Launch ```bash cd /vero uv run vero harbor run \ --config ../harness-opt-bench//baseline/build.yaml \ - --env-file .env \ - --environment modal \ - --agent \ + --env-file \ + --environment \ + --agent \ --model \ + --param inner_env= \ --param optimizer_model= \ - --param wandb_run=__