Seed-complexity blocks: shell stub and opencode full-harness seeds for Terminal-Bench and GAIA

.github/workflows/tests.yml

@@ -38,6 +38,16 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      # The opencode seeds vendor anomalyco/opencode as submodules and a config
+      # test checks the source is really there (an unpopulated submodule compiles
+      # to a hollow seed). Shallow, and only these two: the BrowseComp-Plus
+      # upstream corpus is not read by any test.
+      - name: Check out the vendored opencode source
+        run: |
+          git submodule update --init --depth 1 \
+            harness-opt-bench/terminal-bench/baseline/target-opencode/opencode \
+            harness-opt-bench/gaia/baseline/target-opencode/opencode
+
       - name: Install uv
         run: |
           curl -LsSf https://astral.sh/uv/install.sh | sh

.gitmodules

@@ -1,3 +1,9 @@
 [submodule "browsecomp-plus-upstream"]
 	path = harness-opt-bench/browsecomp-plus/upstream
 	url = https://github.com/texttron/BrowseComp-Plus.git
+[submodule "opencode-terminal-bench"]
+	path = harness-opt-bench/terminal-bench/baseline/target-opencode/opencode
+	url = https://github.com/anomalyco/opencode
+[submodule "opencode-gaia"]
+	path = harness-opt-bench/gaia/baseline/target-opencode/opencode
+	url = https://github.com/anomalyco/opencode

harness-opt-bench/gaia/README.md

@@ -12,7 +12,7 @@ files and images, run shell commands, and submit an exact answer.
 | Development / validation / test | 33 / 66 / 66 |
 | Split strategy | GAIA level and attachment presence |
 | Target model | `gpt-5.4-mini` |
-| Pinned seed baselines | 0.6205 working target; 0.0 shell target |
+| Pinned seed baselines | 0.6205 working target; 0.0 shell target; opencode target unpinned (measure before use) |
 | Scoring | Canonical task verifier |
 
 Development cases expose full results and task resources. Validation exposes
@@ -25,6 +25,7 @@ aggregate scores, and test remains hidden until final scoring.
 | `baseline/build.yaml` | Working tool-using agent | Measure harness improvement |
 | `baseline/build.shell.yaml` | Minimal non-solving skeleton | Measure harness construction from scratch |
 | `baseline/build.shell.e2e.yaml` | Skeleton with eight cases | Exercise the complete pipeline quickly |
+| `baseline/build.opencode.yaml` | opencode, vendored as source and compiled per candidate | Measure improvement of a full coding harness |
 
 The full and shell builds use the same tasks, model, budgets, and evaluation
 policy. See [the baseline guide](baseline/README.md) for their editable surfaces
@@ -52,6 +53,7 @@ regenerate the partitions, and review the manifest before using new results.
 | `baseline/build*.yaml` | Benchmark variants and evaluation settings |
 | `baseline/target/` | Working editable agent |
 | `baseline/target-shell/` | Minimal editable skeleton |
+| `baseline/target-opencode/` | opencode source (git submodule) plus a thin wrapper |
 | `partitions/` | Reported development, validation, and test split |
 | `partitions-e2e/` | Small smoke-test split |
 | `scripts/partition_gaia.py` | Split verification and regeneration |

harness-opt-bench/gaia/baseline/build.opencode.e2e.yaml

@@ -0,0 +1,149 @@
+# Field meanings and the shared rules are in ../../CONFIGURATION.md; comments here
+# cover only what is specific to this benchmark.
+name: vero/optimize-gaia-opencode-e2e
+description: >-
+  Improve opencode -- a full open-source coding agent, vendored as source under
+  opencode/ and compiled for each candidate -- as a GAIA agent, while preserving
+  the Harbor agent interface of the wrapper in src/. The target must answer
+  canonical GAIA tasks by writing the exact answer to /app/answer.txt; the wrapper
+  tells opencode so. Anything in the opencode source is editable: prompts, tools,
+  the agent loop, provider handling, defaults.
+agent_repo: target-opencode
+task_source: gaia/gaia@sha256:bbc356f476e0b70ba77da11a9be7d6345918d1e4a2daade0d6dfb82ee6f7b761
+task_manifest: ../partitions/manifest.json
+agent_import_path: gaia_agent.agent:GaiaAgent
+harbor_requirement: ${harbor_requirement:-harbor[modal]==0.20.0}
+vero_requirement: scaleapi-vero==0.6.0
+
+# E2E SMOKE VARIANT of build.opencode.yaml. Same seed, same target, same gateway
+# scopes; eight cases in total, tiny budgets, and a 25-minute agent clock so the
+# run reaches finalization quickly. It exists to prove the opencode seed end to end
+# (submodule snapshot, Bun build on the evaluation host, binary upload, run, session
+# archive), not to measure anything. Never report its score.
+partition_files:
+  development: ../partitions-e2e/development.json
+  validation: ../partitions-e2e/validation.json
+  test: ../partitions-e2e/test.json
+
+agent_access:
+  - partition: development
+    disclosure: full
+    expose_case_resources: true
+    total_runs: 6
+    total_cases: 12
+  - partition: validation
+    disclosure: aggregate
+    expose_case_resources: false
+    min_aggregate_cases: 1
+    total_runs: 6
+    total_cases: 18
+
+selection_partition: validation
+targets:
+  - partition: test
+    reward_key: reward
+    # TODO(seed-complexity): measure with rescore_candidate.py --seed --build-file
+    # build.opencode.yaml (K=3) and pin here before quoting deltas.
+    baseline_reward: 0.0
+    failure_value: 0.0
+    max_attempts: 1
+    n_attempts: 3
+    aggregate_attempts: mean
+
+evaluation_set_name: gaia
+objective:
+  selector:
+    metric: score
+  direction: maximize
+reward_mode: submit  # agent picks; falls back to auto_best, then current version
+baseline_floor: false  # gates on validation while reward is on test; opt-in only
+score_baseline: false
+rescore_top_k: 1
+rescore_attempts: 1
+
+# GAIA includes image inputs, so the target must be multimodal. Keep this name
+# aligned with the exact value sent by the agent.
+model: gpt-5.4-mini
+environment_name: ${inner_env:?set inner_env to a Harbor environment}
+extra_harbor_args: ["--ek", "app_name=harness-opt-bench", "--ek", "sandbox_idle_timeout_secs=3600"]
+harbor_python_version: "3.12"
+n_attempts: 1
+max_retries: 4
+retry_max_wait_seconds: 120
+infrastructure_max_attempts: 3
+infrastructure_retry_delay_seconds: 5
+aggregate_attempts: best
+feedback_transcripts: true
+feedback_max_bytes: 16000
+expose_attempt_detail: false
+# This smoke variant intentionally uses shorter overall limits.
+timeout_seconds: 1800
+case_timeout_seconds: 600
+task_agent_timeout_seconds: 600
+max_concurrency: 24
+error_rate_threshold: 0.1
+verifier_timeout_seconds: 3600
+optimizer_sandbox_timeout_seconds: 14400
+optimizer_sandbox_idle_timeout_seconds: 1800
+# 25 minutes: long enough for the optimizer to build opencode once and run an
+# evaluation or two, short enough that Harbor hands over to the verifier within
+# the hour, which is the path this variant exists to exercise.
+optimizer_agent_timeout_seconds: 1500
+optimizer_allow_internet: true
+optimizer_harness_versions:
+  claude-code: 2.1.220
+  codex: 0.146.0
+  opencode: 1.18.11
+  kimi-cli: 1.49.0
+  goose: 1.45.0
+  mini-swe-agent: 2.4.6
+
+secrets:
+  - MODAL_TOKEN_ID
+  - MODAL_TOKEN_SECRET
+  - MODAL_ENVIRONMENT
+  - WANDB_API_KEY
+  - WANDB_BASE_URL
+
+wandb:
+  project: harness-engineering-bench
+  group: gaia
+  name: ${wandb_run:-gaia-opencode-e2e}
+  tags: [gaia, opencode-seed, e2e-smoke]
+  log_traces: true
+
+# opencode runs inside the task container, which cannot reach the compose-internal
+# evaluation gateway. This hands the container the public upstream (OPENAI_*), as
+# task-owned services already get; the metered gateway stays on VERO_AGENT_INFERENCE_*
+# for in-process agents. Consequence: the target model is fixed by the wrapper, not by
+# the gateway allow-list, and its use is audited from the upstream per-key request log.
+task_services_use_upstream: true
+# Required by the flag above (as in browsecomp-plus): the isolated harness user
+# would otherwise see the raw upstream credential in its environment.
+harness_user: null
+agent_env:
+  BASH_MAX_TIMEOUT_MS: "3600000"
+  BASH_DEFAULT_TIMEOUT_MS: "3600000"
+  ENABLE_BACKGROUND_TASKS: "0"
+  FORCE_AUTO_BACKGROUND_TASKS: "0"
+  UV_TOOL_BIN_DIR: "/home/agent/.local/bin"
+
+inference_gateway:
+  upstream_api_key_env: OPENAI_API_KEY
+  upstream_base_url_env: OPENAI_BASE_URL
+  request_log_attribution: true
+  producer:
+    allowed_models: ["${optimizer_model:-gpt-5.4}"]
+    max_concurrency: 8
+  evaluation:
+    allowed_models: [gpt-5.4-mini]
+    max_requests: 200000
+    max_tokens: 200000000  # 30 agent case-runs at most
+    max_concurrency: 64
+  finalization:
+    allowed_models: [gpt-5.4-mini]
+    max_requests: 200000
+    max_tokens: 2000000000  # 66 test cases x3 attempts + rescore headroom
+    max_concurrency: 64
+instruct_multifidelity: true
+instruct_exhaust_budget: true

harness-opt-bench/gaia/baseline/build.opencode.yaml

@@ -0,0 +1,144 @@
+# Field meanings and the shared rules are in ../../CONFIGURATION.md; comments here
+# cover only what is specific to this benchmark.
+name: vero/optimize-gaia-opencode
+description: >-
+  Improve opencode -- a full open-source coding agent, vendored as source under
+  opencode/ and compiled for each candidate -- as a GAIA agent, while preserving
+  the Harbor agent interface of the wrapper in src/. The target must answer
+  canonical GAIA tasks by writing the exact answer to /app/answer.txt; the wrapper
+  tells opencode so. Anything in the opencode source is editable: prompts, tools,
+  the agent loop, provider handling, defaults.
+agent_repo: target-opencode
+task_source: gaia/gaia@sha256:bbc356f476e0b70ba77da11a9be7d6345918d1e4a2daade0d6dfb82ee6f7b761
+task_manifest: ../partitions/manifest.json
+agent_import_path: gaia_agent.agent:GaiaAgent
+harbor_requirement: ${harbor_requirement:-harbor[modal]==0.20.0}
+vero_requirement: scaleapi-vero==0.6.0
+
+partition_files:
+  development: ../partitions/development.json
+  validation: ../partitions/validation.json
+  test: ../partitions/test.json
+
+agent_access:
+  - partition: development
+    disclosure: full
+    expose_case_resources: true
+    total_runs: 100
+    total_cases: 132
+  - partition: validation
+    disclosure: aggregate
+    expose_case_resources: false
+    min_aggregate_cases: 5
+    total_runs: 100
+    total_cases: 264
+
+selection_partition: validation
+targets:
+  - partition: test
+    reward_key: reward
+    # TODO(seed-complexity): measure with rescore_candidate.py --seed --build-file
+    # build.opencode.yaml (K=3) and pin here before quoting deltas.
+    baseline_reward: 0.0
+    failure_value: 0.0
+    max_attempts: 1
+    n_attempts: 3
+    aggregate_attempts: mean
+
+evaluation_set_name: gaia
+objective:
+  selector:
+    metric: score
+  direction: maximize
+reward_mode: submit  # agent picks; falls back to auto_best, then current version
+baseline_floor: false  # gates on validation while reward is on test; opt-in only
+score_baseline: false
+rescore_top_k: 3
+rescore_attempts: 1
+
+# GAIA includes image inputs, so the target must be multimodal. Keep this name
+# aligned with the exact value sent by the agent.
+model: gpt-5.4-mini
+environment_name: ${inner_env:?set inner_env to a Harbor environment}
+extra_harbor_args: ["--ek", "app_name=harness-opt-bench", "--ek", "sandbox_idle_timeout_secs=14400"]
+harbor_python_version: "3.12"
+n_attempts: 1
+max_retries: 4
+retry_max_wait_seconds: 120
+infrastructure_max_attempts: 3
+infrastructure_retry_delay_seconds: 5
+aggregate_attempts: best
+feedback_transcripts: true
+feedback_max_bytes: 16000
+expose_attempt_detail: false
+# Above the 5,400-second worst case at the configured concurrency.
+timeout_seconds: 7200
+case_timeout_seconds: 600
+task_agent_timeout_seconds: 600
+max_concurrency: 24
+error_rate_threshold: 0.1
+verifier_timeout_seconds: 14400
+optimizer_sandbox_timeout_seconds: 86400
+optimizer_sandbox_idle_timeout_seconds: 14400
+# The vendored opencode image takes 10-23 min to start on Modal; the 1800 s default
+# timed out ~10 launches (EnvironmentStartTimeoutError).
+build_timeout_seconds: 5400
+optimizer_agent_timeout_seconds: 72000
+optimizer_allow_internet: true
+optimizer_harness_versions:
+  claude-code: 2.1.220
+  codex: 0.146.0
+  opencode: 1.18.11
+  kimi-cli: 1.49.0
+  goose: 1.45.0
+  mini-swe-agent: 2.4.6
+
+secrets:
+  - MODAL_TOKEN_ID
+  - MODAL_TOKEN_SECRET
+  - MODAL_ENVIRONMENT
+  - WANDB_API_KEY
+  - WANDB_BASE_URL
+
+wandb:
+  project: harness-engineering-bench
+  group: gaia
+  name: ${wandb_run:-gaia-opencode}
+  tags: [gaia, opencode-seed]
+  log_traces: true
+
+# opencode runs inside the task container, which cannot reach the compose-internal
+# evaluation gateway. This hands the container the public upstream (OPENAI_*), as
+# task-owned services already get; the metered gateway stays on VERO_AGENT_INFERENCE_*
+# for in-process agents. Consequence: the target model is fixed by the wrapper, not by
+# the gateway allow-list, and its use is audited from the upstream per-key request log.
+task_services_use_upstream: true
+# Required by the flag above (as in browsecomp-plus): the isolated harness user
+# would otherwise see the raw upstream credential in its environment.
+harness_user: null
+agent_env:
+  BASH_MAX_TIMEOUT_MS: "3600000"
+  BASH_DEFAULT_TIMEOUT_MS: "3600000"
+  ENABLE_BACKGROUND_TASKS: "0"
+  FORCE_AUTO_BACKGROUND_TASKS: "0"
+  UV_TOOL_BIN_DIR: "/home/agent/.local/bin"
+
+inference_gateway:
+  upstream_api_key_env: OPENAI_API_KEY
+  upstream_base_url_env: OPENAI_BASE_URL
+  request_log_attribution: true
+  producer:
+    allowed_models: ["${optimizer_model:-gpt-5.4}"]
+    max_concurrency: 8
+  evaluation:
+    allowed_models: [gpt-5.4-mini]
+    max_requests: 200000
+    max_tokens: 2000000000  # 396 agent case-runs (132 dev + 264 validation)
+    max_concurrency: 64
+  finalization:
+    allowed_models: [gpt-5.4-mini]
+    max_requests: 200000
+    max_tokens: 2000000000  # 66 test cases x3 attempts + rescore headroom
+    max_concurrency: 64
+instruct_multifidelity: true
+instruct_exhaust_budget: true