Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/guideline.html
Original file line number Diff line number Diff line change
Expand Up @@ -394,7 +394,7 @@ <h2>Research model backends</h2>
<tr><td><code>claude_chat</code></td><td>Yes</td><td>Yes</td><td>Runs an installed, authenticated Claude Code CLI via <code>claude -p</code>; not a direct Anthropic API client.</td></tr>
<tr><td><code>qwen_chat</code></td><td>Yes</td><td>Yes</td><td>Local or hosted Qwen-compatible server.</td></tr>
<tr><td><code>minimax_chat</code></td><td>Yes</td><td>Yes</td><td>MiniMax chat endpoint.</td></tr>
<tr><td><code>codex_exec</code></td><td>No</td><td>Supported adapters only</td><td>Executes Codex as a target agent.</td></tr>
<tr><td><code>codex_exec</code></td><td>Yes</td><td>Supported adapters only</td><td>Executes Codex for optimizer calls and as a target agent where supported.</td></tr>
<tr><td><code>claude_code_exec</code></td><td>No</td><td>Supported adapters only</td><td>Executes Claude Code as a target agent.</td></tr>
</tbody>
</table>
Expand Down
2 changes: 1 addition & 1 deletion docs/reference/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ not via a base class subclass. Supported values (as of this writing):
| `qwen_chat` | ✓ | ✓ |
| `minimax_chat` | ✓ | ✓ |
| `openai_compatible` | ✓ | ✓ |
| `codex_exec` | | ✓ |
| `codex_exec` | | ✓ |
| `claude_code_exec` | — | ✓ |

See `skillopt/model/backend_config.py` for the live whitelist and
Expand Down
4 changes: 2 additions & 2 deletions docs/reference/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ selecting the generic OpenAI-compatible backend.
| `claude_chat` | ✓ | ✓ |
| `qwen_chat` | ✓ | ✓ |
| `minimax_chat` | ✓ | ✓ |
| `codex_exec` | | ✓ |
| `codex_exec` | | ✓ |
| `claude_code_exec` | — | ✓ |

MiniMax currently has one shared deployment. `model.minimax_model` is applied
Expand All @@ -28,7 +28,7 @@ MiniMax optimizer model and a different target model.
| `model.backend` | str | `azure_openai` | Backward-compatible high-level run label |
| `model.optimizer` | str | `gpt-5.5` | Optimizer deployment/model |
| `model.target` | str | `gpt-5.5` | Target deployment/model |
| `model.optimizer_backend` | str | `openai_chat` | Optimizer client path; chat backends only |
| `model.optimizer_backend` | str | `openai_chat` | Optimizer client path; chat backends plus `codex_exec` |
| `model.target_backend` | str | `openai_chat` | Target client path; chat or exec backend |
| `model.reasoning_effort` | str | `medium` | Shared reasoning effort |
| `model.rewrite_reasoning_effort` | str | empty | Optional full-rewrite effort override |
Expand Down
6 changes: 4 additions & 2 deletions scripts/eval_only.py
Original file line number Diff line number Diff line change
Expand Up @@ -320,8 +320,10 @@ def _has_model_override(dotted_key: str, legacy_key: str) -> bool:
cfg.setdefault("optimizer_backend", "claude_chat")
cfg.setdefault("target_backend", "claude_chat")
elif backend in {"codex", "codex_exec"}:
cfg.setdefault("optimizer_backend", "openai_chat")
cfg.setdefault("target_backend", "codex_exec")
if not _has_model_override("model.optimizer_backend", "optimizer_backend"):
cfg["optimizer_backend"] = "codex_exec"
if not _has_model_override("model.target_backend", "target_backend"):
cfg["target_backend"] = "codex_exec"
elif backend == "claude_code_exec":
cfg.setdefault("optimizer_backend", "openai_chat")
cfg.setdefault("target_backend", "claude_code_exec")
Expand Down
6 changes: 4 additions & 2 deletions scripts/train.py
Original file line number Diff line number Diff line change
Expand Up @@ -438,8 +438,10 @@ def _has_model_override(dotted_key: str, legacy_key: str) -> bool:
flat.setdefault("optimizer_backend", "claude_chat")
flat.setdefault("target_backend", "claude_chat")
elif backend in {"codex", "codex_exec"}:
flat.setdefault("optimizer_backend", "openai_chat")
flat.setdefault("target_backend", "codex_exec")
if not _has_model_override("model.optimizer_backend", "optimizer_backend"):
flat["optimizer_backend"] = "codex_exec"
if not _has_model_override("model.target_backend", "target_backend"):
flat["target_backend"] = "codex_exec"
elif backend == "claude_code_exec":
flat.setdefault("optimizer_backend", "openai_chat")
flat.setdefault("target_backend", "claude_code_exec")
Expand Down
14 changes: 8 additions & 6 deletions skillopt/engine/trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -670,12 +670,14 @@ def _build_eval_env(split: str, env_num: int, seed: int):
optimizer_backend = cfg.get("optimizer_backend")
target_backend = cfg.get("target_backend")
if not optimizer_backend or not target_backend:
if backend in {"claude", "claude_chat"}:
optimizer_backend = optimizer_backend or "claude_chat"
target_backend = target_backend or "claude_chat"
elif backend in {"codex", "codex_exec"}:
optimizer_backend = optimizer_backend or "openai_chat"
target_backend = target_backend or "codex_exec"
if backend in {"claude", "claude_chat"}:
optimizer_backend = optimizer_backend or "claude_chat"
target_backend = target_backend or "claude_chat"
elif backend in {"codex", "codex_exec"}:
if optimizer_backend in (None, "", "openai_chat"):
optimizer_backend = "codex_exec"
if target_backend in (None, "", "openai_chat"):
target_backend = "codex_exec"
elif backend == "claude_code_exec":
optimizer_backend = optimizer_backend or "openai_chat"
target_backend = target_backend or "claude_code_exec"
Expand Down
4 changes: 2 additions & 2 deletions skillopt/envs/_template/config_template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,6 @@ evaluation:
# ── Model ────────────────────────────────────────
# Override only what differs from the inherited defaults.
model:
optimizer_backend: openai_chat # openai_chat | claude_chat | qwen_chat | minimax_chat
target_backend: openai_chat # plus codex_exec / claude_code_exec for target only
optimizer_backend: openai_chat # openai_chat | claude_chat | qwen_chat | minimax_chat | codex_exec
target_backend: openai_chat # chat backends plus codex_exec / claude_code_exec
reasoning_effort: medium
46 changes: 43 additions & 3 deletions skillopt/model/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

from skillopt.model import azure_openai as _openai
from skillopt.model import claude_backend as _claude
from skillopt.model import codex_backend as _codex
from skillopt.model import minimax_backend as _minimax
from skillopt.model import openai_compatible_backend as _openai_compat
from skillopt.model import qwen_backend as _qwen
Expand Down Expand Up @@ -41,10 +42,14 @@ def set_backend(name: str | None) -> str:
set_target_backend("claude_chat")
return "claude_chat"
if normalized == "codex":
set_optimizer_backend("openai_chat")
set_optimizer_backend("codex_exec")
set_target_backend("codex_exec")
return "codex"
if normalized in {"codex_exec", "claude_code_exec"}:
if normalized == "codex_exec":
set_optimizer_backend("codex_exec")
set_target_backend("codex_exec")
return normalized
if normalized == "claude_code_exec":
set_optimizer_backend("openai_chat")
set_target_backend(normalized)
return normalized
Expand Down Expand Up @@ -73,7 +78,7 @@ def get_backend_name() -> str:
return "qwen_chat"
if optimizer == "openai_chat" and target == "openai_chat":
return "azure_openai"
if optimizer == "openai_chat" and target == "codex_exec":
if optimizer == "codex_exec" and target == "codex_exec":
return "codex"
if optimizer == "openai_chat" and target == "qwen_chat":
return "qwen_chat"
Expand Down Expand Up @@ -132,6 +137,15 @@ def chat_optimizer(
reasoning_effort=reasoning_effort,
timeout=timeout,
)
if get_optimizer_backend() == "codex_exec":
return _codex.chat_optimizer(
system=system,
user=user,
max_completion_tokens=max_completion_tokens,
retries=retries,
stage=stage,
timeout=timeout,
)
return _openai.chat_optimizer(
system=system,
user=user,
Expand Down Expand Up @@ -265,6 +279,17 @@ def chat_optimizer_messages(
return_message=return_message,
timeout=timeout,
)
if get_optimizer_backend() == "codex_exec":
return _codex.chat_optimizer_messages(
messages=messages,
max_completion_tokens=max_completion_tokens,
retries=retries,
stage=stage,
tools=tools,
tool_choice=tool_choice,
return_message=return_message,
timeout=timeout,
)
return _openai.chat_optimizer_messages(
messages=messages,
max_completion_tokens=max_completion_tokens,
Expand Down Expand Up @@ -449,6 +474,17 @@ def get_token_summary() -> dict:
summary[stage]["prompt_tokens"] += values["prompt_tokens"]
summary[stage]["completion_tokens"] += values["completion_tokens"]
summary[stage]["total_tokens"] += values["total_tokens"]
codex_summary = _codex.get_token_summary()
for stage, values in codex_summary.items():
if stage == "_total":
continue
if stage not in summary:
summary[stage] = values
continue
summary[stage]["calls"] += values["calls"]
summary[stage]["prompt_tokens"] += values["prompt_tokens"]
summary[stage]["completion_tokens"] += values["completion_tokens"]
summary[stage]["total_tokens"] += values["total_tokens"]
total = {
"calls": 0,
"prompt_tokens": 0,
Expand All @@ -472,6 +508,7 @@ def reset_token_tracker() -> None:
_qwen.reset_token_tracker()
_minimax.reset_token_tracker()
_openai_compat.reset_token_tracker()
_codex.reset_token_tracker()


def configure_azure_openai(
Expand Down Expand Up @@ -622,6 +659,7 @@ def set_reasoning_effort(effort: str | None) -> None:
_qwen.set_reasoning_effort(effort)
_minimax.set_reasoning_effort(effort)
_openai_compat.set_reasoning_effort(effort)
_codex.set_reasoning_effort(effort)


def set_target_deployment(deployment: str) -> None:
Expand All @@ -630,10 +668,12 @@ def set_target_deployment(deployment: str) -> None:
_qwen.set_target_deployment(deployment)
_minimax.set_target_deployment(deployment)
_openai_compat.set_target_deployment(deployment)
_codex.set_target_deployment(deployment)


def set_optimizer_deployment(deployment: str) -> None:
_openai.set_optimizer_deployment(deployment)
_claude.set_optimizer_deployment(deployment)
_qwen.set_optimizer_deployment(deployment)
_openai_compat.set_optimizer_deployment(deployment)
_codex.set_optimizer_deployment(deployment)
21 changes: 18 additions & 3 deletions skillopt/model/backend_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,18 @@ def _parse_int(value: str | None, default: int) -> int:
def set_optimizer_backend(backend: str) -> None:
global OPTIMIZER_BACKEND
OPTIMIZER_BACKEND = normalize_backend_name(backend or "openai_chat")
if OPTIMIZER_BACKEND not in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "openai_compatible"}:
if OPTIMIZER_BACKEND not in {
"openai_chat",
"claude_chat",
"qwen_chat",
"minimax_chat",
"openai_compatible",
"codex_exec",
}:
raise ValueError(
f"Unsupported optimizer backend: {OPTIMIZER_BACKEND!r}. "
"Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', 'minimax_chat', and 'openai_compatible'."
"Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', 'minimax_chat', "
"'openai_compatible', and 'codex_exec'."
)
os.environ["OPTIMIZER_BACKEND"] = OPTIMIZER_BACKEND

Expand Down Expand Up @@ -81,7 +89,14 @@ def is_target_exec_backend() -> bool:


def is_optimizer_chat_backend() -> bool:
return OPTIMIZER_BACKEND in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "openai_compatible"}
return OPTIMIZER_BACKEND in {
"openai_chat",
"claude_chat",
"qwen_chat",
"minimax_chat",
"openai_compatible",
"codex_exec",
}


def is_target_chat_backend() -> bool:
Expand Down
19 changes: 12 additions & 7 deletions skillopt/model/codex_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
CompatToolFunction,
tracker,
)
from skillopt.model.backend_config import get_codex_exec_config


CODEX_BIN = os.environ.get("CODEX_CLI_BIN", "codex")
Expand Down Expand Up @@ -286,20 +287,21 @@ def _run_codex_exec(
timeout: int | None,
) -> tuple[str, dict[str, int]]:
with tempfile.TemporaryDirectory(prefix="skillopt_codex_") as temp_dir:
config = get_codex_exec_config()
output_path = os.path.join(temp_dir, "last_message.txt")
image_paths = _materialize_attachments(attachments, temp_dir)
profile = str(config.get("profile") or os.environ.get("CODEX_PROFILE", "")).strip()
reasoning_effort = str(REASONING_EFFORT or config.get("reasoning_effort") or "").strip()

command = [
CODEX_BIN,
str(config.get("path") or CODEX_BIN),
"exec",
"--json",
"--ephemeral",
"--profile",
CODEX_PROFILE,
"-c",
"approval_policy=\"never\"",
f"approval_policy={json.dumps(str(config.get('approval_policy') or 'never'))}",
"--sandbox",
CODEX_SANDBOX_MODE,
str(config.get("sandbox") or CODEX_SANDBOX_MODE),
"--skip-git-repo-check",
"--cd",
_default_working_directory(),
Expand All @@ -309,8 +311,11 @@ def _run_codex_exec(
output_path,
]

if REASONING_EFFORT:
command.extend(["-c", f"model_reasoning_effort={json.dumps(REASONING_EFFORT)}"])
if profile:
command.extend(["--profile", profile])

if reasoning_effort and reasoning_effort != "none":
command.extend(["-c", f"model_reasoning_effort={json.dumps(reasoning_effort)}"])

schema_path = None
if output_schema is not None:
Expand Down
Loading