diff --git a/docs/guideline.html b/docs/guideline.html index 9dd7d1e2..3bd625db 100644 --- a/docs/guideline.html +++ b/docs/guideline.html @@ -394,7 +394,7 @@

Research model backends

claude_chatYesYesRuns an installed, authenticated Claude Code CLI via claude -p; not a direct Anthropic API client. qwen_chatYesYesLocal or hosted Qwen-compatible server. minimax_chatYesYesMiniMax chat endpoint. - codex_execNoSupported adapters onlyExecutes Codex as a target agent. + codex_execYesSupported adapters onlyExecutes Codex for optimizer calls and as a target agent where supported. claude_code_execNoSupported adapters onlyExecutes Claude Code as a target agent. diff --git a/docs/reference/api.md b/docs/reference/api.md index b9fd6e31..287a7175 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -190,7 +190,7 @@ not via a base class subclass. Supported values (as of this writing): | `qwen_chat` | ✓ | ✓ | | `minimax_chat` | ✓ | ✓ | | `openai_compatible` | ✓ | ✓ | -| `codex_exec` | — | ✓ | +| `codex_exec` | ✓ | ✓ | | `claude_code_exec` | — | ✓ | See `skillopt/model/backend_config.py` for the live whitelist and diff --git a/docs/reference/config.md b/docs/reference/config.md index d057cf1d..b65c204d 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -16,7 +16,7 @@ selecting the generic OpenAI-compatible backend. | `claude_chat` | ✓ | ✓ | | `qwen_chat` | ✓ | ✓ | | `minimax_chat` | ✓ | ✓ | -| `codex_exec` | — | ✓ | +| `codex_exec` | ✓ | ✓ | | `claude_code_exec` | — | ✓ | MiniMax currently has one shared deployment. `model.minimax_model` is applied @@ -28,7 +28,7 @@ MiniMax optimizer model and a different target model. | `model.backend` | str | `azure_openai` | Backward-compatible high-level run label | | `model.optimizer` | str | `gpt-5.5` | Optimizer deployment/model | | `model.target` | str | `gpt-5.5` | Target deployment/model | -| `model.optimizer_backend` | str | `openai_chat` | Optimizer client path; chat backends only | +| `model.optimizer_backend` | str | `openai_chat` | Optimizer client path; chat backends plus `codex_exec` | | `model.target_backend` | str | `openai_chat` | Target client path; chat or exec backend | | `model.reasoning_effort` | str | `medium` | Shared reasoning effort | | `model.rewrite_reasoning_effort` | str | empty | Optional full-rewrite effort override | diff --git a/scripts/eval_only.py b/scripts/eval_only.py index 79dfab09..f3c5f86f 100644 --- a/scripts/eval_only.py +++ b/scripts/eval_only.py @@ -320,8 +320,10 @@ def _has_model_override(dotted_key: str, legacy_key: str) -> bool: cfg.setdefault("optimizer_backend", "claude_chat") cfg.setdefault("target_backend", "claude_chat") elif backend in {"codex", "codex_exec"}: - cfg.setdefault("optimizer_backend", "openai_chat") - cfg.setdefault("target_backend", "codex_exec") + if not _has_model_override("model.optimizer_backend", "optimizer_backend"): + cfg["optimizer_backend"] = "codex_exec" + if not _has_model_override("model.target_backend", "target_backend"): + cfg["target_backend"] = "codex_exec" elif backend == "claude_code_exec": cfg.setdefault("optimizer_backend", "openai_chat") cfg.setdefault("target_backend", "claude_code_exec") diff --git a/scripts/train.py b/scripts/train.py index 5c0621ac..918e684a 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -438,8 +438,10 @@ def _has_model_override(dotted_key: str, legacy_key: str) -> bool: flat.setdefault("optimizer_backend", "claude_chat") flat.setdefault("target_backend", "claude_chat") elif backend in {"codex", "codex_exec"}: - flat.setdefault("optimizer_backend", "openai_chat") - flat.setdefault("target_backend", "codex_exec") + if not _has_model_override("model.optimizer_backend", "optimizer_backend"): + flat["optimizer_backend"] = "codex_exec" + if not _has_model_override("model.target_backend", "target_backend"): + flat["target_backend"] = "codex_exec" elif backend == "claude_code_exec": flat.setdefault("optimizer_backend", "openai_chat") flat.setdefault("target_backend", "claude_code_exec") diff --git a/skillopt/engine/trainer.py b/skillopt/engine/trainer.py index 8312cdb0..ff535b05 100644 --- a/skillopt/engine/trainer.py +++ b/skillopt/engine/trainer.py @@ -670,12 +670,14 @@ def _build_eval_env(split: str, env_num: int, seed: int): optimizer_backend = cfg.get("optimizer_backend") target_backend = cfg.get("target_backend") if not optimizer_backend or not target_backend: - if backend in {"claude", "claude_chat"}: - optimizer_backend = optimizer_backend or "claude_chat" - target_backend = target_backend or "claude_chat" - elif backend in {"codex", "codex_exec"}: - optimizer_backend = optimizer_backend or "openai_chat" - target_backend = target_backend or "codex_exec" + if backend in {"claude", "claude_chat"}: + optimizer_backend = optimizer_backend or "claude_chat" + target_backend = target_backend or "claude_chat" + elif backend in {"codex", "codex_exec"}: + if optimizer_backend in (None, "", "openai_chat"): + optimizer_backend = "codex_exec" + if target_backend in (None, "", "openai_chat"): + target_backend = "codex_exec" elif backend == "claude_code_exec": optimizer_backend = optimizer_backend or "openai_chat" target_backend = target_backend or "claude_code_exec" diff --git a/skillopt/envs/_template/config_template.yaml b/skillopt/envs/_template/config_template.yaml index b482cc71..d98851dc 100644 --- a/skillopt/envs/_template/config_template.yaml +++ b/skillopt/envs/_template/config_template.yaml @@ -50,6 +50,6 @@ evaluation: # ── Model ──────────────────────────────────────── # Override only what differs from the inherited defaults. model: - optimizer_backend: openai_chat # openai_chat | claude_chat | qwen_chat | minimax_chat - target_backend: openai_chat # … plus codex_exec / claude_code_exec for target only + optimizer_backend: openai_chat # openai_chat | claude_chat | qwen_chat | minimax_chat | codex_exec + target_backend: openai_chat # chat backends plus codex_exec / claude_code_exec reasoning_effort: medium diff --git a/skillopt/model/__init__.py b/skillopt/model/__init__.py index ee03f3d9..f58be67b 100644 --- a/skillopt/model/__init__.py +++ b/skillopt/model/__init__.py @@ -6,6 +6,7 @@ from skillopt.model import azure_openai as _openai from skillopt.model import claude_backend as _claude +from skillopt.model import codex_backend as _codex from skillopt.model import minimax_backend as _minimax from skillopt.model import openai_compatible_backend as _openai_compat from skillopt.model import qwen_backend as _qwen @@ -41,10 +42,14 @@ def set_backend(name: str | None) -> str: set_target_backend("claude_chat") return "claude_chat" if normalized == "codex": - set_optimizer_backend("openai_chat") + set_optimizer_backend("codex_exec") set_target_backend("codex_exec") return "codex" - if normalized in {"codex_exec", "claude_code_exec"}: + if normalized == "codex_exec": + set_optimizer_backend("codex_exec") + set_target_backend("codex_exec") + return normalized + if normalized == "claude_code_exec": set_optimizer_backend("openai_chat") set_target_backend(normalized) return normalized @@ -73,7 +78,7 @@ def get_backend_name() -> str: return "qwen_chat" if optimizer == "openai_chat" and target == "openai_chat": return "azure_openai" - if optimizer == "openai_chat" and target == "codex_exec": + if optimizer == "codex_exec" and target == "codex_exec": return "codex" if optimizer == "openai_chat" and target == "qwen_chat": return "qwen_chat" @@ -132,6 +137,15 @@ def chat_optimizer( reasoning_effort=reasoning_effort, timeout=timeout, ) + if get_optimizer_backend() == "codex_exec": + return _codex.chat_optimizer( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) return _openai.chat_optimizer( system=system, user=user, @@ -265,6 +279,17 @@ def chat_optimizer_messages( return_message=return_message, timeout=timeout, ) + if get_optimizer_backend() == "codex_exec": + return _codex.chat_optimizer_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) return _openai.chat_optimizer_messages( messages=messages, max_completion_tokens=max_completion_tokens, @@ -449,6 +474,17 @@ def get_token_summary() -> dict: summary[stage]["prompt_tokens"] += values["prompt_tokens"] summary[stage]["completion_tokens"] += values["completion_tokens"] summary[stage]["total_tokens"] += values["total_tokens"] + codex_summary = _codex.get_token_summary() + for stage, values in codex_summary.items(): + if stage == "_total": + continue + if stage not in summary: + summary[stage] = values + continue + summary[stage]["calls"] += values["calls"] + summary[stage]["prompt_tokens"] += values["prompt_tokens"] + summary[stage]["completion_tokens"] += values["completion_tokens"] + summary[stage]["total_tokens"] += values["total_tokens"] total = { "calls": 0, "prompt_tokens": 0, @@ -472,6 +508,7 @@ def reset_token_tracker() -> None: _qwen.reset_token_tracker() _minimax.reset_token_tracker() _openai_compat.reset_token_tracker() + _codex.reset_token_tracker() def configure_azure_openai( @@ -622,6 +659,7 @@ def set_reasoning_effort(effort: str | None) -> None: _qwen.set_reasoning_effort(effort) _minimax.set_reasoning_effort(effort) _openai_compat.set_reasoning_effort(effort) + _codex.set_reasoning_effort(effort) def set_target_deployment(deployment: str) -> None: @@ -630,6 +668,7 @@ def set_target_deployment(deployment: str) -> None: _qwen.set_target_deployment(deployment) _minimax.set_target_deployment(deployment) _openai_compat.set_target_deployment(deployment) + _codex.set_target_deployment(deployment) def set_optimizer_deployment(deployment: str) -> None: @@ -637,3 +676,4 @@ def set_optimizer_deployment(deployment: str) -> None: _claude.set_optimizer_deployment(deployment) _qwen.set_optimizer_deployment(deployment) _openai_compat.set_optimizer_deployment(deployment) + _codex.set_optimizer_deployment(deployment) diff --git a/skillopt/model/backend_config.py b/skillopt/model/backend_config.py index f82b393b..d74bd135 100644 --- a/skillopt/model/backend_config.py +++ b/skillopt/model/backend_config.py @@ -49,10 +49,18 @@ def _parse_int(value: str | None, default: int) -> int: def set_optimizer_backend(backend: str) -> None: global OPTIMIZER_BACKEND OPTIMIZER_BACKEND = normalize_backend_name(backend or "openai_chat") - if OPTIMIZER_BACKEND not in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "openai_compatible"}: + if OPTIMIZER_BACKEND not in { + "openai_chat", + "claude_chat", + "qwen_chat", + "minimax_chat", + "openai_compatible", + "codex_exec", + }: raise ValueError( f"Unsupported optimizer backend: {OPTIMIZER_BACKEND!r}. " - "Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', 'minimax_chat', and 'openai_compatible'." + "Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', 'minimax_chat', " + "'openai_compatible', and 'codex_exec'." ) os.environ["OPTIMIZER_BACKEND"] = OPTIMIZER_BACKEND @@ -81,7 +89,14 @@ def is_target_exec_backend() -> bool: def is_optimizer_chat_backend() -> bool: - return OPTIMIZER_BACKEND in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "openai_compatible"} + return OPTIMIZER_BACKEND in { + "openai_chat", + "claude_chat", + "qwen_chat", + "minimax_chat", + "openai_compatible", + "codex_exec", + } def is_target_chat_backend() -> bool: diff --git a/skillopt/model/codex_backend.py b/skillopt/model/codex_backend.py index 64b6f355..ff7f79cd 100644 --- a/skillopt/model/codex_backend.py +++ b/skillopt/model/codex_backend.py @@ -18,6 +18,7 @@ CompatToolFunction, tracker, ) +from skillopt.model.backend_config import get_codex_exec_config CODEX_BIN = os.environ.get("CODEX_CLI_BIN", "codex") @@ -286,20 +287,21 @@ def _run_codex_exec( timeout: int | None, ) -> tuple[str, dict[str, int]]: with tempfile.TemporaryDirectory(prefix="skillopt_codex_") as temp_dir: + config = get_codex_exec_config() output_path = os.path.join(temp_dir, "last_message.txt") image_paths = _materialize_attachments(attachments, temp_dir) + profile = str(config.get("profile") or os.environ.get("CODEX_PROFILE", "")).strip() + reasoning_effort = str(REASONING_EFFORT or config.get("reasoning_effort") or "").strip() command = [ - CODEX_BIN, + str(config.get("path") or CODEX_BIN), "exec", "--json", "--ephemeral", - "--profile", - CODEX_PROFILE, "-c", - "approval_policy=\"never\"", + f"approval_policy={json.dumps(str(config.get('approval_policy') or 'never'))}", "--sandbox", - CODEX_SANDBOX_MODE, + str(config.get("sandbox") or CODEX_SANDBOX_MODE), "--skip-git-repo-check", "--cd", _default_working_directory(), @@ -309,8 +311,11 @@ def _run_codex_exec( output_path, ] - if REASONING_EFFORT: - command.extend(["-c", f"model_reasoning_effort={json.dumps(REASONING_EFFORT)}"]) + if profile: + command.extend(["--profile", profile]) + + if reasoning_effort and reasoning_effort != "none": + command.extend(["-c", f"model_reasoning_effort={json.dumps(reasoning_effort)}"]) schema_path = None if output_schema is not None: diff --git a/tests/test_codex_optimizer_backend.py b/tests/test_codex_optimizer_backend.py new file mode 100644 index 00000000..8701a0e9 --- /dev/null +++ b/tests/test_codex_optimizer_backend.py @@ -0,0 +1,342 @@ +from __future__ import annotations + +import importlib.util +import os +import sys +import types +from collections.abc import Iterator +from typing import Any + +import pytest + + +class _OpenAIClientStub: + def __init__(self, *args: Any, **kwargs: Any) -> None: + self.args = args + self.kwargs = kwargs + + +def _install_openai_stub() -> None: + if "openai" in sys.modules or importlib.util.find_spec("openai") is not None: + return + openai_stub = types.ModuleType("openai") + openai_stub.AzureOpenAI = _OpenAIClientStub + openai_stub.OpenAI = _OpenAIClientStub + sys.modules["openai"] = openai_stub + + +def _import_model_modules() -> tuple[Any, Any, Any, Any]: + _install_openai_stub() + import skillopt.model as model_module + from skillopt.model import azure_openai, backend_config, codex_backend + + return model_module, backend_config, codex_backend, azure_openai + + +@pytest.fixture(autouse=True) +def isolate_backend_state() -> Iterator[tuple[Any, Any, Any, Any]]: + model_module, backend_config, codex_backend, azure_openai = _import_model_modules() + optimizer_backend = backend_config.get_optimizer_backend() + target_backend = backend_config.get_target_backend() + env = { + key: os.environ.get(key) + for key in ( + "OPTIMIZER_BACKEND", + "TARGET_BACKEND", + "OPTIMIZER_DEPLOYMENT", + "TARGET_DEPLOYMENT", + ) + } + yield model_module, backend_config, codex_backend, azure_openai + backend_config.set_optimizer_backend(optimizer_backend) + backend_config.set_target_backend(target_backend) + for key, value in env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def test_codex_exec_can_be_optimizer_backend( + isolate_backend_state: tuple[Any, Any, Any, Any], +) -> None: + _model_module, backend_config, _codex_backend, _azure_openai = isolate_backend_state + + backend_config.set_optimizer_backend("codex_exec") + + assert backend_config.get_optimizer_backend() == "codex_exec" + + +def test_set_backend_codex_uses_codex_for_optimizer_and_target( + isolate_backend_state: tuple[Any, Any, Any, Any], +) -> None: + model_module, backend_config, _codex_backend, _azure_openai = isolate_backend_state + + assert model_module.set_backend("codex") == "codex" + + assert backend_config.get_optimizer_backend() == "codex_exec" + assert backend_config.get_target_backend() == "codex_exec" + assert model_module.get_backend_name() == "codex" + + +def test_chat_optimizer_routes_to_codex_backend( + monkeypatch: pytest.MonkeyPatch, + isolate_backend_state: tuple[Any, Any, Any, Any], +) -> None: + model_module, backend_config, codex_backend, azure_openai = isolate_backend_state + codex_calls: list[dict[str, Any]] = [] + + def fake_codex_optimizer(**kwargs: Any) -> tuple[str, dict[str, int]]: + codex_calls.append(kwargs) + return "codex result", { + "prompt_tokens": 1, + "completion_tokens": 2, + "total_tokens": 3, + } + + def fail_openai_optimizer(**_kwargs: Any) -> tuple[str, dict[str, int]]: + raise AssertionError("openai optimizer should not be called for codex_exec") + + monkeypatch.setattr(codex_backend, "chat_optimizer", fake_codex_optimizer) + monkeypatch.setattr(azure_openai, "chat_optimizer", fail_openai_optimizer) + backend_config.set_optimizer_backend("codex_exec") + + text, usage = model_module.chat_optimizer("system", "user", retries=1, timeout=5) + + assert text == "codex result" + assert usage["total_tokens"] == 3 + assert codex_calls[0]["system"] == "system" + assert codex_calls[0]["user"] == "user" + assert codex_calls[0]["timeout"] == 5 + + +def test_openai_compatible_still_allowed_as_optimizer_backend( + isolate_backend_state: tuple[Any, Any, Any, Any], +) -> None: + _model_module, backend_config, _codex_backend, _azure_openai = isolate_backend_state + + backend_config.set_optimizer_backend("openai_compatible") + + assert backend_config.get_optimizer_backend() == "openai_compatible" + + +def test_chat_optimizer_routes_to_openai_compatible_when_selected( + monkeypatch: pytest.MonkeyPatch, + isolate_backend_state: tuple[Any, Any, Any, Any], +) -> None: + model_module, backend_config, codex_backend, azure_openai = isolate_backend_state + from skillopt.model import openai_compatible_backend + + compat_calls: list[dict[str, Any]] = [] + + def fake_compat_optimizer(**kwargs: Any) -> tuple[str, dict[str, int]]: + compat_calls.append(kwargs) + return "compat result", {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + + def fail_codex_optimizer(**_kwargs: Any) -> tuple[str, dict[str, int]]: + raise AssertionError("codex optimizer should not be called for openai_compatible") + + def fail_openai_optimizer(**_kwargs: Any) -> tuple[str, dict[str, int]]: + raise AssertionError("openai optimizer should not be called for openai_compatible") + + monkeypatch.setattr(openai_compatible_backend, "chat_optimizer", fake_compat_optimizer) + monkeypatch.setattr(codex_backend, "chat_optimizer", fail_codex_optimizer) + monkeypatch.setattr(azure_openai, "chat_optimizer", fail_openai_optimizer) + backend_config.set_optimizer_backend("openai_compatible") + + text, usage = model_module.chat_optimizer("system", "user", retries=1, timeout=7) + + assert text == "compat result" + assert usage["total_tokens"] == 2 + assert compat_calls[0]["timeout"] == 7 + + +def test_chat_optimizer_messages_routes_to_codex_backend( + monkeypatch: pytest.MonkeyPatch, + isolate_backend_state: tuple[Any, Any, Any, Any], +) -> None: + model_module, backend_config, codex_backend, azure_openai = isolate_backend_state + codex_calls: list[dict[str, Any]] = [] + + def fake_codex_messages(**kwargs: Any) -> tuple[str, dict[str, int]]: + codex_calls.append(kwargs) + return "codex messages", {"prompt_tokens": 2, "completion_tokens": 3, "total_tokens": 5} + + def fail_openai_messages(**_kwargs: Any) -> tuple[str, dict[str, int]]: + raise AssertionError("openai messages should not be called for codex_exec") + + monkeypatch.setattr(codex_backend, "chat_optimizer_messages", fake_codex_messages) + monkeypatch.setattr(azure_openai, "chat_optimizer_messages", fail_openai_messages) + backend_config.set_optimizer_backend("codex_exec") + + text, usage = model_module.chat_optimizer_messages( + [{"role": "user", "content": "hi"}], + retries=1, + tools=[{"name": "lookup"}], + tool_choice="required", + return_message=True, + timeout=9, + ) + + assert text == "codex messages" + assert usage["total_tokens"] == 5 + assert codex_calls[0]["tools"] == [{"name": "lookup"}] + assert codex_calls[0]["tool_choice"] == "required" + assert codex_calls[0]["return_message"] is True + assert codex_calls[0]["timeout"] == 9 + + +def test_codex_optimizer_does_not_change_openai_target_routing( + monkeypatch: pytest.MonkeyPatch, + isolate_backend_state: tuple[Any, Any, Any, Any], +) -> None: + model_module, backend_config, codex_backend, azure_openai = isolate_backend_state + + def fake_openai_target(**_kwargs: Any) -> tuple[str, dict[str, int]]: + return "openai target", {"prompt_tokens": 1, "completion_tokens": 0, "total_tokens": 1} + + def fail_codex_target(**_kwargs: Any) -> tuple[str, dict[str, int]]: + raise AssertionError("codex target should not be called when target_backend=openai_chat") + + monkeypatch.setattr(azure_openai, "chat_target", fake_openai_target) + monkeypatch.setattr(codex_backend, "chat_target", fail_codex_target) + backend_config.set_optimizer_backend("codex_exec") + backend_config.set_target_backend("openai_chat") + + text, usage = model_module.chat_target("system", "user", retries=1) + + assert text == "openai target" + assert usage["total_tokens"] == 1 + + +def test_get_backend_name_keeps_openai_compatible( + isolate_backend_state: tuple[Any, Any, Any, Any], +) -> None: + model_module, backend_config, _codex_backend, _azure_openai = isolate_backend_state + + backend_config.set_optimizer_backend("openai_compatible") + backend_config.set_target_backend("openai_compatible") + + assert model_module.get_backend_name() == "openai_compatible" + + +def test_token_summary_merges_codex_once_with_existing_backends( + monkeypatch: pytest.MonkeyPatch, + isolate_backend_state: tuple[Any, Any, Any, Any], +) -> None: + model_module, _backend_config, codex_backend, azure_openai = isolate_backend_state + from skillopt.model import claude_backend, minimax_backend, openai_compatible_backend, qwen_backend + + empty = {"_total": {"calls": 0, "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}} + monkeypatch.setattr(azure_openai, "get_token_summary", lambda: empty.copy()) + monkeypatch.setattr(claude_backend, "get_token_summary", lambda: empty.copy()) + monkeypatch.setattr(qwen_backend, "get_token_summary", lambda: empty.copy()) + monkeypatch.setattr(minimax_backend, "get_token_summary", lambda: empty.copy()) + monkeypatch.setattr( + openai_compatible_backend, + "get_token_summary", + lambda: {"optimizer": {"calls": 1, "prompt_tokens": 5, "completion_tokens": 7, "total_tokens": 12}}, + ) + monkeypatch.setattr( + codex_backend, + "get_token_summary", + lambda: {"optimizer": {"calls": 1, "prompt_tokens": 11, "completion_tokens": 13, "total_tokens": 24}}, + ) + + summary = model_module.get_token_summary() + + assert summary["optimizer"]["calls"] == 2 + assert summary["optimizer"]["prompt_tokens"] == 16 + assert summary["optimizer"]["completion_tokens"] == 20 + assert summary["_total"]["total_tokens"] == 36 + + +def test_reset_token_tracker_resets_codex_and_existing_backends( + monkeypatch: pytest.MonkeyPatch, + isolate_backend_state: tuple[Any, Any, Any, Any], +) -> None: + model_module, _backend_config, codex_backend, azure_openai = isolate_backend_state + from skillopt.model import claude_backend, minimax_backend, openai_compatible_backend, qwen_backend + + called: list[str] = [] + for name, module in [ + ("openai", azure_openai), + ("claude", claude_backend), + ("qwen", qwen_backend), + ("minimax", minimax_backend), + ("compat", openai_compatible_backend), + ("codex", codex_backend), + ]: + monkeypatch.setattr(module, "reset_token_tracker", lambda name=name: called.append(name)) + + model_module.reset_token_tracker() + + assert called == ["openai", "claude", "qwen", "minimax", "compat", "codex"] + + +def test_set_reasoning_effort_updates_codex_and_existing_backends( + monkeypatch: pytest.MonkeyPatch, + isolate_backend_state: tuple[Any, Any, Any, Any], +) -> None: + model_module, _backend_config, codex_backend, azure_openai = isolate_backend_state + from skillopt.model import claude_backend, minimax_backend, openai_compatible_backend, qwen_backend + + called: list[tuple[str, str]] = [] + for name, module in [ + ("openai", azure_openai), + ("claude", claude_backend), + ("qwen", qwen_backend), + ("minimax", minimax_backend), + ("compat", openai_compatible_backend), + ("codex", codex_backend), + ]: + monkeypatch.setattr(module, "set_reasoning_effort", lambda effort, name=name: called.append((name, effort))) + + model_module.set_reasoning_effort("high") + + assert called == [ + ("openai", "high"), + ("claude", "high"), + ("qwen", "high"), + ("minimax", "high"), + ("compat", "high"), + ("codex", "high"), + ] + + +def test_deployment_setters_update_codex_without_dropping_openai_compatible( + monkeypatch: pytest.MonkeyPatch, + isolate_backend_state: tuple[Any, Any, Any, Any], +) -> None: + model_module, _backend_config, codex_backend, _azure_openai = isolate_backend_state + from skillopt.model import openai_compatible_backend + + called: list[tuple[str, str, str]] = [] + monkeypatch.setattr( + openai_compatible_backend, + "set_target_deployment", + lambda deployment: called.append(("compat", "target", deployment)), + ) + monkeypatch.setattr( + openai_compatible_backend, + "set_optimizer_deployment", + lambda deployment: called.append(("compat", "optimizer", deployment)), + ) + monkeypatch.setattr( + codex_backend, + "set_target_deployment", + lambda deployment: called.append(("codex", "target", deployment)), + ) + monkeypatch.setattr( + codex_backend, + "set_optimizer_deployment", + lambda deployment: called.append(("codex", "optimizer", deployment)), + ) + + model_module.set_target_deployment("target-model") + model_module.set_optimizer_deployment("optimizer-model") + + assert ("compat", "target", "target-model") in called + assert ("codex", "target", "target-model") in called + assert ("compat", "optimizer", "optimizer-model") in called + assert ("codex", "optimizer", "optimizer-model") in called