From 0381332c5166dfe2482ab2d53c1ec47585e5b234 Mon Sep 17 00:00:00 2001 From: Y1fe1Zh0u Date: Tue, 11 Aug 2026 21:29:01 +0800 Subject: [PATCH 1/2] Give Tool repair paths one consistent retry ceiling Set the existing protocol repair, safe-read replay, and model-visible Tool episode limits to ten while preserving their current independent state and execution semantics. Update focused tests and planning artifacts to make the off-by-one behavior explicit. Constraint: Tool-related retry and repair limits must be ten without restructuring the existing counters Rejected: Unify protocol, Receipt, and model-visible repair state now | counter redesign is intentionally deferred Confidence: high Scope-risk: moderate Directive: Keep the independent counters until the planned repair-control refactor; do not infer identical attempt semantics from the shared numeric limit Tested: 911 Runtime and Tool pytest cases; scoped Ruff; fatal-level caller Ruff; py_compile; git diff --check Not-tested: Live Provider credentials --- .../services/agent_runtime/node_executor.py | 2 ++ .../services/agent_runtime/tool_execution.py | 2 +- .../agent_runtime/tool_repair_budget.py | 2 +- backend/app/services/llm/caller.py | 4 +-- .../test_agent_runtime_model_step_service.py | 2 +- .../tests/test_agent_runtime_node_executor.py | 35 +++++++++---------- .../test_agent_runtime_tool_repair_budget.py | 4 +-- .../test_agent_runtime_tool_step_service.py | 4 +-- backend/tests/test_finish_protocol.py | 10 +++--- backend/tests/test_tool_execution.py | 5 ++- .../checklists/requirements.md | 2 +- .../contracts/repair-and-lifecycle.md | 3 +- specs/002-tool-runtime-contract/plan.md | 6 ++-- specs/002-tool-runtime-contract/quickstart.md | 4 +-- specs/002-tool-runtime-contract/research.md | 2 +- specs/002-tool-runtime-contract/spec.md | 13 +++---- specs/002-tool-runtime-contract/tasks.md | 8 ++--- 17 files changed, 57 insertions(+), 51 deletions(-) diff --git a/backend/app/services/agent_runtime/node_executor.py b/backend/app/services/agent_runtime/node_executor.py index 9f9f8024f..8a0a00893 100644 --- a/backend/app/services/agent_runtime/node_executor.py +++ b/backend/app/services/agent_runtime/node_executor.py @@ -778,6 +778,8 @@ async def _model( repair_limit = ( WRITE_FILE_PROTOCOL_REPAIR_LIMIT if is_write_file_repair + else 10 + if repair_code == "invalid_tool_call" else 1 ) repair_counter_key = ( diff --git a/backend/app/services/agent_runtime/tool_execution.py b/backend/app/services/agent_runtime/tool_execution.py index 2266c5ba7..161b552a2 100644 --- a/backend/app/services/agent_runtime/tool_execution.py +++ b/backend/app/services/agent_runtime/tool_execution.py @@ -44,7 +44,7 @@ "reconcile", ] ToolSideEffectState = Literal["none", "confirmed", "possible", "unknown"] -SAFE_READ_MAX_ATTEMPTS = 3 +SAFE_READ_MAX_ATTEMPTS = 10 # These tools dispatch an external image-generation request and can therefore # leave the provider outcome uncertain after a response timeout. Direct Chat diff --git a/backend/app/services/agent_runtime/tool_repair_budget.py b/backend/app/services/agent_runtime/tool_repair_budget.py index 91d9abe84..9077371b7 100644 --- a/backend/app/services/agent_runtime/tool_repair_budget.py +++ b/backend/app/services/agent_runtime/tool_repair_budget.py @@ -10,7 +10,7 @@ from app.services.agent_runtime.state import JsonObject SAME_FINGERPRINT_FAILURE_LIMIT = 10 -TOOL_EPISODE_FAILURE_LIMIT = 20 +TOOL_EPISODE_FAILURE_LIMIT = 10 _REPAIRABLE_MODEL_ACTIONS = frozenset( {"repair_arguments", "choose_other_tool"} ) diff --git a/backend/app/services/llm/caller.py b/backend/app/services/llm/caller.py index f854bc06a..fcc9809e9 100644 --- a/backend/app/services/llm/caller.py +++ b/backend/app/services/llm/caller.py @@ -66,7 +66,7 @@ async def execute_tool(*args, **kwargs): "send_message_to_agent", "send_feishu_message", "send_email" }) -WRITE_FILE_PROTOCOL_REPAIR_LIMIT = 3 +WRITE_FILE_PROTOCOL_REPAIR_LIMIT = 10 WRITE_FILE_PROTOCOL_REPAIR_COUNTER_KEY = "invalid_tool_call:write_file" WRITE_FILE_PROTOCOL_REPAIR_INSTRUCTION = ( "Your previous `write_file` call was not executed because `function.arguments` " @@ -788,7 +788,7 @@ async def _buffer_chunk(_text: str) -> None: repair_limit = ( WRITE_FILE_PROTOCOL_REPAIR_LIMIT if retry_tool_name == "write_file" - else 1 + else 10 ) repair_counter_key = ( WRITE_FILE_PROTOCOL_REPAIR_COUNTER_KEY diff --git a/backend/tests/test_agent_runtime_model_step_service.py b/backend/tests/test_agent_runtime_model_step_service.py index 16cc1c390..767659f9d 100644 --- a/backend/tests/test_agent_runtime_model_step_service.py +++ b/backend/tests/test_agent_runtime_model_step_service.py @@ -598,7 +598,7 @@ async def complete(model_arg, _messages, **_kwargs): @pytest.mark.asyncio -async def test_invalid_write_file_arguments_request_three_protocol_repairs() -> None: +async def test_invalid_write_file_arguments_request_ten_protocol_repairs() -> None: tenant_id = uuid.uuid4() model = _model(tenant_id) agent = _agent(tenant_id) diff --git a/backend/tests/test_agent_runtime_node_executor.py b/backend/tests/test_agent_runtime_node_executor.py index 74d2634e9..19fd4aa3a 100644 --- a/backend/tests/test_agent_runtime_node_executor.py +++ b/backend/tests/test_agent_runtime_node_executor.py @@ -1351,15 +1351,16 @@ async def test_empty_output_is_repaired_once_then_fails_explicitly() -> None: @pytest.mark.asyncio @pytest.mark.parametrize( - ("repair_code", "instruction"), + ("repair_code", "instruction", "repair_limit"), [ - ("invalid_finish", "Retry finish with valid content."), - ("invalid_tool_call", "Retry with valid JSON tool arguments."), + ("invalid_finish", "Retry finish with valid content.", 1), + ("invalid_tool_call", "Retry with valid JSON tool arguments.", 10), ], ) async def test_repeated_model_tool_protocol_repair_code_fails_explicitly( repair_code: str, instruction: str, + repair_limit: int, ) -> None: run_id = uuid.uuid4() repair = ModelStepResult( @@ -1368,7 +1369,7 @@ async def test_repeated_model_tool_protocol_repair_code_fails_explicitly( repair_instruction=instruction, repair_code=repair_code, ) - model = ModelService(repair, repair) + model = ModelService(*([repair] * (repair_limit + 1))) executor = _executor(model) result = await _invoke(run_id, executor, model_turn_limit=50) @@ -1377,13 +1378,13 @@ async def test_repeated_model_tool_protocol_repair_code_fails_explicitly( assert lifecycle["status"] == "failed" assert lifecycle["reason"] == "model_tool_protocol_violation" assert lifecycle["error"]["code"] == "model_tool_protocol_violation" - assert lifecycle["model_protocol_repairs"] == {repair_code: 1} - assert lifecycle["model_step_count"] == 2 - assert model.calls == 2 + assert lifecycle["model_protocol_repairs"] == {repair_code: repair_limit} + assert lifecycle["model_step_count"] == repair_limit + 1 + assert model.calls == repair_limit + 1 @pytest.mark.asyncio -async def test_write_file_protocol_repair_uses_three_attempts_then_guides_user() -> None: +async def test_write_file_protocol_repair_uses_ten_attempts_then_guides_user() -> None: run_id = uuid.uuid4() repair = ModelStepResult( intent="text", @@ -1392,7 +1393,7 @@ async def test_write_file_protocol_repair_uses_three_attempts_then_guides_user() repair_code="invalid_tool_call", repair_tool_name="write_file", ) - model = ModelService(repair, repair, repair, repair) + model = ModelService(*([repair] * 11)) executor = _executor(model) result = await _invoke(run_id, executor, model_turn_limit=50) @@ -1408,14 +1409,14 @@ async def test_write_file_protocol_repair_uses_three_attempts_then_guides_user() ), } assert lifecycle["model_protocol_repairs"] == { - "invalid_tool_call:write_file": 3, + "invalid_tool_call:write_file": 10, } - assert lifecycle["model_step_count"] == 4 - assert model.calls == 4 + assert lifecycle["model_step_count"] == 11 + assert model.calls == 11 @pytest.mark.asyncio -async def test_write_file_protocol_can_recover_on_the_third_repair() -> None: +async def test_write_file_protocol_can_recover_on_the_tenth_repair() -> None: run_id = uuid.uuid4() repair = ModelStepResult( intent="text", @@ -1424,9 +1425,7 @@ async def test_write_file_protocol_can_recover_on_the_third_repair() -> None: repair_tool_name="write_file", ) model = ModelService( - repair, - repair, - repair, + *([repair] * 10), ModelStepResult(intent="finish", finish_content="Recovered"), ) executor = _executor(model) @@ -1435,9 +1434,9 @@ async def test_write_file_protocol_can_recover_on_the_third_repair() -> None: assert result["lifecycle"]["status"] == "completed" assert result["lifecycle"]["model_protocol_repairs"] == { - "invalid_tool_call:write_file": 3, + "invalid_tool_call:write_file": 10, } - assert model.calls == 4 + assert model.calls == 11 @pytest.mark.asyncio diff --git a/backend/tests/test_agent_runtime_tool_repair_budget.py b/backend/tests/test_agent_runtime_tool_repair_budget.py index a2779f06b..2e9916216 100644 --- a/backend/tests/test_agent_runtime_tool_repair_budget.py +++ b/backend/tests/test_agent_runtime_tool_repair_budget.py @@ -50,7 +50,7 @@ def test_tenth_consecutive_fingerprint_pauses_without_off_by_one() -> None: assert _episode(state)["total_failures"] == 10 -def test_twentieth_tool_failure_pauses_even_when_fingerprint_changes() -> None: +def test_tenth_tool_failure_pauses_even_when_fingerprint_changes() -> None: state: dict = {} transition = None for model_step in range(1, TOOL_EPISODE_FAILURE_LIMIT + 1): @@ -63,7 +63,7 @@ def test_twentieth_tool_failure_pauses_even_when_fingerprint_changes() -> None: assert transition is not None assert transition.pause_reason == "tool_repair_episode_limit_reached" - assert _episode(state)["total_failures"] == 20 + assert _episode(state)["total_failures"] == 10 assert _episode(state)["same_fingerprint_failures"] == 1 diff --git a/backend/tests/test_agent_runtime_tool_step_service.py b/backend/tests/test_agent_runtime_tool_step_service.py index d8dcfce22..a63e1f26b 100644 --- a/backend/tests/test_agent_runtime_tool_step_service.py +++ b/backend/tests/test_agent_runtime_tool_step_service.py @@ -2977,7 +2977,7 @@ async def test_retryable_read_exhaustion_returns_one_non_retryable_result( "call-read-exhausted", "read_file", ) - execution.attempt_count = 3 + execution.attempt_count = 10 async def reserve(db, **kwargs): del db @@ -3017,7 +3017,7 @@ async def mark_failed(db, **kwargs): assert "Do not repeat the identical tool call unchanged" in result.messages[0][ "content" ] - assert execution.result_metadata["runtime_attempt_count"] == 3 + assert execution.result_metadata["runtime_attempt_count"] == 10 assert execution.result_metadata["runtime_retry_exhausted"] is True assert execution.result_metadata["last_error_code"] == "temporary_read_failure" diff --git a/backend/tests/test_finish_protocol.py b/backend/tests/test_finish_protocol.py index a63f0497a..e61e66227 100644 --- a/backend/tests/test_finish_protocol.py +++ b/backend/tests/test_finish_protocol.py @@ -811,7 +811,7 @@ async def test_repeated_invalid_tool_json_is_bounded_by_protocol_code(monkeypatc } ], ) - fake_client = FakeStreamClient([invalid, invalid]) + fake_client = FakeStreamClient([invalid] * 11) monkeypatch.setattr(caller, "_get_agent_config", lambda _agent_id: _async_return((50, None))) monkeypatch.setattr(caller, "_get_user_name", lambda _user_id: _async_return("Ray")) monkeypatch.setattr( @@ -841,12 +841,12 @@ async def test_repeated_invalid_tool_json_is_bounded_by_protocol_code(monkeypatc ) assert result.startswith("[Error] invalid_tool_call_protocol_violation:") - assert len(fake_client.messages_seen) == 2 + assert len(fake_client.messages_seen) == 11 assert fake_client.closed is True @pytest.mark.asyncio -async def test_invalid_write_file_json_gets_three_bounded_repairs(monkeypatch): +async def test_invalid_write_file_json_gets_ten_bounded_repairs(monkeypatch): from app.services.llm import caller from app.services.llm.client import LLMResponse @@ -863,7 +863,7 @@ async def test_invalid_write_file_json_gets_three_bounded_repairs(monkeypatch): } ], ) - fake_client = FakeStreamClient([invalid, invalid, invalid, invalid]) + fake_client = FakeStreamClient([invalid] * 11) monkeypatch.setattr(caller, "_get_agent_config", lambda _agent_id: _async_return((50, None))) monkeypatch.setattr(caller, "_get_user_name", lambda _user_id: _async_return("Ray")) monkeypatch.setattr( @@ -897,7 +897,7 @@ async def test_invalid_write_file_json_gets_three_bounded_repairs(monkeypatch): "本次文件生成未完成：write_file 工具参数无效或被截断，连续重试后仍无法执行。" "请回复「重新生成」，我会基于当前对话重新尝试。" ) - assert len(fake_client.messages_seen) == 4 + assert len(fake_client.messages_seen) == 11 assert fake_client.closed is True diff --git a/backend/tests/test_tool_execution.py b/backend/tests/test_tool_execution.py index 8932247bc..37157be5f 100644 --- a/backend/tests/test_tool_execution.py +++ b/backend/tests/test_tool_execution.py @@ -881,7 +881,10 @@ async def test_expired_final_safe_read_attempt_closes_without_provider_replay(): assert reservation.prior_failure is not None assert reservation.prior_failure.error_code == "tool_retry_exhausted" assert execution.status == "failed" - assert execution.result_metadata["runtime_attempt_count"] == 3 + assert ( + execution.result_metadata["runtime_attempt_count"] + == tool_execution.SAFE_READ_MAX_ATTEMPTS + ) assert execution.result_metadata["runtime_retry_exhausted"] is True assert db.flush_count == 1 diff --git a/specs/002-tool-runtime-contract/checklists/requirements.md b/specs/002-tool-runtime-contract/checklists/requirements.md index 8c377ca7b..9c4a731f5 100644 --- a/specs/002-tool-runtime-contract/checklists/requirements.md +++ b/specs/002-tool-runtime-contract/checklists/requirements.md @@ -33,4 +33,4 @@ - 第一次校验即通过，无 `[NEEDS CLARIFICATION]` 项。 - `Tool Call`、`Run`、`Receipt`、`checkpoint` 等词是本产品领域对象，不是具体实现方案；具体数据结构、文件和迁移步骤将在 Plan 阶段定义。 -- Spec 已覆盖用户确认的 10/20 repair budget、模型可见错误反馈、unknown write 禁止自动重放和旧 checkpoint 兼容边界。 +- Spec 已覆盖用户确认的 Tool repair/retry 上限统一为 10、模型可见错误反馈、unknown write 禁止自动重放和旧 checkpoint 兼容边界；计数结构统一重构已明确延期。 diff --git a/specs/002-tool-runtime-contract/contracts/repair-and-lifecycle.md b/specs/002-tool-runtime-contract/contracts/repair-and-lifecycle.md index 9524e391d..dc6db0791 100644 --- a/specs/002-tool-runtime-contract/contracts/repair-and-lifecycle.md +++ b/specs/002-tool-runtime-contract/contracts/repair-and-lifecycle.md @@ -3,7 +3,8 @@ ## Tool Repair Episode - `same_fingerprint_failures` reaches 10: pause immediately after recording the 10th failure; do not invoke model step 11 for that loop. -- `total_failures` reaches 20 for the same Tool episode: pause immediately; do not invoke the next model step. +- `total_failures` reaches 10 for the same Tool episode: pause immediately; do not invoke the next model step. +- Generic Tool protocol repair, `write_file` protocol repair, and safe-read replay retain their current independent counters but each uses a limit of 10; counter unification is deferred. - Changing fingerprint resets only the consecutive counter. - Success of the same Tool, new Run, or explicit user correction resets the Tool episode. - Success of another Tool does not reset it. diff --git a/specs/002-tool-runtime-contract/plan.md b/specs/002-tool-runtime-contract/plan.md index fc0227bf2..50d3fef64 100644 --- a/specs/002-tool-runtime-contract/plan.md +++ b/specs/002-tool-runtime-contract/plan.md @@ -5,7 +5,7 @@ ## Summary -在现有 Durable Runtime、`AgentToolExecution` Receipt、safe-read replay 和 unknown/reconcile 机制之上，增加一次 Model Step 固化、checkpoint 可恢复的 `StepToolContext`。新 Tool Step 只使用已接受的 Tool Contract/Execution Binding，不再调用 ToolProvider 重建 Workset；同时把 Provider Call ID、Runtime Call Instance 和 Execution Receipt 分离，统一 schema validation、authorization/approval、模型可见失败反馈及 10/20 repair budget。操作 deadline、取消传播和 Receipt lease 继续保持三个独立控制面。长期通过可渐进迁移的 RegisteredTool 收敛模型定义与执行能力，不一次性替换现有 Handler。 +在现有 Durable Runtime、`AgentToolExecution` Receipt、safe-read replay 和 unknown/reconcile 机制之上，增加一次 Model Step 固化、checkpoint 可恢复的 `StepToolContext`。新 Tool Step 只使用已接受的 Tool Contract/Execution Binding，不再调用 ToolProvider 重建 Workset；同时把 Provider Call ID、Runtime Call Instance 和 Execution Receipt 分离，统一 schema validation、authorization/approval、模型可见失败反馈，并将现有独立 Tool repair/retry 上限统一为 10。操作 deadline、取消传播和 Receipt lease 继续保持三个独立控制面。长期通过可渐进迁移的 RegisteredTool 收敛模型定义与执行能力，不一次性替换现有 Handler。 ## Technical Context @@ -50,7 +50,7 @@ ### Phase C — Repair budgets 1. checkpoint 保存 per-tool repair episode、连续 fingerprint 计数和总计数。 -2. 第 10 次连续相同失败或第 20 次同 Tool episode 失败后暂停，且不发起下一次模型调用。 +2. 第 10 次连续相同失败或第 10 次同 Tool episode 失败后暂停，且不发起下一次模型调用；普通 Tool JSON repair、`write_file` JSON repair 和 safe-read replay 也只把现有独立上限改为 10，不在本轮重构计数结构。 3. Tool 成功、新 Run、用户明确纠正按 contract 重置；Provider retry、safe internal replay、permission/confirmation、pending、cancel、unknown 不计数。 4. Verifier repair 改为当前 issue episode 计数，保留全局 `model_turn_limit` 独立语义。 @@ -122,7 +122,7 @@ backend/ 2. Runtime integration tests：Model Step → checkpoint → 新 Worker Tool Step；普通 availability 变化不影响已接受 Call；安全状态变化仍阻断。 3. Receipt tests：replay 复用同一 execution；lease renewal/loss/fence；unknown write no replay；safe read bounded retry。 4. Compatibility tests：旧 checkpoint 单次 resolver、新 checkpoint 禁止 resolver、mixed-version nullable fields。 -5. Lifecycle tests：10/20 off-by-one、reset/exclusion、operation deadline、cancel propagation。 +5. Lifecycle tests：统一上限 10 的 off-by-one、reset/exclusion、operation deadline、cancel propagation。 6. Static gates：scoped Ruff、pytest、Alembic single head + upgrade/downgrade、`scripts/arch-guard.sh`。 ## Complexity Tracking diff --git a/specs/002-tool-runtime-contract/quickstart.md b/specs/002-tool-runtime-contract/quickstart.md index dbdff9656..40c60830f 100644 --- a/specs/002-tool-runtime-contract/quickstart.md +++ b/specs/002-tool-runtime-contract/quickstart.md @@ -26,7 +26,7 @@ Expected branch: `002-tool-runtime-contract`; base contains `upstream/main@251ae 3. Remove ToolProvider access from new-format Tool Step; add legacy batch resolver. 4. Add DB columns/migration and projection metadata. 5. Add shared validation/authorization/failure envelope. -6. Add repair episode state and 10/20 gates. +6. Add repair episode state and uniform Tool repair/retry limit 10 gates. 7. Harden operation deadlines/cancel/lease tests. 8. Add RegisteredTool boundary and migrate representative tools only. @@ -66,7 +66,7 @@ cd backend - checkpoint restart on another Worker uses the same binding and execution row; - repeated Provider-local ID in another Assistant Turn does not collide; - schema failure returns exactly one sanitized Tool Result; -- failure 10 and 20 pause before the next model invocation; +- the 10th repair failure pauses before the next model invocation; - provider retry, safe replay, pending, cancel and unknown do not increment repair budget; - lease loss blocks stale settlement; uncertain write is never auto-replayed; - legacy checkpoint resolves once per pending batch, new checkpoint never uses legacy fallback. diff --git a/specs/002-tool-runtime-contract/research.md b/specs/002-tool-runtime-contract/research.md index 359f331da..d8c1591f2 100644 --- a/specs/002-tool-runtime-contract/research.md +++ b/specs/002-tool-runtime-contract/research.md @@ -56,7 +56,7 @@ ### D7. Repair budget 是 Tool episode，不是 Provider/Receipt retry -**Decision**: 连续同 fingerprint 第 10 次、同 Tool episode 第 20 次暂停；只计模型可见、可修复失败。 +**Decision**: 连续同 fingerprint 第 10 次、同 Tool episode 第 10 次暂停；只计模型可见、可修复失败。普通 Tool protocol repair、`write_file` protocol repair 和 safe-read replay 继续使用各自现有计数入口，但上限统一为 10，状态结构后续再整体重构。 **Rationale**: Provider transport retry 和 Receipt safe replay 都不代表模型做了错误决策；混计会过早停机或掩盖循环。 diff --git a/specs/002-tool-runtime-contract/spec.md b/specs/002-tool-runtime-contract/spec.md index 092302cef..807f362de 100644 --- a/specs/002-tool-runtime-contract/spec.md +++ b/specs/002-tool-runtime-contract/spec.md @@ -70,11 +70,12 @@ **Acceptance Scenarios**: 1. **Given** 同一稳定错误已经连续作为模型可见失败出现 9 次，**When** 第 10 次相同失败被记录，**Then** 系统保存该失败并暂停，不能开始第 11 次模型调用。 -2. **Given** 同一个 Tool 在当前 episode 中出现 19 次可计数失败，错误指纹可以变化，**When** 第 20 次失败被记录，**Then** 系统暂停，不能开始下一次模型调用。 +2. **Given** 同一个 Tool 在当前 episode 中出现 9 次可计数失败，错误指纹可以变化，**When** 第 10 次失败被记录，**Then** 系统暂停，不能开始下一次模型调用。 3. **Given** 失败指纹变化但 Tool 相同，**When** 记录新失败，**Then** 连续相同错误计数重新开始，但同 Tool episode 总数保留。 4. **Given** 被跟踪 Tool 成功、新 Run 开始，或用户明确纠正后恢复，**When** 后续再发生失败，**Then** 按对应规则开启新的 repair episode。 5. **Given** 事件属于 Provider transport retry、安全内部 replay、permission/confirmation wait、async pending、cancel 或 unknown external write，**When** 系统处理事件，**Then** 不增加模型修复计数。 6. **Given** 全局 Run 模型轮次已经达到上限，**When** 本地 Tool repair budget 尚未耗尽，**Then** 全局上限仍独立生效并展示不同的停止原因。 +7. **Given** 普通 Tool 或 `write_file` 的 arguments JSON 无效或截断，**When** Runtime 请求模型修复，**Then** 两类 Tool 都分别最多提供 10 次重写机会；safe-read Runtime replay 最多执行同一调用 10 次。本轮只统一上限数值，不重构这些独立计数器。 --- @@ -121,7 +122,7 @@ - 管理员关闭 Tool，但当前已接受调用仍在等待人工确认；用户随后拒绝、接受或取消。 - Safe-read 内部 retry 已耗尽，最终只应产生一次模型可见失败和一次 repair 计数。 - Tool Result 已写入 checkpoint，但节点被重新调度；结果消息和 repair counter 不能重复追加。 -- 同一 Tool 在不同错误之间交替，连续相同错误计数不断重置，但同 Tool episode 最终达到 20。 +- 同一 Tool 在不同错误之间交替，连续相同错误计数不断重置，但同 Tool episode 最终达到 10。 - Unknown external write 在重启、重连、用户输入或模型继续推理时仍不得自动重放。 - Handler 完成时 lease 已丢失；旧 owner 不能覆盖新 owner 或绕过 fence 结算。 - 底层线程调用无法真正取消；系统必须停止等待并明确记录底层取消能力限制。 @@ -148,10 +149,10 @@ - **FR-015**: Permission/confirmation、async pending、cancel、unknown external write 和协议损坏 MUST 使用各自独立状态，不得伪装成普通可修复 Tool failure。 - **FR-016**: Unknown possible write MUST 阻止自动重放，直到通过外部查询、稳定幂等结果或明确人工处理完成协调。 - **FR-017**: 系统 MUST 在第 10 次连续相同且模型可见的可修复失败后暂停，并且 MUST NOT 启动第 11 次模型调用。 -- **FR-018**: 系统 MUST 在同一个 Tool repair episode 的第 20 次可计数失败后暂停，并且 MUST NOT 启动下一次模型调用。 +- **FR-018**: 系统 MUST 在同一个 Tool repair episode 的第 10 次可计数失败后暂停，并且 MUST NOT 启动下一次模型调用。 - **FR-019**: 不同错误指纹 MUST 只重置连续相同错误计数，不得清除同 Tool episode 总数。 - **FR-020**: 对应 Tool 成功、新 Run 或用户明确纠正后恢复 MUST 按定义重置 repair episode；无关 Tool 成功不得清除其他 Tool 的失败 episode。 -- **FR-021**: Provider transport retry、安全内部 replay、permission/confirmation wait、async pending、cancel 和 unknown external write MUST NOT 增加模型修复计数。 +- **FR-021**: Provider transport retry、安全内部 replay、permission/confirmation wait、async pending、cancel 和 unknown external write MUST NOT 增加模型修复计数。普通 Tool protocol repair、`write_file` protocol repair 和 safe-read replay 保留独立计数结构，但各自上限 MUST 统一为 10；计数结构重构不属于本轮改动。 - **FR-022**: 全局模型轮次上限 MUST 与 Tool repair budget、Provider retry、Command retry 和 Verifier repair 保持独立，并报告不同停止原因。 - **FR-023**: Verifier repair MUST 按当前问题 episode 计数；历史已结束问题不得耗尽新的 verifier episode。 - **FR-024**: 外部 I/O 和长时间操作 MUST 具有与具体操作匹配的最长等待规则；系统 MUST NOT 用单一固定秒数替代所有 Tool 的时限。 @@ -183,7 +184,7 @@ - **SC-002**: 在所有受支持 Provider 的多轮 Tool 场景中，重复的 Provider-local Call ID 产生 0 次执行记录、Tool Result、Activity、Chat 或 A2A correlation 碰撞。 - **SC-003**: 同一调用实例在至少一次 checkpoint replay 后仍只产生一条有效执行记录；未知外部写的自动重放次数为 0。 - **SC-004**: 100% 带有效身份的可修复参数、binding 和明确业务失败产生恰好一个模型可见 Tool Result；敏感信息泄漏测试通过率为 100%。 -- **SC-005**: 第 10 次连续相同错误和第 20 次同 Tool episode 失败均在规定边界暂停，所有 off-by-one、reset 和 exclusion 测试通过率为 100%。 +- **SC-005**: 第 10 次连续相同错误和第 10 次同 Tool episode 失败均在规定边界暂停；普通 Tool JSON repair、`write_file` JSON repair 和 safe-read replay 的独立上限均为 10；所有 off-by-one、reset 和 exclusion 测试通过率为 100%。 - **SC-006**: Permission、confirmation、pending、cancel、unknown write、Provider retry 和全局模型轮次上限均显示独立原因，测试中不存在跨预算误计数。 - **SC-007**: 所有列入范围的 IMAP、DNS、AgentBay read 和代码执行路径在配置的最长等待内返回结果或明确状态，不产生无限等待测试用例。 - **SC-008**: 长时间 Handler 的 lease renewal、lease loss 和 cancel 测试均不会产生并发双执行或旧 owner 越权结算。 @@ -198,4 +199,4 @@ - 完整 Provider Schema capability matrix、默认 Tool 集合收窄、通用 Tool Search 和通用并行执行不属于本功能。 - 未迁移的 AgentBay Action 继续保持隐藏，后续按 Tool family 分批迁移。 - 旧 checkpoint 兼容路径只在有观测证据证明不再使用后删除。 -- 用户已经确定 repair budget 为：连续相同错误 10 次、同 Tool episode 20 次，并保留独立的全局 Run 模型轮次上限。 +- 用户已经确定所有 Tool 相关 repair/retry 上限统一为 10，并保留独立的计数结构与全局 Run 模型轮次上限；计数结构后续统一重构。 diff --git a/specs/002-tool-runtime-contract/tasks.md b/specs/002-tool-runtime-contract/tasks.md index ad234a195..965a3ab40 100644 --- a/specs/002-tool-runtime-contract/tasks.md +++ b/specs/002-tool-runtime-contract/tasks.md @@ -106,13 +106,13 @@ ## Phase 6: User Story 4 — 修复次数按问题边界计算 (Priority: P2) -**Goal**: 实现连续同错 10、同 Tool episode 20，并与其他 retry budget 分离。 +**Goal**: 实现连续同错 10、同 Tool episode 10，并将现有独立 Tool repair/retry 上限统一为 10；本轮不重构计数结构。 -**Independent Test**: 10/20 边界、fingerprint 变化、Tool success、新 Run、用户纠正、无关 Tool success 及所有 exclusion 均按 contract 转移。 +**Independent Test**: 统一上限 10 的边界、fingerprint 变化、Tool success、新 Run、用户纠正、无关 Tool success及所有 exclusion 均按 contract 转移。 ### Tests -- [x] T035 [P] [US4] 在 `backend/tests/test_agent_runtime_tool_repair_budget.py` 增加 10/20 off-by-one 与 fingerprint 测试 +- [x] T035 [P] [US4] 在 `backend/tests/test_agent_runtime_tool_repair_budget.py` 增加统一上限 10 的 off-by-one 与 fingerprint 测试 - [x] T036 [P] [US4] 在 `backend/tests/test_agent_runtime_tool_repair_budget.py` 增加 success/new Run/user correction/reset scope 测试 - [x] T037 [P] [US4] 在 `backend/tests/test_agent_runtime_tool_repair_budget.py` 增加 Provider retry/safe replay/approval/pending/cancel/unknown exclusion 测试 - [x] T038 [P] [US4] 在 `backend/tests/test_agent_runtime_node_executor.py` 增加暂停发生在下一次 Model 调用之前的集成测试 @@ -237,7 +237,7 @@ T012 live safety revocation tests 1. US1 消除已接受 Call 的 Workset 漂移。 2. US2/US3 补齐模型可修复反馈和身份兼容。 -3. US4 落地 10/20 修复次数。 +3. US4 落地统一上限 10 的修复次数，保留现有独立计数结构。 4. US5 加固长任务生命周期。 5. US6 建立长期 Registry 迁移边界。 From afaa19103eb1d9dcce444819381a13eefc8692d8 Mon Sep 17 00:00:00 2001 From: Y1fe1Zh0u Date: Wed, 12 Aug 2026 16:58:44 +0800 Subject: [PATCH 2/2] Keep Runtime deadlines outside provider budgets Network reads and image generation now carry deadline policies that cover their declared provider timeout. Custom image generation retains its existing 600 second configurable upper bound instead of being cancelled by the generic Runtime default. Constraint: Provider-specific internal timeouts may remain shorter, but the Runtime wrapper must not expire first. Rejected: Read live Tool configuration inside the Tool Step deadline resolver | would reintroduce mutable routing policy after call acceptance. Confidence: high Scope-risk: narrow Directive: Keep builtin timeout declarations and Runtime deadline policy tests aligned when changing provider budgets. Tested: 171 Tool contract, Model Step, Tool Step, and builtin contract tests; scoped Ruff; git diff --check. Not-tested: Wall-clock calls to web and image providers. --- .../services/agent_runtime/tool_contracts.py | 16 +++++++++++- .../app/services/builtin_tool_definitions.py | 2 +- .../test_agent_runtime_tool_contracts.py | 25 +++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/backend/app/services/agent_runtime/tool_contracts.py b/backend/app/services/agent_runtime/tool_contracts.py index 0096a83b2..3223395ec 100644 --- a/backend/app/services/agent_runtime/tool_contracts.py +++ b/backend/app/services/agent_runtime/tool_contracts.py @@ -66,7 +66,13 @@ def __post_init__(self) -> None: "runtime_default", 60.0, 300.0, "stop_waiting_only" ), "network_read": ToolDeadlinePolicy( - "network_read", 30.0, 60.0, "stop_waiting_only" + "network_read", 60.0, 60.0, "stop_waiting_only" + ), + "image_generation": ToolDeadlinePolicy( + "image_generation", 120.0, 120.0, "stop_waiting_only" + ), + "custom_image_generation": ToolDeadlinePolicy( + "custom_image_generation", 600.0, 600.0, "stop_waiting_only" ), "local_code": ToolDeadlinePolicy( "local_code", 30.0, 3600.0, "cooperative" @@ -93,6 +99,14 @@ def deadline_policy_for_tool(tool_name: str) -> ToolDeadlinePolicy: return _DEADLINE_POLICIES["agentbay_read"] if tool_name in {"read_emails", "read_webpage", "jina_read"}: return _DEADLINE_POLICIES["network_read"] + if tool_name == "generate_image_custom": + return _DEADLINE_POLICIES["custom_image_generation"] + if tool_name in { + "generate_image_siliconflow", + "generate_image_openai", + "generate_image_google", + }: + return _DEADLINE_POLICIES["image_generation"] return _DEADLINE_POLICIES["runtime_default"] diff --git a/backend/app/services/builtin_tool_definitions.py b/backend/app/services/builtin_tool_definitions.py index 0fce33a70..8503fc390 100644 --- a/backend/app/services/builtin_tool_definitions.py +++ b/backend/app/services/builtin_tool_definitions.py @@ -3837,7 +3837,7 @@ "generate_image_siliconflow": 120, "generate_image_openai": 120, "generate_image_google": 120, - "generate_image_custom": 120, + "generate_image_custom": 600, } diff --git a/backend/tests/test_agent_runtime_tool_contracts.py b/backend/tests/test_agent_runtime_tool_contracts.py index 548fbe093..b0ea8f1e0 100644 --- a/backend/tests/test_agent_runtime_tool_contracts.py +++ b/backend/tests/test_agent_runtime_tool_contracts.py @@ -8,9 +8,34 @@ ToolContractError, ToolExecutionBinding, ToolWorksetEntry, + deadline_policy_for_tool, + resolve_tool_deadline_seconds, parse_step_tool_context, workset_version, ) +from app.services.builtin_tool_definitions import BUILTIN_TOOL_DEFINITIONS + + +def test_runtime_deadlines_cover_declared_network_and_image_provider_budgets() -> None: + expected = { + "read_webpage": 60.0, + "jina_read": 60.0, + "generate_image_siliconflow": 120.0, + "generate_image_openai": 120.0, + "generate_image_google": 120.0, + "generate_image_custom": 600.0, + } + + assert { + name: resolve_tool_deadline_seconds(deadline_policy_for_tool(name).name) + for name in expected + } == expected + declared = { + item["name"]: float(item["timeout_seconds"]) + for item in BUILTIN_TOOL_DEFINITIONS + if item["name"] in expected + } + assert declared == expected def _entry() -> ToolWorksetEntry: