Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions backend/app/services/agent_runtime/node_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -778,6 +778,8 @@ async def _model(
repair_limit = (
WRITE_FILE_PROTOCOL_REPAIR_LIMIT
if is_write_file_repair
else 10
if repair_code == "invalid_tool_call"
else 1
)
repair_counter_key = (
Expand Down
2 changes: 1 addition & 1 deletion backend/app/services/agent_runtime/tool_execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
"reconcile",
]
ToolSideEffectState = Literal["none", "confirmed", "possible", "unknown"]
SAFE_READ_MAX_ATTEMPTS = 3
SAFE_READ_MAX_ATTEMPTS = 10

# These tools dispatch an external image-generation request and can therefore
# leave the provider outcome uncertain after a response timeout. Direct Chat
Expand Down
2 changes: 1 addition & 1 deletion backend/app/services/agent_runtime/tool_repair_budget.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from app.services.agent_runtime.state import JsonObject

SAME_FINGERPRINT_FAILURE_LIMIT = 10
TOOL_EPISODE_FAILURE_LIMIT = 20
TOOL_EPISODE_FAILURE_LIMIT = 10
_REPAIRABLE_MODEL_ACTIONS = frozenset(
{"repair_arguments", "choose_other_tool"}
)
Expand Down
19 changes: 18 additions & 1 deletion backend/app/services/agent_runtime/tool_step_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -1496,6 +1496,7 @@ def _group_unknown_failure(
outcome: ToolExecutionOutcome,
messages: Sequence[JsonObject],
pending_tool_calls: Sequence[JsonObject],
step_tool_context: JsonObject | None = None,
) -> ToolStepResult:
"""End an unresumable Group Run without creating a user interrupt."""
normalized, _ = normalize_tool_outcome(
Expand Down Expand Up @@ -1525,6 +1526,7 @@ def _group_unknown_failure(
),
),
pending_tool_calls=tuple(pending_tool_calls),
step_tool_context=step_tool_context,
error={"code": error_code, "message": error_message},
)

Expand Down Expand Up @@ -1977,6 +1979,7 @@ async def execute_pending(
messages=tuple(messages),
waiting_request=waiting_request,
pending_tool_calls=tool_calls[index + 1 :],
step_tool_context=step_context_update,
)
continue
if reservation.blocked:
Expand Down Expand Up @@ -2090,6 +2093,7 @@ async def execute_pending(
outcome=outcome,
messages=messages,
pending_tool_calls=tool_calls[index + 1 :],
step_tool_context=step_context_update,
)
messages.append(
_result_message(
Expand Down Expand Up @@ -2133,6 +2137,7 @@ async def execute_pending(
outcome=outcome,
messages=messages,
pending_tool_calls=tool_calls[index + 1 :],
step_tool_context=step_context_update,
)
messages.append(
_result_message(
Expand All @@ -2155,6 +2160,7 @@ async def execute_pending(
outcome=execution_outcome(reservation.execution),
messages=messages,
pending_tool_calls=tool_calls[index + 1 :],
step_tool_context=step_context_update,
)
return ToolStepResult(
messages=tuple(messages),
Expand All @@ -2165,6 +2171,7 @@ async def execute_pending(
error_code=reservation.error_code,
),
pending_tool_calls=tool_calls[index:],
step_tool_context=step_context_update,
)

if autonomy_outcome is not None:
Expand Down Expand Up @@ -2256,6 +2263,7 @@ async def execute_pending(
outcome=outcome,
messages=messages,
pending_tool_calls=tool_calls[index + 1 :],
step_tool_context=step_context_update,
)
return ToolStepResult(
messages=tuple(messages),
Expand All @@ -2266,6 +2274,7 @@ async def execute_pending(
error_code="tool_outcome_unknown",
),
pending_tool_calls=tool_calls[index:],
step_tool_context=step_context_update,
)
else:
if a2a_result is not None:
Expand All @@ -2281,6 +2290,7 @@ async def execute_pending(
outcome=a2a_result.outcome,
messages=messages,
pending_tool_calls=tool_calls[index + 1 :],
step_tool_context=step_context_update,
)
messages.append(
_result_message(
Expand All @@ -2295,6 +2305,7 @@ async def execute_pending(
messages=tuple(messages),
waiting_request=a2a_result.waiting_request,
pending_tool_calls=tool_calls[index + 1 :],
step_tool_context=step_context_update,
)
continue

Expand Down Expand Up @@ -2411,6 +2422,7 @@ async def execute_pending(
outcome=outcome,
messages=messages,
pending_tool_calls=tool_calls[index + 1 :],
step_tool_context=step_context_update,
)
return ToolStepResult(
messages=tuple(messages),
Expand All @@ -2421,6 +2433,7 @@ async def execute_pending(
error_code="tool_outcome_unknown",
),
pending_tool_calls=tool_calls[index:],
step_tool_context=step_context_update,
)
else:
if isinstance(raw_result, ToolExecutionOutcome):
Expand Down Expand Up @@ -2497,6 +2510,7 @@ async def execute_pending(
outcome=outcome,
messages=messages,
pending_tool_calls=tool_calls[index + 1 :],
step_tool_context=step_context_update,
)
return ToolStepResult(
messages=tuple(messages),
Expand All @@ -2507,6 +2521,7 @@ async def execute_pending(
error_code=outcome.error_code or "tool_outcome_unknown",
),
pending_tool_calls=tool_calls[index:],
step_tool_context=step_context_update,
)
messages.append(
_result_message(
Expand All @@ -2531,13 +2546,15 @@ async def execute_pending(
except ToolExecutionError as exc:
return ToolStepResult(
error={"code": exc.code, "message": str(exc)},
step_tool_context=step_context_update,
)
except Exception as exc:
return ToolStepResult(
error={
"code": "tool_execution_failed",
"message": f"Runtime tool step failed: {type(exc).__name__}",
}
},
step_tool_context=step_context_update,
)


Expand Down
4 changes: 2 additions & 2 deletions backend/app/services/llm/caller.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ async def execute_tool(*args, **kwargs):
"send_message_to_agent", "send_feishu_message", "send_email"
})

WRITE_FILE_PROTOCOL_REPAIR_LIMIT = 3
WRITE_FILE_PROTOCOL_REPAIR_LIMIT = 10
WRITE_FILE_PROTOCOL_REPAIR_COUNTER_KEY = "invalid_tool_call:write_file"
WRITE_FILE_PROTOCOL_REPAIR_INSTRUCTION = (
"Your previous `write_file` call was not executed because `function.arguments` "
Expand Down Expand Up @@ -788,7 +788,7 @@ async def _buffer_chunk(_text: str) -> None:
repair_limit = (
WRITE_FILE_PROTOCOL_REPAIR_LIMIT
if retry_tool_name == "write_file"
else 1
else 10
)
repair_counter_key = (
WRITE_FILE_PROTOCOL_REPAIR_COUNTER_KEY
Expand Down
2 changes: 1 addition & 1 deletion backend/tests/test_agent_runtime_model_step_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -598,7 +598,7 @@ async def complete(model_arg, _messages, **_kwargs):


@pytest.mark.asyncio
async def test_invalid_write_file_arguments_request_three_protocol_repairs() -> None:
async def test_invalid_write_file_arguments_request_ten_protocol_repairs() -> None:
tenant_id = uuid.uuid4()
model = _model(tenant_id)
agent = _agent(tenant_id)
Expand Down
35 changes: 17 additions & 18 deletions backend/tests/test_agent_runtime_node_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -1351,15 +1351,16 @@ async def test_empty_output_is_repaired_once_then_fails_explicitly() -> None:

@pytest.mark.asyncio
@pytest.mark.parametrize(
("repair_code", "instruction"),
("repair_code", "instruction", "repair_limit"),
[
("invalid_finish", "Retry finish with valid content."),
("invalid_tool_call", "Retry with valid JSON tool arguments."),
("invalid_finish", "Retry finish with valid content.", 1),
("invalid_tool_call", "Retry with valid JSON tool arguments.", 10),
],
)
async def test_repeated_model_tool_protocol_repair_code_fails_explicitly(
repair_code: str,
instruction: str,
repair_limit: int,
) -> None:
run_id = uuid.uuid4()
repair = ModelStepResult(
Expand All @@ -1368,7 +1369,7 @@ async def test_repeated_model_tool_protocol_repair_code_fails_explicitly(
repair_instruction=instruction,
repair_code=repair_code,
)
model = ModelService(repair, repair)
model = ModelService(*([repair] * (repair_limit + 1)))
executor = _executor(model)

result = await _invoke(run_id, executor, model_turn_limit=50)
Expand All @@ -1377,13 +1378,13 @@ async def test_repeated_model_tool_protocol_repair_code_fails_explicitly(
assert lifecycle["status"] == "failed"
assert lifecycle["reason"] == "model_tool_protocol_violation"
assert lifecycle["error"]["code"] == "model_tool_protocol_violation"
assert lifecycle["model_protocol_repairs"] == {repair_code: 1}
assert lifecycle["model_step_count"] == 2
assert model.calls == 2
assert lifecycle["model_protocol_repairs"] == {repair_code: repair_limit}
assert lifecycle["model_step_count"] == repair_limit + 1
assert model.calls == repair_limit + 1


@pytest.mark.asyncio
async def test_write_file_protocol_repair_uses_three_attempts_then_guides_user() -> None:
async def test_write_file_protocol_repair_uses_ten_attempts_then_guides_user() -> None:
run_id = uuid.uuid4()
repair = ModelStepResult(
intent="text",
Expand All @@ -1392,7 +1393,7 @@ async def test_write_file_protocol_repair_uses_three_attempts_then_guides_user()
repair_code="invalid_tool_call",
repair_tool_name="write_file",
)
model = ModelService(repair, repair, repair, repair)
model = ModelService(*([repair] * 11))
executor = _executor(model)

result = await _invoke(run_id, executor, model_turn_limit=50)
Expand All @@ -1408,14 +1409,14 @@ async def test_write_file_protocol_repair_uses_three_attempts_then_guides_user()
),
}
assert lifecycle["model_protocol_repairs"] == {
"invalid_tool_call:write_file": 3,
"invalid_tool_call:write_file": 10,
}
assert lifecycle["model_step_count"] == 4
assert model.calls == 4
assert lifecycle["model_step_count"] == 11
assert model.calls == 11


@pytest.mark.asyncio
async def test_write_file_protocol_can_recover_on_the_third_repair() -> None:
async def test_write_file_protocol_can_recover_on_the_tenth_repair() -> None:
run_id = uuid.uuid4()
repair = ModelStepResult(
intent="text",
Expand All @@ -1424,9 +1425,7 @@ async def test_write_file_protocol_can_recover_on_the_third_repair() -> None:
repair_tool_name="write_file",
)
model = ModelService(
repair,
repair,
repair,
*([repair] * 10),
ModelStepResult(intent="finish", finish_content="Recovered"),
)
executor = _executor(model)
Expand All @@ -1435,9 +1434,9 @@ async def test_write_file_protocol_can_recover_on_the_third_repair() -> None:

assert result["lifecycle"]["status"] == "completed"
assert result["lifecycle"]["model_protocol_repairs"] == {
"invalid_tool_call:write_file": 3,
"invalid_tool_call:write_file": 10,
}
assert model.calls == 4
assert model.calls == 11


@pytest.mark.asyncio
Expand Down
4 changes: 2 additions & 2 deletions backend/tests/test_agent_runtime_tool_repair_budget.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def test_tenth_consecutive_fingerprint_pauses_without_off_by_one() -> None:
assert _episode(state)["total_failures"] == 10


def test_twentieth_tool_failure_pauses_even_when_fingerprint_changes() -> None:
def test_tenth_tool_failure_pauses_even_when_fingerprint_changes() -> None:
state: dict = {}
transition = None
for model_step in range(1, TOOL_EPISODE_FAILURE_LIMIT + 1):
Expand All @@ -63,7 +63,7 @@ def test_twentieth_tool_failure_pauses_even_when_fingerprint_changes() -> None:

assert transition is not None
assert transition.pause_reason == "tool_repair_episode_limit_reached"
assert _episode(state)["total_failures"] == 20
assert _episode(state)["total_failures"] == 10
assert _episode(state)["same_fingerprint_failures"] == 1


Expand Down
Loading