Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .flocks/flocks.json.example
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@
"readMaxBytes": 51200,
"readMaxLineLength": 2000
},
"toolFailure": {
"disableOnRepeatedFailure": true
},
"sandbox": {
"mode": "off",
"scope": "agent",
Expand Down
19 changes: 19 additions & 0 deletions flocks/config/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,20 @@ class ToolOutputConfig(BaseModel):
)


class ToolFailureConfig(BaseModel):
"""Repeated tool-failure handling."""

model_config = {"populate_by_name": True}

disable_on_repeated_failure: bool = Field(
True,
alias="disableOnRepeatedFailure",
description=(
"Disable a standalone custom tool after repeated identical failures."
),
)


class EnterpriseConfig(BaseModel):
"""Enterprise configuration"""

Expand Down Expand Up @@ -675,6 +689,11 @@ class ConfigInfo(BaseModel):
alias="toolOutput",
description="Tool output size limits (read, truncation caps).",
)
tool_failure: Optional[ToolFailureConfig] = Field(
None,
alias="toolFailure",
description="Repeated tool-failure handling.",
)
experimental: Optional[ExperimentalConfig] = None

# Memory system configuration (added for memory system integration)
Expand Down
58 changes: 58 additions & 0 deletions flocks/server/routes/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,17 @@ class UIConfigUpdateRequest(BaseModel):
display_name: Optional[str] = Field(None, alias="displayName")


class ToolFailurePreference(BaseModel):
"""Repeated tool-failure preference exposed to the WebUI."""

model_config = {"populate_by_name": True}

disable_on_repeated_failure: bool = Field(
...,
alias="disableOnRepeatedFailure",
)


DEFAULT_UI_DISPLAY_NAME = "Flocks"
DEFAULT_UI_PRO_DISPLAY_NAME = "Flocks Pro"
FAVICON_MAX_BYTES = 512 * 1024
Expand Down Expand Up @@ -400,6 +411,12 @@ def _persist_ui_section(data: Dict[str, Any], ui_section: Dict[str, Any]) -> Non
ConfigWriter._write_raw(data)


def _effective_tool_failure_preference(config: ConfigInfoModel) -> bool:
if config.tool_failure is None:
return True
return config.tool_failure.disable_on_repeated_failure


@router.get("/ui-display", response_model=UIDisplayResponse, summary="Get public UI display name")
async def get_ui_display() -> UIDisplayResponse:
"""Return only the effective WebUI display name for public screens."""
Expand Down Expand Up @@ -521,6 +538,47 @@ async def reset_ui_favicon() -> UIDisplayResponse:
return await get_ui_display()


@router.get(
"/tool-failure",
response_model=ToolFailurePreference,
summary="Get repeated tool-failure preference",
)
async def get_tool_failure_preference() -> ToolFailurePreference:
"""Return whether repeated identical failures automatically disable tools."""
try:
config = await Config.get()
return ToolFailurePreference(
disableOnRepeatedFailure=_effective_tool_failure_preference(config)
)
except Exception as e:
log.error("config.tool_failure.get.error", {"error": str(e)})
raise HTTPException(status_code=500, detail=str(e))


@router.patch(
"/tool-failure",
response_model=ToolFailurePreference,
summary="Update repeated tool-failure preference",
)
async def update_tool_failure_preference(
request: ToolFailurePreference,
) -> ToolFailurePreference:
"""Update only the repeated-failure switch in flocks.json."""
try:
data = ConfigWriter._read_raw()
existing = data.get("toolFailure", data.get("tool_failure", {}))
section = dict(existing) if isinstance(existing, dict) else {}
section.pop("disable_on_repeated_failure", None)
section["disableOnRepeatedFailure"] = request.disable_on_repeated_failure
data.pop("tool_failure", None)
data["toolFailure"] = section
ConfigWriter._write_raw(data)
return await get_tool_failure_preference()
except Exception as e:
log.error("config.tool_failure.update.error", {"error": str(e)})
raise HTTPException(status_code=400, detail=str(e))


@router.get("", summary="Get configuration")
async def get_config() -> Dict[str, Any]:
"""
Expand Down
19 changes: 18 additions & 1 deletion flocks/tool/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -878,7 +878,11 @@ async def execute(
if result.success:
cls._reset_failure_state(tool_name)
else:
disabled = cls._record_failure(tool, kwargs, result.error)
if await cls._failure_auto_disable_enabled():
disabled = cls._record_failure(tool, kwargs, result.error)
else:
cls._reset_failure_state(tool_name)
disabled = False
if disabled:
result.metadata = {**(result.metadata or {}), "disabled": True, "disabled_reason": "repeated_error"}
suffix = f"tool disabled after {cls._failure_disable_threshold} identical errors"
Expand Down Expand Up @@ -1743,6 +1747,19 @@ def _reset_failure_state(cls, tool_name: str) -> None:
if tool_name in cls._failure_state:
cls._failure_state.pop(tool_name, None)

@classmethod
async def _failure_auto_disable_enabled(cls) -> bool:
"""Return the configured repeated-failure behavior, defaulting on."""
try:
from flocks.config.config import Config

config = await Config.get()
if config.tool_failure is not None:
return config.tool_failure.disable_on_repeated_failure
except Exception:
pass
return True

@classmethod
def _should_track_failure(cls, tool: Tool) -> bool:
"""Track failures only for standalone custom tools.
Expand Down
37 changes: 37 additions & 0 deletions tests/server/routes/test_remaining_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -508,6 +508,43 @@ async def test_config_has_expected_top_level_keys(self, client: AsyncClient):
f"No expected keys found. Got: {list(data.keys())}"
)

@pytest.mark.asyncio
async def test_tool_failure_preference_defaults_on_and_updates_only_its_section(
self,
client: AsyncClient,
tmp_path,
monkeypatch,
):
from flocks.config.config import Config
from flocks.config.config_writer import ConfigWriter

monkeypatch.setenv("FLOCKS_CONFIG_DIR", str(tmp_path / "config"))
Config._global_config = None
Config._cached_config = None
ConfigWriter._write_raw({"theme": "dark"})

resp = await client.get("/api/config/tool-failure")
assert resp.status_code == status.HTTP_200_OK
assert resp.json() == {"disableOnRepeatedFailure": True}

resp = await client.patch(
"/api/config/tool-failure",
json={"disableOnRepeatedFailure": False},
)
assert resp.status_code == status.HTTP_200_OK
assert resp.json() == {"disableOnRepeatedFailure": False}
assert ConfigWriter._read_raw() == {
"theme": "dark",
"toolFailure": {"disableOnRepeatedFailure": False},
}

resp = await client.get("/api/config/tool-failure")
assert resp.status_code == status.HTTP_200_OK
assert resp.json() == {"disableOnRepeatedFailure": False}

resp = await client.patch("/api/config/tool-failure", json={})
assert resp.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT

@pytest.mark.asyncio
async def test_ui_display_defaults_and_updates(
self,
Expand Down
97 changes: 97 additions & 0 deletions tests/tool/test_failure_auto_disable_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
from __future__ import annotations

import pytest

from flocks.config.config import Config, ConfigInfo, ToolFailureConfig
from flocks.tool.registry import (
Tool,
ToolCategory,
ToolContext,
ToolInfo,
ToolRegistry,
ToolResult,
)


def _failing_tool(name: str = "failing_custom_tool") -> Tool:
async def handler(_ctx: ToolContext, **_kwargs) -> ToolResult:
return ToolResult(success=False, error="synthetic repeated failure")

return Tool(
info=ToolInfo(
name=name,
description="Always fails for repeated-failure tests",
category=ToolCategory.CUSTOM,
source="plugin_py",
),
handler=handler,
)


@pytest.fixture
def isolated_failure_tracking(monkeypatch: pytest.MonkeyPatch):
tool = _failing_tool()
monkeypatch.setattr(ToolRegistry, "_initialized", True)
monkeypatch.setattr(ToolRegistry, "_tools", {tool.info.name: tool})
monkeypatch.setattr(ToolRegistry, "_failure_state", {})
monkeypatch.setattr(Config, "_cached_config", ConfigInfo())
return tool


def test_tool_failure_config_defaults_to_enabled() -> None:
section = ToolFailureConfig()

assert section.disable_on_repeated_failure is True


def test_tool_failure_config_accepts_camel_and_snake_case() -> None:
camel = ConfigInfo.model_validate(
{"toolFailure": {"disableOnRepeatedFailure": False}}
)
snake = ConfigInfo.model_validate(
{"tool_failure": {"disable_on_repeated_failure": False}}
)

assert camel.tool_failure is not None
assert camel.tool_failure.disable_on_repeated_failure is False
assert snake.tool_failure is not None
assert snake.tool_failure.disable_on_repeated_failure is False


@pytest.mark.asyncio
async def test_repeated_failures_disable_tool_by_default(
isolated_failure_tracking: Tool,
) -> None:
tool = isolated_failure_tracking

for _ in range(ToolRegistry._failure_disable_threshold):
result = await ToolRegistry.execute(tool.info.name, query="same")

assert tool.info.enabled is False
assert result.metadata == {
"disabled": True,
"disabled_reason": "repeated_error",
}


@pytest.mark.asyncio
async def test_config_can_turn_off_repeated_failure_auto_disable(
isolated_failure_tracking: Tool,
monkeypatch: pytest.MonkeyPatch,
) -> None:
tool = isolated_failure_tracking
monkeypatch.setattr(
Config,
"_cached_config",
ConfigInfo.model_validate(
{"toolFailure": {"disableOnRepeatedFailure": False}}
),
)

for _ in range(ToolRegistry._failure_disable_threshold + 1):
result = await ToolRegistry.execute(tool.info.name, query="same")

assert result.success is False
assert "disabled" not in result.metadata
assert tool.info.enabled is True
assert ToolRegistry._failure_state == {}
19 changes: 19 additions & 0 deletions webui/src/api/toolFailureConfig.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import client from './client';

export interface ToolFailureConfig {
disableOnRepeatedFailure: boolean;
}

export const toolFailureConfigApi = {
async get(): Promise<ToolFailureConfig> {
const response = await client.get<ToolFailureConfig>('/api/config/tool-failure');
return response.data;
},

async update(disableOnRepeatedFailure: boolean): Promise<ToolFailureConfig> {
const response = await client.patch<ToolFailureConfig>('/api/config/tool-failure', {
disableOnRepeatedFailure,
});
return response.data;
},
};
7 changes: 6 additions & 1 deletion webui/src/locales/en-US/nav.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"settingsTitle": "Settings",
"settingsDescription": "Manage preferences, account, and system settings",
"settingsPreferences": "Preferences",
"settingsPreferencesDescription": "Adjust the workspace name, console language, and interface theme.",
"settingsPreferencesDescription": "Adjust workspace display, language, theme, and tool failure handling.",
"settingsGroupPreferences": "Preferences",
"settingsGroupSystem": "Account & System",
"settingsGroupIntegrations": "Models & Channels",
Expand All @@ -53,6 +53,11 @@
"themeDescription": "Choose the default light or dark mode for the console.",
"lightTheme": "Light",
"darkTheme": "Dark",
"toolFailureAutoDisable": "Auto-disable repeatedly failing tools",
"toolFailureAutoDisableDescription": "Disable a standalone custom tool after the same parameters repeatedly produce the same error up to the configured threshold.",
"toolFailureSettingSaved": "Tool failure handling preference saved",
"toolFailureSettingSaveFailed": "Failed to save tool failure handling preference",
"toolFailureSettingLoadFailed": "Failed to load tool failure handling preference",
"logout": "Log out",
"expandNav": "Expand navigation",
"collapseNav": "Collapse navigation",
Expand Down
7 changes: 6 additions & 1 deletion webui/src/locales/zh-CN/nav.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"settingsTitle": "设置",
"settingsDescription": "管理偏好、账号与系统配置",
"settingsPreferences": "偏好设置",
"settingsPreferencesDescription": "调整当前工作台的名称、显示语言和界面主题。",
"settingsPreferencesDescription": "调整当前工作台的显示、语言、主题和工具失败处理。",
"settingsGroupPreferences": "偏好",
"settingsGroupSystem": "账号与系统",
"settingsGroupIntegrations": "模型与通道",
Expand All @@ -53,6 +53,11 @@
"themeDescription": "选择控制台默认使用的浅色或深色模式。",
"lightTheme": "浅色",
"darkTheme": "深色",
"toolFailureAutoDisable": "自动禁用重复失败工具",
"toolFailureAutoDisableDescription": "同一独立自定义工具以相同参数重复返回相同错误达到阈值后,自动禁用该工具。",
"toolFailureSettingSaved": "工具失败处理设置已保存",
"toolFailureSettingSaveFailed": "保存工具失败处理设置失败",
"toolFailureSettingLoadFailed": "加载工具失败处理设置失败",
"logout": "退出登录",
"expandNav": "展开导航",
"collapseNav": "收起导航",
Expand Down
Loading