From 1f20ae05bf616ac026d1839286c3b6f37ede62a2 Mon Sep 17 00:00:00 2001 From: SweetenedSuzuka <188338189+SweetenedSuzuka@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:54:41 +0800 Subject: [PATCH 1/4] fix(core): caption tool-returned images when the main model lacks image modality (#9675) When the main chat model does not support image input, images returned by tools (e.g. via ImageContent in a CallToolResult) were silently dropped: the cached_images block only injected base64 image parts when the model supported the image modality, with no fallback. This adds an else branch that reuses the configured default_image_caption_provider_id to generate a text caption per cached image, mirroring astr_main_agent._ensure_img_caption(), and injects it as a user message so the model can still see the content. --- .../agent/runners/tool_loop_agent_runner.py | 72 ++++++- tests/test_tool_loop_agent_runner.py | 186 ++++++++++++++++++ 2 files changed, 257 insertions(+), 1 deletion(-) diff --git a/astrbot/core/agent/runners/tool_loop_agent_runner.py b/astrbot/core/agent/runners/tool_loop_agent_runner.py index 8c91adbbfd..1a9666b8f8 100644 --- a/astrbot/core/agent/runners/tool_loop_agent_runner.py +++ b/astrbot/core/agent/runners/tool_loop_agent_runner.py @@ -27,7 +27,7 @@ from astrbot import logger from astrbot.core.agent.message import ImageURLPart, TextPart, ThinkPart from astrbot.core.agent.tool import FunctionTool, ToolSet -from astrbot.core.agent.tool_image_cache import tool_image_cache +from astrbot.core.agent.tool_image_cache import CachedImage, tool_image_cache from astrbot.core.exceptions import EmptyModelOutputError from astrbot.core.message.components import Json from astrbot.core.message.message_event_result import ( @@ -1054,9 +1054,79 @@ async def step(self): logger.debug( f"Appended {len(cached_images)} cached image(s) to context for LLM review" ) + else: + # The main model cannot view images: fall back to the configured + # image caption provider so the image content is not silently + # dropped. Mirrors astr_main_agent._ensure_img_caption(). + caption_parts = await self._caption_cached_tool_images( + cached_images + ) + if caption_parts: + self.run_context.messages.append( + Message(role="user", content=caption_parts) + ) + logger.debug( + f"Appended captions for {len(cached_images)} cached image(s) to context for LLM review" + ) self.req.append_tool_calls_result(tool_calls_result) + async def _caption_cached_tool_images( + self, cached_images: list[CachedImage] + ) -> list[TextPart]: + """主模型不支持图片时,用默认图片转述模型生成文字描述,失败或未配置则退化为占位文本。""" + plugin_context = getattr(self.run_context.context, "context", None) + event = getattr(self.run_context.context, "event", None) + cfg: dict = {} + img_cap_prov_id = "" + if plugin_context is not None: + try: + cfg = plugin_context.get_config( + umo=getattr(event, "unified_msg_origin", None) + ).get("provider_settings", {}) + except Exception as exc: # noqa: BLE001 + logger.debug("Failed to read provider settings: %s", exc) + img_cap_prov_id = cfg.get("default_image_caption_provider_id") or "" + + fallback_note = ( + "[Image not visible to the current model]" + if not img_cap_prov_id + else "[Image Captioning Failed]" + ) + + parts: list[TextPart] = [] + for cached_img in cached_images: + parts.append( + TextPart( + text=( + f"[Image from tool '{cached_img.tool_name}', " + f"path='{cached_img.file_path}']" + ) + ) + ) + caption = None + if img_cap_prov_id and Path(cached_img.file_path).exists(): + try: + from astrbot.core.astr_main_agent import _request_img_caption + + caption = await _request_img_caption( + img_cap_prov_id, + cfg, + [cached_img.file_path], + plugin_context, + ) + except Exception as exc: # noqa: BLE001 + logger.error( + "Failed to caption tool image %s: %s", + cached_img.file_path, + exc, + ) + if caption: + parts.append(TextPart(text=f"{caption}")) + else: + parts.append(TextPart(text=fallback_note)) + return parts + async def step_until_done( self, max_step: int ) -> T.AsyncGenerator[AgentResponse, None]: diff --git a/tests/test_tool_loop_agent_runner.py b/tests/test_tool_loop_agent_runner.py index 1e679de4aa..ba7ed84ab2 100644 --- a/tests/test_tool_loop_agent_runner.py +++ b/tests/test_tool_loop_agent_runner.py @@ -142,6 +142,48 @@ async def generator(): return generator() +class MockCaptionProvider(MockProvider): + """模拟默认图片转述模型,记录收到的图片路径并返回固定描述。""" + + def __init__(self, caption_text: str = "图片中是一只猫"): + super().__init__() + self.caption_text = caption_text + self.captioned_paths: list[str] = [] + + async def text_chat(self, **kwargs) -> LLMResponse: + self.captioned_paths.extend(kwargs.get("image_urls", [])) + return LLMResponse( + role="assistant", + completion_text=self.caption_text, + usage=TokenUsage(input_other=10, output=5), + ) + + +class MockFailingCaptionProvider(MockCaptionProvider): + """模拟转述模型调用失败的场景。""" + + async def text_chat(self, **kwargs) -> LLMResponse: + raise RuntimeError("caption provider down") + + +class MockImageCaptionContext: + """模拟含 default_image_caption_provider_id 配置的 star Context。""" + + def __init__(self, caption_provider: Any, provider_id: str = "cap_prov"): + self.caption_provider = caption_provider + self.provider_id = provider_id + + def get_config(self, umo=None): + return { + "provider_settings": { + "default_image_caption_provider_id": self.provider_id, + } + } + + def get_provider_by_id(self, provider_id: str): + return self.caption_provider + + class VaryingUsageProvider(MockProvider): """Return distinct token usage values for each tool-loop request.""" @@ -906,6 +948,150 @@ def fake_save_image( ] +def _all_context_texts(contexts: list[Any]) -> list[str]: + texts: list[str] = [] + for msg in contexts: + content = msg.get("content") + if isinstance(content, str): + texts.append(content) + elif isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + texts.append(str(part.get("text", ""))) + return texts + + +def _tool_image_runner_setup( + runner, + provider_request, + mock_hooks, + monkeypatch, + tmp_path, + caption_context, + modalities: list[str] | None = ["tool_use"], +): + from astrbot.core.agent.tool_image_cache import CachedImage, tool_image_cache + + img_path = tmp_path / "tool_img.png" + img_path.write_bytes(b"fake-png-bytes") + + def fake_save_image( + base64_data, tool_call_id, tool_name, index=0, mime_type="image/png" + ): + return CachedImage( + tool_call_id=tool_call_id, + tool_name=tool_name, + file_path=str(img_path), + mime_type=mime_type, + ) + + monkeypatch.setattr(tool_image_cache, "save_image", fake_save_image) + + event = MockEvent("test:FriendMessage:tool_caption", "u1") + provider = CapturingToolLoopProvider("test_tool") + provider.provider_config["modalities"] = modalities + + async def _run(): + await runner.reset( + provider=provider, + request=provider_request, + run_context=ContextWrapper( + context=SimpleNamespace(event=event, context=caption_context) + ), + tool_executor=MockMixedContentToolExecutor, + agent_hooks=mock_hooks, + streaming=False, + ) + async for _ in runner.step_until_done(3): + pass + + return provider, img_path, _run + + +@pytest.mark.asyncio +async def test_tool_image_caption_injected_when_main_model_lacks_image( + runner, provider_request, mock_hooks, monkeypatch, tmp_path +): + """主模型不支持图片时,工具返回的图片应经默认转述模型生成文字描述注入上下文。""" + caption_provider = MockCaptionProvider(caption_text="图片中是一只猫") + provider, img_path, run = _tool_image_runner_setup( + runner, + provider_request, + mock_hooks, + monkeypatch, + tmp_path, + caption_context=MockImageCaptionContext(caption_provider), + ) + await run() + + assert provider.call_count == 2 + assert caption_provider.captioned_paths == [str(img_path)] + second_contexts = provider.received_contexts[1] + context_texts = _all_context_texts(second_contexts) + assert any( + "图片中是一只猫" in t for t in context_texts + ) + tool_msg_index = next( + i for i, msg in enumerate(second_contexts) if msg.get("role") == "tool" + ) + caption_msg_index = next( + i + for i, msg in enumerate(second_contexts) + if isinstance(msg.get("content"), list) + and any( + isinstance(part, dict) + and part.get("type") == "text" + and "" in str(part.get("text", "")) + for part in msg.get("content", []) + ) + ) + assert caption_msg_index > tool_msg_index + + +@pytest.mark.asyncio +async def test_tool_image_placeholder_when_no_caption_provider( + runner, provider_request, mock_hooks, monkeypatch, tmp_path +): + """未配置默认转述模型时,注入占位文本而非静默丢弃图片内容。""" + provider, _, run = _tool_image_runner_setup( + runner, + provider_request, + mock_hooks, + monkeypatch, + tmp_path, + caption_context=MockImageCaptionContext(caption_provider=None, provider_id=""), + ) + await run() + + assert provider.call_count == 2 + second_contexts = provider.received_contexts[1] + context_texts = _all_context_texts(second_contexts) + assert any("[Image not visible to the current model]" in t for t in context_texts) + assert not any("" in t for t in context_texts) + + +@pytest.mark.asyncio +async def test_tool_image_caption_failure_placeholder( + runner, provider_request, mock_hooks, monkeypatch, tmp_path +): + """转述模型调用失败时注入 [Image Captioning Failed],工具循环不被中断。""" + provider, _, run = _tool_image_runner_setup( + runner, + provider_request, + mock_hooks, + monkeypatch, + tmp_path, + caption_context=MockImageCaptionContext(MockFailingCaptionProvider()), + ) + await run() + + assert provider.call_count == 2 + second_contexts = provider.received_contexts[1] + context_texts = _all_context_texts(second_contexts) + assert any("[Image Captioning Failed]" in t for t in context_texts) + + + @pytest.mark.asyncio async def test_runner_replaces_runtime_image_context_before_provider_call( runner, provider_request, mock_hooks From 735c056d6d76e23b22e2b4327818e6d089855f33 Mon Sep 17 00:00:00 2001 From: SweetenedSuzuka <188338189+SweetenedSuzuka@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:06:07 +0800 Subject: [PATCH 2/4] style(tests): remove extra blank line in tool caption tests --- tests/test_tool_loop_agent_runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_tool_loop_agent_runner.py b/tests/test_tool_loop_agent_runner.py index ba7ed84ab2..84225f9cb0 100644 --- a/tests/test_tool_loop_agent_runner.py +++ b/tests/test_tool_loop_agent_runner.py @@ -1091,7 +1091,6 @@ async def test_tool_image_caption_failure_placeholder( assert any("[Image Captioning Failed]" in t for t in context_texts) - @pytest.mark.asyncio async def test_runner_replaces_runtime_image_context_before_provider_call( runner, provider_request, mock_hooks From 8f6e70ac83ce19b6f4c8e658314662973d1629c9 Mon Sep 17 00:00:00 2001 From: SweetenedSuzuka <188338189+SweetenedSuzuka@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:17:07 +0800 Subject: [PATCH 3/4] refactor(agent): caption all tool images in a single caption-provider call (#9675) Caption all cached tool images with one _request_img_caption call per step instead of one call per image, matching astr_main_agent._ensure_img_caption(). Bounding the caption calls to one per image-returning step keeps long tool loops from being slowed by N serial caption round-trips when a tool returns many images. Add a test asserting N images trigger exactly one caption call. --- .../agent/runners/tool_loop_agent_runner.py | 59 +++++++++++-------- tests/test_tool_loop_agent_runner.py | 56 +++++++++++++++++- 2 files changed, 88 insertions(+), 27 deletions(-) diff --git a/astrbot/core/agent/runners/tool_loop_agent_runner.py b/astrbot/core/agent/runners/tool_loop_agent_runner.py index 1a9666b8f8..f23669bf77 100644 --- a/astrbot/core/agent/runners/tool_loop_agent_runner.py +++ b/astrbot/core/agent/runners/tool_loop_agent_runner.py @@ -1074,7 +1074,7 @@ async def step(self): async def _caption_cached_tool_images( self, cached_images: list[CachedImage] ) -> list[TextPart]: - """主模型不支持图片时,用默认图片转述模型生成文字描述,失败或未配置则退化为占位文本。""" + """主模型不支持图片时,用默认图片转述模型为缓存图片生成一段文字描述,失败或未配置则退化为占位文本。""" plugin_context = getattr(self.run_context.context, "context", None) event = getattr(self.run_context.context, "event", None) cfg: dict = {} @@ -1088,43 +1088,50 @@ async def _caption_cached_tool_images( logger.debug("Failed to read provider settings: %s", exc) img_cap_prov_id = cfg.get("default_image_caption_provider_id") or "" - fallback_note = ( - "[Image not visible to the current model]" - if not img_cap_prov_id - else "[Image Captioning Failed]" - ) - - parts: list[TextPart] = [] - for cached_img in cached_images: - parts.append( - TextPart( - text=( - f"[Image from tool '{cached_img.tool_name}', " - f"path='{cached_img.file_path}']" - ) + parts: list[TextPart] = [ + TextPart( + text=( + f"[Image from tool '{cached_img.tool_name}', " + f"path='{cached_img.file_path}']" ) ) - caption = None - if img_cap_prov_id and Path(cached_img.file_path).exists(): + for cached_img in cached_images + ] + + # 与 astr_main_agent._ensure_img_caption() 一致:一次调用处理全部图片, + # 避免按图逐个调用导致长对话中工具循环被多次串行转述拖慢。 + caption = None + if img_cap_prov_id: + caption_paths = [ + cached_img.file_path + for cached_img in cached_images + if Path(cached_img.file_path).exists() + ] + if caption_paths: try: from astrbot.core.astr_main_agent import _request_img_caption caption = await _request_img_caption( img_cap_prov_id, cfg, - [cached_img.file_path], + caption_paths, plugin_context, ) except Exception as exc: # noqa: BLE001 - logger.error( - "Failed to caption tool image %s: %s", - cached_img.file_path, - exc, + logger.error("Failed to caption tool images: %s", exc) + + if caption: + parts.append(TextPart(text=f"{caption}")) + else: + parts.append( + TextPart( + text=( + "[Image not visible to the current model]" + if not img_cap_prov_id + else "[Image Captioning Failed]" ) - if caption: - parts.append(TextPart(text=f"{caption}")) - else: - parts.append(TextPart(text=fallback_note)) + ) + ) return parts async def step_until_done( diff --git a/tests/test_tool_loop_agent_runner.py b/tests/test_tool_loop_agent_runner.py index 84225f9cb0..b7903b443a 100644 --- a/tests/test_tool_loop_agent_runner.py +++ b/tests/test_tool_loop_agent_runner.py @@ -142,6 +142,33 @@ async def generator(): return generator() +class MockMultiImageToolExecutor: + """模拟一次工具调用返回多张图片的工具执行器。""" + + @classmethod + def execute(cls, tool, run_context, **tool_args): + async def generator(): + from mcp.types import CallToolResult, ImageContent + + result = CallToolResult( + content=[ + ImageContent( + type="image", + data="dGVzdA==", + mimeType="image/png", + ), + ImageContent( + type="image", + data="dGVzdA==", + mimeType="image/png", + ), + ] + ) + yield result + + return generator() + + class MockCaptionProvider(MockProvider): """模拟默认图片转述模型,记录收到的图片路径并返回固定描述。""" @@ -151,6 +178,7 @@ def __init__(self, caption_text: str = "图片中是一只猫"): self.captioned_paths: list[str] = [] async def text_chat(self, **kwargs) -> LLMResponse: + self.call_count += 1 self.captioned_paths.extend(kwargs.get("image_urls", [])) return LLMResponse( role="assistant", @@ -969,6 +997,7 @@ def _tool_image_runner_setup( tmp_path, caption_context, modalities: list[str] | None = ["tool_use"], + tool_executor=MockMixedContentToolExecutor, ): from astrbot.core.agent.tool_image_cache import CachedImage, tool_image_cache @@ -998,7 +1027,7 @@ async def _run(): run_context=ContextWrapper( context=SimpleNamespace(event=event, context=caption_context) ), - tool_executor=MockMixedContentToolExecutor, + tool_executor=tool_executor, agent_hooks=mock_hooks, streaming=False, ) @@ -1048,6 +1077,31 @@ async def test_tool_image_caption_injected_when_main_model_lacks_image( assert caption_msg_index > tool_msg_index +@pytest.mark.asyncio +async def test_tool_multi_image_caption_single_call( + runner, provider_request, mock_hooks, monkeypatch, tmp_path +): + """一次工具调用返回多张图片时,只发起一次转述调用,避免长对话中被多次串行转述拖慢。""" + caption_provider = MockCaptionProvider(caption_text="两张截图") + provider, img_path, run = _tool_image_runner_setup( + runner, + provider_request, + mock_hooks, + monkeypatch, + tmp_path, + caption_context=MockImageCaptionContext(caption_provider), + tool_executor=MockMultiImageToolExecutor, + ) + await run() + + assert provider.call_count == 2 + assert caption_provider.call_count == 1 + assert caption_provider.captioned_paths == [str(img_path), str(img_path)] + second_contexts = provider.received_contexts[1] + context_texts = _all_context_texts(second_contexts) + assert context_texts.count("两张截图") == 1 + + @pytest.mark.asyncio async def test_tool_image_placeholder_when_no_caption_provider( runner, provider_request, mock_hooks, monkeypatch, tmp_path From 4ffc2e5bd7f8b09fe7135876c9a4c24eb9f51ab5 Mon Sep 17 00:00:00 2001 From: SweetenedSuzuka <188338189+SweetenedSuzuka@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:59:20 +0800 Subject: [PATCH 4/4] refactor(agent): split image caption helpers and simplify flow (#9675) Address automated review feedback: extract config resolution and the caption request into dedicated helpers, drop the redundant `if caption_parts:` guard (the method always returns at least one TextPart), and avoid a mutable default argument in the test helper. --- .../agent/runners/tool_loop_agent_runner.py | 98 +++++++++++-------- tests/test_tool_loop_agent_runner.py | 4 +- 2 files changed, 62 insertions(+), 40 deletions(-) diff --git a/astrbot/core/agent/runners/tool_loop_agent_runner.py b/astrbot/core/agent/runners/tool_loop_agent_runner.py index f23669bf77..7283e0ff43 100644 --- a/astrbot/core/agent/runners/tool_loop_agent_runner.py +++ b/astrbot/core/agent/runners/tool_loop_agent_runner.py @@ -103,6 +103,24 @@ class _ToolExecutionInterrupted(Exception): """Raised when a running tool call is interrupted by a stop request.""" +def _get_image_caption_config( + run_context: ContextWrapper[TContext], +) -> tuple[str, dict]: + """Resolve the configured image caption provider id and provider settings.""" + plugin_context = getattr(run_context.context, "context", None) + event = getattr(run_context.context, "event", None) + if plugin_context is None: + return "", {} + try: + cfg = plugin_context.get_config( + umo=getattr(event, "unified_msg_origin", None) + ).get("provider_settings", {}) + except Exception as exc: # noqa: BLE001 + logger.debug("Failed to read provider settings: %s", exc) + return "", {} + return cfg.get("default_image_caption_provider_id") or "", cfg + + ToolExecutorResultT = T.TypeVar("ToolExecutorResultT") AwaitableResultT = T.TypeVar("AwaitableResultT") @@ -1061,13 +1079,12 @@ async def step(self): caption_parts = await self._caption_cached_tool_images( cached_images ) - if caption_parts: - self.run_context.messages.append( - Message(role="user", content=caption_parts) - ) - logger.debug( - f"Appended captions for {len(cached_images)} cached image(s) to context for LLM review" - ) + self.run_context.messages.append( + Message(role="user", content=caption_parts) + ) + logger.debug( + f"Appended captions for {len(cached_images)} cached image(s) to context for LLM review" + ) self.req.append_tool_calls_result(tool_calls_result) @@ -1075,18 +1092,7 @@ async def _caption_cached_tool_images( self, cached_images: list[CachedImage] ) -> list[TextPart]: """主模型不支持图片时,用默认图片转述模型为缓存图片生成一段文字描述,失败或未配置则退化为占位文本。""" - plugin_context = getattr(self.run_context.context, "context", None) - event = getattr(self.run_context.context, "event", None) - cfg: dict = {} - img_cap_prov_id = "" - if plugin_context is not None: - try: - cfg = plugin_context.get_config( - umo=getattr(event, "unified_msg_origin", None) - ).get("provider_settings", {}) - except Exception as exc: # noqa: BLE001 - logger.debug("Failed to read provider settings: %s", exc) - img_cap_prov_id = cfg.get("default_image_caption_provider_id") or "" + img_cap_prov_id, cfg = _get_image_caption_config(self.run_context) parts: list[TextPart] = [ TextPart( @@ -1100,26 +1106,9 @@ async def _caption_cached_tool_images( # 与 astr_main_agent._ensure_img_caption() 一致:一次调用处理全部图片, # 避免按图逐个调用导致长对话中工具循环被多次串行转述拖慢。 - caption = None - if img_cap_prov_id: - caption_paths = [ - cached_img.file_path - for cached_img in cached_images - if Path(cached_img.file_path).exists() - ] - if caption_paths: - try: - from astrbot.core.astr_main_agent import _request_img_caption - - caption = await _request_img_caption( - img_cap_prov_id, - cfg, - caption_paths, - plugin_context, - ) - except Exception as exc: # noqa: BLE001 - logger.error("Failed to caption tool images: %s", exc) - + caption = await self._request_tool_image_captions( + img_cap_prov_id, cfg, cached_images + ) if caption: parts.append(TextPart(text=f"{caption}")) else: @@ -1134,6 +1123,37 @@ async def _caption_cached_tool_images( ) return parts + async def _request_tool_image_captions( + self, + img_cap_prov_id: str, + cfg: dict, + cached_images: list[CachedImage], + ) -> str | None: + if not img_cap_prov_id: + return None + caption_paths = [ + cached_img.file_path + for cached_img in cached_images + if Path(cached_img.file_path).exists() + ] + if not caption_paths: + return None + plugin_context = getattr(self.run_context.context, "context", None) + if plugin_context is None: + return None + try: + from astrbot.core.astr_main_agent import _request_img_caption + + return await _request_img_caption( + img_cap_prov_id, + cfg, + caption_paths, + plugin_context, + ) + except Exception as exc: # noqa: BLE001 + logger.error("Failed to caption tool images: %s", exc) + return None + async def step_until_done( self, max_step: int ) -> T.AsyncGenerator[AgentResponse, None]: diff --git a/tests/test_tool_loop_agent_runner.py b/tests/test_tool_loop_agent_runner.py index b7903b443a..56d60dcd52 100644 --- a/tests/test_tool_loop_agent_runner.py +++ b/tests/test_tool_loop_agent_runner.py @@ -996,9 +996,11 @@ def _tool_image_runner_setup( monkeypatch, tmp_path, caption_context, - modalities: list[str] | None = ["tool_use"], + modalities: list[str] | None = None, tool_executor=MockMixedContentToolExecutor, ): + if modalities is None: + modalities = ["tool_use"] from astrbot.core.agent.tool_image_cache import CachedImage, tool_image_cache img_path = tmp_path / "tool_img.png"