From d9058f0846fd898c574a94be65d184424ea1a9e5 Mon Sep 17 00:00:00 2001 From: syf2211 Date: Sun, 28 Jun 2026 04:03:47 +0000 Subject: [PATCH 1/2] fix(python): forward binary tool results in HandlePendingToolCall RPC ToolResult.binary_results_for_llm and session_log were dropped when building ExternalToolTextResultForLlm in _execute_tool_and_respond, so image/binary tool outputs never reached the model. Fixes #1644 --- python/copilot/session.py | 17 +++++++-------- python/copilot/tools.py | 33 ++++++++++++++++++++++++++++ python/test_tools.py | 46 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 9 deletions(-) diff --git a/python/copilot/session.py b/python/copilot/session.py index d9fc04dcef..c5e13579de 100644 --- a/python/copilot/session.py +++ b/python/copilot/session.py @@ -36,7 +36,6 @@ CanvasProviderOpenResult, ClientSessionApiHandlers, CommandsHandlePendingCommandRequest, - ExternalToolTextResultForLlm, HandlePendingToolCallRequest, LogRequest, MCPOauthHandlePendingRequest, @@ -83,7 +82,13 @@ from .generated.session_events import ( ReasoningSummary as _RpcReasoningSummary, ) -from .tools import Tool, ToolHandler, ToolInvocation, ToolResult +from .tools import ( + Tool, + ToolHandler, + ToolInvocation, + ToolResult, + tool_result_to_external_tool_text_result_for_llm, +) logger = logging.getLogger(__name__) @@ -2063,13 +2068,7 @@ async def _execute_tool_and_respond( await self.rpc.tools.handle_pending_tool_call( HandlePendingToolCallRequest( request_id=request_id, - result=ExternalToolTextResultForLlm( - text_result_for_llm=tool_result.text_result_for_llm, - error=tool_result.error, - result_type=tool_result.result_type, - tool_references=tool_result.tool_references, - tool_telemetry=tool_result.tool_telemetry, - ), + result=tool_result_to_external_tool_text_result_for_llm(tool_result), ) ) log_timing( diff --git a/python/copilot/tools.py b/python/copilot/tools.py index b96e303e3b..762b79c45b 100644 --- a/python/copilot/tools.py +++ b/python/copilot/tools.py @@ -18,6 +18,12 @@ if TYPE_CHECKING: from .generated.rpc import CurrentToolMetadata +from .generated.rpc import ( + ExternalToolTextResultForLlm, + ExternalToolTextResultForLlmBinaryResultsForLlm, + ExternalToolTextResultForLlmBinaryResultsForLlmType, +) + ToolResultType = Literal["success", "failure", "rejected", "denied", "timeout"] @@ -408,3 +414,30 @@ def convert_mcp_call_tool_result(call_result: dict[str, Any]) -> ToolResult: result_type="failure" if call_result.get("isError") is True else "success", binary_results_for_llm=binary_results if binary_results else None, ) + + +def tool_result_to_external_tool_text_result_for_llm( + tool_result: ToolResult, +) -> ExternalToolTextResultForLlm: + """Convert a ToolResult into the RPC payload sent to HandlePendingToolCall.""" + binary_results_for_llm = None + if tool_result.binary_results_for_llm: + binary_results_for_llm = [ + ExternalToolTextResultForLlmBinaryResultsForLlm( + data=binary_result.data, + mime_type=binary_result.mime_type, + type=ExternalToolTextResultForLlmBinaryResultsForLlmType(binary_result.type), + description=binary_result.description or None, + ) + for binary_result in tool_result.binary_results_for_llm + ] + + return ExternalToolTextResultForLlm( + text_result_for_llm=tool_result.text_result_for_llm, + binary_results_for_llm=binary_results_for_llm, + error=tool_result.error, + result_type=tool_result.result_type, + session_log=tool_result.session_log, + tool_references=tool_result.tool_references, + tool_telemetry=tool_result.tool_telemetry, + ) diff --git a/python/test_tools.py b/python/test_tools.py index 90498c2b8e..646f17c03a 100644 --- a/python/test_tools.py +++ b/python/test_tools.py @@ -8,10 +8,12 @@ from copilot import define_tool from copilot.generated.rpc import ExternalToolTextResultForLlm from copilot.tools import ( + ToolBinaryResult, ToolInvocation, ToolResult, _normalize_result, convert_mcp_call_tool_result, + tool_result_to_external_tool_text_result_for_llm, ) @@ -550,3 +552,47 @@ def test_tool_references_round_trip_from_wire(self): } ) assert wire.tool_references == ["alpha", "beta"] + + +class TestToolResultToExternalToolTextResultForLlm: + def test_forwards_binary_results_and_session_log(self): + tool_result = ToolResult( + text_result_for_llm="screenshot captured", + binary_results_for_llm=[ + ToolBinaryResult( + data="base64data", + mime_type="image/png", + type="image", + description="screenshot.png", + ) + ], + session_log="tool execution details", + tool_telemetry={"duration_ms": 42}, + ) + + rpc_result = tool_result_to_external_tool_text_result_for_llm(tool_result) + + assert rpc_result.text_result_for_llm == "screenshot captured" + assert rpc_result.session_log == "tool execution details" + assert rpc_result.tool_telemetry == {"duration_ms": 42} + assert rpc_result.binary_results_for_llm is not None + assert len(rpc_result.binary_results_for_llm) == 1 + assert rpc_result.binary_results_for_llm[0].data == "base64data" + assert rpc_result.binary_results_for_llm[0].mime_type == "image/png" + assert rpc_result.binary_results_for_llm[0].type.value == "image" + assert rpc_result.binary_results_for_llm[0].description == "screenshot.png" + + def test_omits_binary_results_when_none(self): + tool_result = ToolResult(text_result_for_llm="done") + rpc_result = tool_result_to_external_tool_text_result_for_llm(tool_result) + assert rpc_result.binary_results_for_llm is None + assert rpc_result.session_log is None + + def test_forwards_tool_references(self): + tool_result = ToolResult( + text_result_for_llm="found tools", + result_type="success", + tool_references=["get_weather", "check_status"], + ) + rpc_result = tool_result_to_external_tool_text_result_for_llm(tool_result) + assert rpc_result.tool_references == ["get_weather", "check_status"] From 2c7c73345ede582429348f2a23bad8d034a2a8a2 Mon Sep 17 00:00:00 2001 From: Steve Sanderson Date: Fri, 31 Jul 2026 14:59:49 +0000 Subject: [PATCH 2/2] fix(go): forward binary_results_for_llm and session_log in HandlePendingToolCall RPC ToolResult.BinaryResultsForLLM and SessionLog were dropped when building ExternalToolTextResultForLlm in executeToolAndRespond, mirroring the same bug fixed in Python by this PR. Binary tool outputs (e.g. images) returned by Go tool handlers never reached the model. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- go/session.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/go/session.go b/go/session.go index 3059e3b3ae..56458f27a5 100644 --- a/go/session.go +++ b/go/session.go @@ -1575,6 +1575,20 @@ func (s *Session) executeToolAndRespond(requestID, toolName, toolCallID string, if result.Error != "" { rpcResult.Error = &result.Error } + if result.SessionLog != "" { + rpcResult.SessionLog = &result.SessionLog + } + for _, b := range result.BinaryResultsForLLM { + entry := rpc.ExternalToolTextResultForLlmBinaryResultsForLlm{ + Data: b.Data, + MIMEType: b.MIMEType, + Type: rpc.ExternalToolTextResultForLlmBinaryResultsForLlmType(b.Type), + } + if b.Description != "" { + entry.Description = &b.Description + } + rpcResult.BinaryResultsForLlm = append(rpcResult.BinaryResultsForLlm, entry) + } s.RPC.Tools.HandlePendingToolCall(ctx, &rpc.HandlePendingToolCallRequest{ RequestID: requestID, Result: rpcResult,