From 11bccd996addb3ee16a0b363a4f9fe7f519cb20a Mon Sep 17 00:00:00 2001 From: isheng Date: Tue, 7 Jul 2026 02:44:23 +0800 Subject: [PATCH] fix(fetch): handle 5xx responses and add timeout in robots.txt check Two fixes in check_may_autonomously_fetch_url: 1. 5xx responses from robots.txt were silently falling through to the robots.txt parsing block, causing error pages to be parsed as robots.txt content. Now raises McpError for server errors. 2. The robots.txt fetch had no timeout, unlike the main fetch_url function which uses timeout=30. Added the same 30s timeout to prevent the client from hanging indefinitely on unresponsive robots.txt endpoints. --- src/fetch/src/mcp_server_fetch/server.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/fetch/src/mcp_server_fetch/server.py b/src/fetch/src/mcp_server_fetch/server.py index b42c7b1f6b..bb874aabaa 100644 --- a/src/fetch/src/mcp_server_fetch/server.py +++ b/src/fetch/src/mcp_server_fetch/server.py @@ -78,6 +78,7 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: robot_txt_url, follow_redirects=True, headers={"User-Agent": user_agent}, + timeout=30, ) except HTTPError: raise McpError(ErrorData( @@ -91,6 +92,11 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: )) elif 400 <= response.status_code < 500: return + elif response.status_code >= 500: + raise McpError(ErrorData( + code=INTERNAL_ERROR, + message=f"Failed to fetch robots.txt {robot_txt_url} - server error (status {response.status_code})", + )) robot_txt = response.text processed_robot_txt = "\n".join( line for line in robot_txt.splitlines() if not line.strip().startswith("#")