Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/fetch/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ The fetch tool will truncate the response, but by using the `start_index` argume
- `max_length` (integer, optional): Maximum number of characters to return (default: 5000)
- `start_index` (integer, optional): Start content from this character index (default: 0)
- `raw` (boolean, optional): Get raw content without markdown conversion (default: false)
- `timeout` (number, optional): Request timeout in seconds for this fetch; overrides the server default (default: 30)

### Prompts

Expand Down Expand Up @@ -170,6 +171,12 @@ This can be customized by adding the argument `--user-agent=YourUserAgent` to th

The server can be configured to use a proxy by using the `--proxy-url` argument.

### Customization - Timeout

By default, requests time out after 30 seconds. The default can be changed with the `--timeout` argument (in seconds) or
the `FETCH_TIMEOUT` environment variable, and a single request can override it with the `timeout` tool argument
(most specific wins).

## Windows Configuration

If you're experiencing timeout issues on Windows, you may need to set the `PYTHONIOENCODING` environment variable to ensure proper character encoding:
Expand Down
30 changes: 28 additions & 2 deletions src/fetch/src/mcp_server_fetch/__init__.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
from .server import serve
from .server import serve, DEFAULT_REQUEST_TIMEOUT


def main():
"""MCP Fetch Server - HTTP fetching functionality for MCP"""
import argparse
import asyncio
import os

parser = argparse.ArgumentParser(
description="give a model the ability to make web requests"
Expand All @@ -17,8 +18,33 @@ def main():
)
parser.add_argument("--proxy-url", type=str, help="Proxy URL to use for requests")

def positive_timeout(value: str) -> float:
seconds = float(value)
if seconds <= 0:
raise argparse.ArgumentTypeError(
"timeout must be a positive number of seconds"
)
return seconds

env_timeout = os.environ.get("FETCH_TIMEOUT")
try:
default_timeout = (
positive_timeout(env_timeout) if env_timeout else DEFAULT_REQUEST_TIMEOUT
)
except (ValueError, argparse.ArgumentTypeError) as error:
parser.error(f"invalid FETCH_TIMEOUT environment variable: {error}")

parser.add_argument(
"--timeout",
type=positive_timeout,
default=default_timeout,
help="Request timeout in seconds (default: 30, or the FETCH_TIMEOUT env var)",
)

args = parser.parse_args()
asyncio.run(serve(args.user_agent, args.ignore_robots_txt, args.proxy_url))
asyncio.run(
serve(args.user_agent, args.ignore_robots_txt, args.proxy_url, args.timeout)
)


if __name__ == "__main__":
Expand Down
37 changes: 33 additions & 4 deletions src/fetch/src/mcp_server_fetch/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,12 @@
DEFAULT_USER_AGENT_AUTONOMOUS = "ModelContextProtocol/1.0 (Autonomous; +https://github.com/modelcontextprotocol/servers)"
DEFAULT_USER_AGENT_MANUAL = "ModelContextProtocol/1.0 (User-Specified; +https://github.com/modelcontextprotocol/servers)"

# Default per-request timeout, in seconds, for the outbound content fetch.
# Matches httpx's timeout unit and the value previously hardcoded here.
# Overridable via the serve() timeout argument, the --timeout CLI flag, the
# FETCH_TIMEOUT environment variable, or the per-request "timeout" tool argument.
DEFAULT_REQUEST_TIMEOUT = 30.0


def extract_content_from_html(html: str) -> str:
"""Extract and convert HTML content to Markdown format.
Expand Down Expand Up @@ -109,7 +115,11 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url:


async def fetch_url(
url: str, user_agent: str, force_raw: bool = False, proxy_url: str | None = None
url: str,
user_agent: str,
force_raw: bool = False,
proxy_url: str | None = None,
timeout: float = DEFAULT_REQUEST_TIMEOUT,
) -> Tuple[str, str]:
"""
Fetch the URL and return the content in a form ready for the LLM, as well as a prefix string with status information.
Expand All @@ -122,7 +132,7 @@ async def fetch_url(
url,
follow_redirects=True,
headers={"User-Agent": user_agent},
timeout=30,
timeout=timeout,
)
except HTTPError as e:
raise McpError(ErrorData(code=INTERNAL_ERROR, message=f"Failed to fetch {url}: {e!r}"))
Expand Down Expand Up @@ -176,19 +186,30 @@ class Fetch(BaseModel):
description="Get the actual HTML content of the requested page, without simplification.",
),
]
timeout: Annotated[
float | None,
Field(
default=None,
description="Request timeout in seconds for this fetch. Overrides the server-wide default when set.",
gt=0,
),
]


async def serve(
custom_user_agent: str | None = None,
ignore_robots_txt: bool = False,
proxy_url: str | None = None,
timeout: float = DEFAULT_REQUEST_TIMEOUT,
) -> None:
"""Run the fetch MCP server.

Args:
custom_user_agent: Optional custom User-Agent string to use for requests
ignore_robots_txt: Whether to ignore robots.txt restrictions
proxy_url: Optional proxy URL to use for requests
timeout: Default request timeout in seconds for the content fetch, used
when a fetch call does not specify its own timeout
"""
server = Server("mcp-fetch")
user_agent_autonomous = custom_user_agent or DEFAULT_USER_AGENT_AUTONOMOUS
Expand Down Expand Up @@ -231,11 +252,17 @@ async def call_tool(name, arguments: dict) -> list[TextContent]:
if not url:
raise McpError(ErrorData(code=INVALID_PARAMS, message="URL is required"))

request_timeout = args.timeout if args.timeout is not None else timeout

if not ignore_robots_txt:
await check_may_autonomously_fetch_url(url, user_agent_autonomous, proxy_url)

content, prefix = await fetch_url(
url, user_agent_autonomous, force_raw=args.raw, proxy_url=proxy_url
url,
user_agent_autonomous,
force_raw=args.raw,
proxy_url=proxy_url,
timeout=request_timeout,
)
original_length = len(content)
if args.start_index >= original_length:
Expand All @@ -262,7 +289,9 @@ async def get_prompt(name: str, arguments: dict | None) -> GetPromptResult:
url = arguments["url"]

try:
content, prefix = await fetch_url(url, user_agent_manual, proxy_url=proxy_url)
content, prefix = await fetch_url(
url, user_agent_manual, proxy_url=proxy_url, timeout=timeout
)
# TODO: after SDK bug is addressed, don't catch the exception
except McpError as e:
return GetPromptResult(
Expand Down
86 changes: 86 additions & 0 deletions src/fetch/tests/test_server.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,20 @@
"""Tests for the fetch MCP server."""

import os
import sys

import pytest
from unittest.mock import AsyncMock, patch, MagicMock
from mcp.shared.exceptions import McpError

from mcp_server_fetch import main
from mcp_server_fetch.server import (
extract_content_from_html,
get_robots_txt_url,
check_may_autonomously_fetch_url,
fetch_url,
DEFAULT_USER_AGENT_AUTONOMOUS,
DEFAULT_REQUEST_TIMEOUT,
)


Expand Down Expand Up @@ -324,3 +329,84 @@ async def test_fetch_with_proxy(self):

# Verify AsyncClient was called with proxy
mock_client_class.assert_called_once_with(proxy="http://proxy.example.com:8080")

@pytest.mark.asyncio
async def test_fetch_uses_default_timeout(self):
"""Test that fetch_url applies the default timeout to the request."""
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.text = '{"data": "test"}'
mock_response.headers = {"content-type": "application/json"}

with patch("httpx.AsyncClient") as mock_client_class:
mock_client = AsyncMock()
mock_client.get = AsyncMock(return_value=mock_response)
mock_client_class.return_value.__aenter__ = AsyncMock(return_value=mock_client)
mock_client_class.return_value.__aexit__ = AsyncMock(return_value=None)

await fetch_url(
"https://example.com/data",
DEFAULT_USER_AGENT_AUTONOMOUS,
)

assert mock_client.get.call_args.kwargs["timeout"] == DEFAULT_REQUEST_TIMEOUT

@pytest.mark.asyncio
async def test_fetch_uses_custom_timeout(self):
"""Test that an explicit timeout is passed through to the request."""
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.text = '{"data": "test"}'
mock_response.headers = {"content-type": "application/json"}

with patch("httpx.AsyncClient") as mock_client_class:
mock_client = AsyncMock()
mock_client.get = AsyncMock(return_value=mock_response)
mock_client_class.return_value.__aenter__ = AsyncMock(return_value=mock_client)
mock_client_class.return_value.__aexit__ = AsyncMock(return_value=None)

await fetch_url(
"https://api.example.com/data",
DEFAULT_USER_AGENT_AUTONOMOUS,
timeout=60.0,
)

assert mock_client.get.call_args.kwargs["timeout"] == 60.0


class TestServerTimeoutConfiguration:
"""Tests for timeout configuration via the CLI flag and environment variable."""

def _serve_timeout_for(self, argv, env=None):
"""Run main() with the given argv/env and return the timeout passed to serve()."""
import mcp_server_fetch

with patch.object(mcp_server_fetch, "serve", new=AsyncMock()) as mock_serve, \
patch.dict(os.environ, env or {}, clear=False), \
patch.object(sys, "argv", argv):
if not (env and "FETCH_TIMEOUT" in env):
os.environ.pop("FETCH_TIMEOUT", None)
main()

# serve(custom_user_agent, ignore_robots_txt, proxy_url, timeout)
return mock_serve.call_args.args[3]

def test_default_timeout_when_unset(self):
"""The default timeout is used when neither flag nor env var is set."""
assert self._serve_timeout_for(["mcp-server-fetch"]) == DEFAULT_REQUEST_TIMEOUT

def test_timeout_from_cli_flag(self):
"""The --timeout flag sets the server default."""
assert self._serve_timeout_for(["mcp-server-fetch", "--timeout", "45"]) == 45.0

def test_timeout_from_env_var(self):
"""FETCH_TIMEOUT sets the server default when no flag is given."""
assert self._serve_timeout_for(
["mcp-server-fetch"], {"FETCH_TIMEOUT": "50"}
) == 50.0

def test_cli_flag_overrides_env_var(self):
"""The --timeout flag takes precedence over FETCH_TIMEOUT."""
assert self._serve_timeout_for(
["mcp-server-fetch", "--timeout", "45"], {"FETCH_TIMEOUT": "50"}
) == 45.0
Loading