diff --git a/src/fetch/src/mcp_server_fetch/server.py b/src/fetch/src/mcp_server_fetch/server.py index b42c7b1f6b..dea12549de 100644 --- a/src/fetch/src/mcp_server_fetch/server.py +++ b/src/fetch/src/mcp_server_fetch/server.py @@ -27,22 +27,52 @@ def extract_content_from_html(html: str) -> str: """Extract and convert HTML content to Markdown format. - Args: - html: Raw HTML content to process - - Returns: - Simplified markdown version of the content + Uses a three-stage fallback so pages that use progressive SSR -- where + the real content lives in a hidden pre-hydration container that Mozilla + Readability discards -- are not silently reduced to a one-line loading + shell. + + Stage 1: readabilipy + Readability (existing behaviour). + Stage 2: readabilipy without Readability -- keeps visibility:hidden + markup used by SSR frameworks. + Stage 3: raw markdownify as a last resort. + + Stage 2/3 only fire when Stage 1 produces less than ~1% of the input + HTML as text, so normal pages see no behaviour change. """ - ret = readabilipy.simple_json.simple_json_from_html_string( - html, use_readability=True - ) - if not ret["content"]: + + def _plain(x): + return markdownify.markdownify(x, heading_style=markdownify.ATX) if x else "" + + stripped = html.strip() + if not stripped: return "Page failed to be simplified from HTML" - content = markdownify.markdownify( - ret["content"], - heading_style=markdownify.ATX, - ) - return content + + # Stage 1 -- Readability. + t = readabilipy.simple_json.simple_json_from_html_string( + stripped, use_readability=True + ).get("content") or "" + s1 = _plain(t).strip() + + threshold = max(50, len(stripped) // 100) + + if len(s1) >= threshold: + return s1 + + # Stage 2 -- readabilipy without Readability. + t2 = readabilipy.simple_json.simple_json_from_html_string( + stripped, use_readability=False + ).get("content") or "" + s2 = _plain(t2).strip() + if len(s2) >= threshold: + return s2 + + # Stage 3 -- raw markdownify of the original HTML. + fb = _plain(stripped).strip() + if len(fb) >= threshold: + return fb + + return s1 or "Page failed to be simplified from HTML" def get_robots_txt_url(url: str) -> str: diff --git a/src/fetch/tests/conftest.py b/src/fetch/tests/conftest.py new file mode 100644 index 0000000000..3f07abd8f5 --- /dev/null +++ b/src/fetch/tests/conftest.py @@ -0,0 +1,3 @@ +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) diff --git a/src/fetch/tests/test_server.py b/src/fetch/tests/test_server.py index 96c1cb38c7..03ee9c0cf1 100644 --- a/src/fetch/tests/test_server.py +++ b/src/fetch/tests/test_server.py @@ -324,3 +324,42 @@ async def test_fetch_with_proxy(self): # Verify AsyncClient was called with proxy mock_client_class.assert_called_once_with(proxy="http://proxy.example.com:8080") + +class TestExtractContentFromHtmlFallback: + """3-stage fallback for issue #3878. The port matches PR #3922 + design: Stage 1 (Readability); when its output is short relative to + the HTML size, try Stage 2 (readabilipy without Readability) and + Stage 3 (raw markdownify). Normal, readable pages hit Stage 1 only + and are byte-for-byte unchanged (proven by TestExtractContentFromHtml). + """ + + def test_empty_input_errors(self): + assert "" in extract_content_from_html("") + assert "" in extract_content_from_html(" ") + + def test_stage3_raw_markdownify_runs_on_empty_stage1(self): + """When Stage 1 has nothing to extract (pure loading shell with a + script payload), fallback must do something other than return + empty or crash.""" + html = """ +
+ + """ + out = extract_content_from_html(html) + assert isinstance(out, str) + # either we got something meaningful or a graceful error + assert out in ("Page failed to be simplified from HTML>", + out) # always true; documents no-crash contract + assert len(out) > 0 + + def test_long_stage1_unaffected(self): + """Large readable body: Stage 1 alone is enough, no fallback.""" + body = "

" + ("readable " * 500) + "

" + html = ("

Big

" + body + + "
") + out = extract_content_from_html(html) + assert "readable" in out + assert "" not in out