Evaluate Markdown tool responses

Add opt-in lossless Markdown rendering for the selected high-usage tools, plus a reproducible token and byte benchmark harness with public response-body-free results.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 83bff8f9-aa73-410d-be31-af14dad4cb15

Author: tommaso-moro

evals/markdown_response/.gitignore

@@ -0,0 +1,3 @@
+.venv/
+__pycache__/
+out/

evals/markdown_response/README.md

@@ -0,0 +1,36 @@
+# Markdown response eval
+
+This harness compares the current JSON responses with the lossless Markdown representation enabled by `markdown_output` for `get_file_contents`, `pull_request_read`, `create_pull_request`, `list_pull_requests`, and `search_pull_requests`.
+
+Each read-only scenario is captured once from the MCP server and then converted offline with the same Go renderer used by the feature flag. This ensures both arms contain identical source data. The directory, list, and search tools are measured both with their full output and after existing `fields` filtering. `create_pull_request` uses a representative fixture matching its exact `{id, url}` response shape and is never called against GitHub.
+
+The headline measurement serializes the complete MCP `tools/call` result with compact JSON, approximating the model-facing tool-result message and accounting for the escaping paid when JSON is nested inside a text content block. The output also records inner-text bytes and tokens for structured text responses. Token counts use `tiktoken` with `o200k_base`; `--approx` forces a chars/4 fallback for smoke tests.
+
+## Run
+
+```bash
+cd evals/markdown_response
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+export GITHUB_PERSONAL_ACCESS_TOKEN=...
+python3 markdown_response_eval.py
+```
+
+By default the live scenarios use public data from `github/github-mcp-server` and pull request `#2658`. The script writes metrics only to `out/markdown-response-eval.json`; it does not persist response bodies.
+
+```bash
+python3 markdown_response_eval.py --owner cli --repo cli --pull-number 12345 --per-page 30
+```
+
+To measure the table break-even point, run the same dataset with multiple page sizes:
+
+```bash
+for page_size in 1 10 30; do
+  python3 markdown_response_eval.py --pull-number 2797 --per-page "$page_size" --out "out/page-${page_size}.json"
+done
+```
+
+The included `results/github-mcp-server-2026-07-31.json` file is a response-body-free summary of this scale run.
+
+`get_file_contents.file` and `pull_request_read.get_diff` are included as controls. They already return resource/plain-text content and should show zero change.

evals/markdown_response/_mcp_client.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""Minimal MCP stdio client for response capture."""
+
+from __future__ import annotations
+
+import json
+import os
+import select
+import shlex
+import subprocess
+import sys
+from pathlib import Path
+from types import TracebackType
+from typing import Any
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+PROTOCOL_VERSION = "2025-06-18"
+
+
+class MCPServer:
+    def __init__(
+        self,
+        server_cmd: str = "go run ./cmd/github-mcp-server stdio",
+        extra_args: list[str] | None = None,
+        timeout: float = 180.0,
+    ) -> None:
+        self.cmd = shlex.split(server_cmd) + list(extra_args or [])
+        self.timeout = timeout
+        self.proc: subprocess.Popen[str] | None = None
+        self._id = 0
+
+    def __enter__(self) -> "MCPServer":
+        self.start()
+        return self
+
+    def __exit__(
+        self,
+        _exc_type: type[BaseException] | None,
+        _exc_value: BaseException | None,
+        _traceback: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def start(self) -> None:
+        if not os.environ.get("GITHUB_PERSONAL_ACCESS_TOKEN"):
+            raise RuntimeError("GITHUB_PERSONAL_ACCESS_TOKEN is required for live captures")
+
+        print(f"[mcp] starting: {' '.join(self.cmd)}", file=sys.stderr)
+        self.proc = subprocess.Popen(
+            self.cmd,
+            cwd=REPO_ROOT,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=sys.stderr,
+            text=True,
+            env=os.environ.copy(),
+        )
+        self._request(
+            "initialize",
+            {
+                "protocolVersion": PROTOCOL_VERSION,
+                "capabilities": {},
+                "clientInfo": {"name": "markdown-response-eval", "version": "0"},
+            },
+        )
+        self._notify("notifications/initialized")
+
+    def call_tool(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]:
+        return self._request("tools/call", {"name": name, "arguments": arguments})
+
+    def close(self) -> None:
+        if self.proc is None:
+            return
+        if self.proc.stdin is not None:
+            try:
+                self.proc.stdin.close()
+            except BrokenPipeError:
+                pass
+        self.proc.terminate()
+        try:
+            self.proc.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            self.proc.kill()
+            self.proc.wait(timeout=10)
+        self.proc = None
+
+    def _send(self, payload: dict[str, Any]) -> None:
+        if self.proc is None or self.proc.stdin is None:
+            raise RuntimeError("MCP server is not running")
+        self.proc.stdin.write(json.dumps(payload) + "\n")
+        self.proc.stdin.flush()
+
+    def _notify(self, method: str, params: dict[str, Any] | None = None) -> None:
+        self._send({"jsonrpc": "2.0", "method": method, "params": params or {}})
+
+    def _read(self) -> dict[str, Any]:
+        if self.proc is None or self.proc.stdout is None:
+            raise RuntimeError("MCP server is not running")
+        while True:
+            ready, _, _ = select.select([self.proc.stdout], [], [], self.timeout)
+            if not ready:
+                raise TimeoutError("timed out waiting for the MCP server")
+            line = self.proc.stdout.readline()
+            if line == "":
+                raise EOFError("MCP server closed stdout unexpectedly")
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                return json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+    def _request(self, method: str, params: dict[str, Any]) -> dict[str, Any]:
+        self._id += 1
+        request_id = self._id
+        self._send(
+            {
+                "jsonrpc": "2.0",
+                "id": request_id,
+                "method": method,
+                "params": params,
+            }
+        )
+        while True:
+            message = self._read()
+            if message.get("id") != request_id:
+                continue
+            if "error" in message:
+                raise RuntimeError(f"{method} error: {message['error']}")
+            result = message.get("result")
+            if not isinstance(result, dict):
+                raise RuntimeError(f"{method} returned a non-object result")
+            return result

evals/markdown_response/_tokenize.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""Token counting for the Markdown response eval."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+
+def get_tokenizer(approx: bool = False) -> tuple[Callable[[str], int], str]:
+    if approx:
+        return (lambda text: max(1, len(text) // 4), "approx(chars/4)")
+
+    try:
+        import tiktoken
+    except ImportError as exc:
+        raise RuntimeError(
+            "tiktoken is required; install requirements.txt or pass --approx"
+        ) from exc
+
+    encoder = tiktoken.get_encoding("o200k_base")
+    return (lambda text: len(encoder.encode(text)), "tiktoken(o200k_base)")