Read interactive input files as UTF-8

python/src/typechat/_internal/interactive.py

@@ -12,7 +12,7 @@ async def process_requests(interactive_prompt: str, input_file_name: str | None,
         process_request: Async callback function that is invoked for each interactive input or each line in text file.
     """
     if input_file_name is not None:
-        with open(input_file_name, "r") as file:
+        with open(input_file_name, "r", encoding="utf-8") as file:
             lines = filter(str.rstrip, file)
             for line in lines:
                 if line.startswith("# "):

python/tests/test_interactive.py

@@ -0,0 +1,68 @@
+"""
+Tests for process_requests reading input files.
+
+Input files are read as UTF-8 regardless of the platform's default text encoding.
+Bare open() would instead use locale.getpreferredencoding(False), which is a non-UTF-8
+codec on many Windows installs (commonly cp1252) and cannot decode non-ASCII input.
+
+The platform default is simulated here rather than taken from the host locale, so these
+tests exercise the same code path on every platform and Python version.
+"""
+
+import asyncio
+import builtins
+from pathlib import Path
+from typing import Any
+
+import pytest
+from typechat._internal.interactive import process_requests
+
+# A line no non-UTF-8 single-byte codec can round-trip.
+NON_ASCII_LINE = "\u30b3\u30fc\u30d2\u30fc\u3092\u4e00\u3064\u304f\u3060\u3055\u3044"
+
+
+@pytest.fixture
+def default_encoding_is_cp1252(monkeypatch: pytest.MonkeyPatch):
+    """Make encoding-less open() calls behave as they do under a cp1252 locale."""
+    real_open = builtins.open
+
+    def fake_open(file: Any, mode: str = "r", *args: Any, **kwargs: Any) -> Any:
+        if "b" not in mode and kwargs.get("encoding") is None:
+            kwargs["encoding"] = "cp1252"
+        return real_open(file, mode, *args, **kwargs)
+
+    monkeypatch.setattr(builtins, "open", fake_open)
+
+
+def _write_utf8(tmp_path: Path, text: str) -> str:
+    path = tmp_path / "input.txt"
+    path.write_text(text + "\n", encoding="utf-8")
+    return str(path)
+
+
+def _collect(input_file_name: str) -> list[str]:
+    seen: list[str] = []
+
+    async def handler(request: str) -> None:
+        seen.append(request)
+
+    asyncio.run(process_requests("> ", input_file_name, handler))
+    return seen
+
+
+@pytest.mark.usefixtures("default_encoding_is_cp1252")
+def test_reads_non_ascii_input_file_under_non_utf8_default_encoding(tmp_path: Path):
+    input_file_name = _write_utf8(tmp_path, NON_ASCII_LINE)
+
+    requests = _collect(input_file_name)
+
+    assert [line.rstrip("\n") for line in requests] == [NON_ASCII_LINE]
+
+
+@pytest.mark.usefixtures("default_encoding_is_cp1252")
+def test_reads_ascii_input_file_under_non_utf8_default_encoding(tmp_path: Path):
+    input_file_name = _write_utf8(tmp_path, "one cappuccino please")
+
+    requests = _collect(input_file_name)
+
+    assert [line.rstrip("\n") for line in requests] == ["one cappuccino please"]