From 70b28cc02bbfc1324837c6048e47d37e5ab182ab Mon Sep 17 00:00:00 2001 From: Mohammed Alkindi Date: Tue, 15 Sep 2026 15:39:13 +0300 Subject: [PATCH] Read interactive input files as UTF-8 Bare open() follows locale.getpreferredencoding(False), so on a Windows box with a cp1252 code page an input file either raises UnicodeDecodeError or, when its characters happen to map into cp1252, is read as mojibake with no error and passed to the request handler. The TypeScript twin reads the same files with fs.readFileSync().toString(), which is UTF-8 on every platform, so this also makes the two ports agree. --- python/src/typechat/_internal/interactive.py | 2 +- python/tests/test_interactive.py | 68 ++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 python/tests/test_interactive.py diff --git a/python/src/typechat/_internal/interactive.py b/python/src/typechat/_internal/interactive.py index 8c137b66..d7ebddcf 100644 --- a/python/src/typechat/_internal/interactive.py +++ b/python/src/typechat/_internal/interactive.py @@ -12,7 +12,7 @@ async def process_requests(interactive_prompt: str, input_file_name: str | None, process_request: Async callback function that is invoked for each interactive input or each line in text file. """ if input_file_name is not None: - with open(input_file_name, "r") as file: + with open(input_file_name, "r", encoding="utf-8") as file: lines = filter(str.rstrip, file) for line in lines: if line.startswith("# "): diff --git a/python/tests/test_interactive.py b/python/tests/test_interactive.py new file mode 100644 index 00000000..e5c8e83f --- /dev/null +++ b/python/tests/test_interactive.py @@ -0,0 +1,68 @@ +""" +Tests for process_requests reading input files. + +Input files are read as UTF-8 regardless of the platform's default text encoding. +Bare open() would instead use locale.getpreferredencoding(False), which is a non-UTF-8 +codec on many Windows installs (commonly cp1252) and cannot decode non-ASCII input. + +The platform default is simulated here rather than taken from the host locale, so these +tests exercise the same code path on every platform and Python version. +""" + +import asyncio +import builtins +from pathlib import Path +from typing import Any + +import pytest +from typechat._internal.interactive import process_requests + +# A line no non-UTF-8 single-byte codec can round-trip. +NON_ASCII_LINE = "\u30b3\u30fc\u30d2\u30fc\u3092\u4e00\u3064\u304f\u3060\u3055\u3044" + + +@pytest.fixture +def default_encoding_is_cp1252(monkeypatch: pytest.MonkeyPatch): + """Make encoding-less open() calls behave as they do under a cp1252 locale.""" + real_open = builtins.open + + def fake_open(file: Any, mode: str = "r", *args: Any, **kwargs: Any) -> Any: + if "b" not in mode and kwargs.get("encoding") is None: + kwargs["encoding"] = "cp1252" + return real_open(file, mode, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", fake_open) + + +def _write_utf8(tmp_path: Path, text: str) -> str: + path = tmp_path / "input.txt" + path.write_text(text + "\n", encoding="utf-8") + return str(path) + + +def _collect(input_file_name: str) -> list[str]: + seen: list[str] = [] + + async def handler(request: str) -> None: + seen.append(request) + + asyncio.run(process_requests("> ", input_file_name, handler)) + return seen + + +@pytest.mark.usefixtures("default_encoding_is_cp1252") +def test_reads_non_ascii_input_file_under_non_utf8_default_encoding(tmp_path: Path): + input_file_name = _write_utf8(tmp_path, NON_ASCII_LINE) + + requests = _collect(input_file_name) + + assert [line.rstrip("\n") for line in requests] == [NON_ASCII_LINE] + + +@pytest.mark.usefixtures("default_encoding_is_cp1252") +def test_reads_ascii_input_file_under_non_utf8_default_encoding(tmp_path: Path): + input_file_name = _write_utf8(tmp_path, "one cappuccino please") + + requests = _collect(input_file_name) + + assert [line.rstrip("\n") for line in requests] == ["one cappuccino please"]