diff --git a/python/src/typechat/_internal/interactive.py b/python/src/typechat/_internal/interactive.py index 8c137b66..d7ebddcf 100644 --- a/python/src/typechat/_internal/interactive.py +++ b/python/src/typechat/_internal/interactive.py @@ -12,7 +12,7 @@ async def process_requests(interactive_prompt: str, input_file_name: str | None, process_request: Async callback function that is invoked for each interactive input or each line in text file. """ if input_file_name is not None: - with open(input_file_name, "r") as file: + with open(input_file_name, "r", encoding="utf-8") as file: lines = filter(str.rstrip, file) for line in lines: if line.startswith("# "): diff --git a/python/tests/test_interactive.py b/python/tests/test_interactive.py new file mode 100644 index 00000000..e5c8e83f --- /dev/null +++ b/python/tests/test_interactive.py @@ -0,0 +1,68 @@ +""" +Tests for process_requests reading input files. + +Input files are read as UTF-8 regardless of the platform's default text encoding. +Bare open() would instead use locale.getpreferredencoding(False), which is a non-UTF-8 +codec on many Windows installs (commonly cp1252) and cannot decode non-ASCII input. + +The platform default is simulated here rather than taken from the host locale, so these +tests exercise the same code path on every platform and Python version. +""" + +import asyncio +import builtins +from pathlib import Path +from typing import Any + +import pytest +from typechat._internal.interactive import process_requests + +# A line no non-UTF-8 single-byte codec can round-trip. +NON_ASCII_LINE = "\u30b3\u30fc\u30d2\u30fc\u3092\u4e00\u3064\u304f\u3060\u3055\u3044" + + +@pytest.fixture +def default_encoding_is_cp1252(monkeypatch: pytest.MonkeyPatch): + """Make encoding-less open() calls behave as they do under a cp1252 locale.""" + real_open = builtins.open + + def fake_open(file: Any, mode: str = "r", *args: Any, **kwargs: Any) -> Any: + if "b" not in mode and kwargs.get("encoding") is None: + kwargs["encoding"] = "cp1252" + return real_open(file, mode, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", fake_open) + + +def _write_utf8(tmp_path: Path, text: str) -> str: + path = tmp_path / "input.txt" + path.write_text(text + "\n", encoding="utf-8") + return str(path) + + +def _collect(input_file_name: str) -> list[str]: + seen: list[str] = [] + + async def handler(request: str) -> None: + seen.append(request) + + asyncio.run(process_requests("> ", input_file_name, handler)) + return seen + + +@pytest.mark.usefixtures("default_encoding_is_cp1252") +def test_reads_non_ascii_input_file_under_non_utf8_default_encoding(tmp_path: Path): + input_file_name = _write_utf8(tmp_path, NON_ASCII_LINE) + + requests = _collect(input_file_name) + + assert [line.rstrip("\n") for line in requests] == [NON_ASCII_LINE] + + +@pytest.mark.usefixtures("default_encoding_is_cp1252") +def test_reads_ascii_input_file_under_non_utf8_default_encoding(tmp_path: Path): + input_file_name = _write_utf8(tmp_path, "one cappuccino please") + + requests = _collect(input_file_name) + + assert [line.rstrip("\n") for line in requests] == ["one cappuccino please"]