diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 4339ef5a61397dd..1d6b33fe854c943 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -250,7 +250,7 @@ def read(self, filename, strict=True): list of standard types, else to the list of non-standard types. """ - with open(filename, encoding='utf-8') as fp: + with open(filename, encoding='utf-8', errors='surrogateescape') as fp: self.readfp(fp, strict) def readfp(self, fp, strict=True): @@ -444,7 +444,7 @@ def init(files=None): def read_mime_types(file): try: - f = open(file, encoding='utf-8') + f = open(file, encoding='utf-8', errors='surrogateescape') except OSError: return None with f: diff --git a/Lib/test/test_free_threading/test_io.py b/Lib/test/test_free_threading/test_io.py index 057e0adf3b42bc4..a2b1ec8eb72bb1a 100644 --- a/Lib/test/test_free_threading/test_io.py +++ b/Lib/test/test_free_threading/test_io.py @@ -232,3 +232,23 @@ def reset_worker(): decoder.reset() run_concurrently([decode_worker] * 2 + [reset_worker] * 2) + + +class TextIOWrapperTest(TestCase): + def test_buffer_detach_race(self): + make = lambda: io.TextIOWrapper(io.BytesIO()) + slot = [make()] + + def reader(): + for _ in range(1000): + try: + slot[0].buffer + except ValueError: + pass + + def detacher(): + for _ in range(1000): + slot[0] = make() + slot[0].detach() + + run_concurrently([reader, detacher]) diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 19983fa3fa7628d..1e0f6664af0d066 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -67,9 +67,50 @@ def test_read_mime_types(self): with unittest.mock.patch.object(mimetypes, 'open', return_value=fp) as mock_open: mime_dict = mimetypes.read_mime_types(filename) - mock_open.assert_called_with(filename, encoding='utf-8') + mock_open.assert_called_with(filename, encoding='utf-8', + errors='surrogateescape') eq(mime_dict[".Français"], "application/no-mans-land") + def test_read_mime_types_invalid_utf8_comment(self): + with os_helper.temp_dir() as directory: + data = (b"# non-UTF-8 comment: \x83\n" + b"x-application/x-unittest pyunit\n") + file = os.path.join(directory, "sample.mimetype") + with open(file, "wb") as f: + f.write(data) + + mime_dict = mimetypes.read_mime_types(file) + self.assertEqual( + mime_dict[".pyunit"], "x-application/x-unittest") + + db = mimetypes.MimeTypes() + db.read(file) + self.assertEqual( + db.guess_file_type("sample.pyunit")[0], + "x-application/x-unittest") + + mimetypes.init(files=[file]) + self.assertEqual( + mimetypes.guess_file_type("sample.pyunit")[0], + "x-application/x-unittest") + + def test_read_mime_types_invalid_utf8_type(self): + # A non-UTF-8 byte in a type or extension (not only in a comment) is + # preserved via surrogateescape, so the mapping is not corrupted. + with os_helper.temp_dir() as directory: + data = (b"x-application/x-unittest pyunit\n" + b"application/bad\x83 badext\x83\n") + file = os.path.join(directory, "sample.mimetype") + with open(file, "wb") as f: + f.write(data) + + bad_type = b"application/bad\x83".decode("utf-8", "surrogateescape") + bad_ext = b".badext\x83".decode("utf-8", "surrogateescape") + + mime_dict = mimetypes.read_mime_types(file) + self.assertEqual(mime_dict[".pyunit"], "x-application/x-unittest") + self.assertEqual(mime_dict[bad_ext], bad_type) + def test_init_reinitializes(self): # Issue 4936: make sure an init starts clean # First, put some poison into the types table diff --git a/Misc/NEWS.d/next/Library/2026-06-10-00-00-01.gh-issue-117807.Cx1178.rst b/Misc/NEWS.d/next/Library/2026-06-10-00-00-01.gh-issue-117807.Cx1178.rst new file mode 100644 index 000000000000000..d6a874a84867a2f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-10-00-00-01.gh-issue-117807.Cx1178.rst @@ -0,0 +1,2 @@ +Fix :mod:`mimetypes` initialization from MIME map files containing invalid +UTF-8 bytes. diff --git a/Misc/NEWS.d/next/Library/2026-07-23-18-53-48.gh-issue-154523.XUSRBO.rst b/Misc/NEWS.d/next/Library/2026-07-23-18-53-48.gh-issue-154523.XUSRBO.rst new file mode 100644 index 000000000000000..23b25a26effb642 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-23-18-53-48.gh-issue-154523.XUSRBO.rst @@ -0,0 +1,2 @@ +Fixed data-race when calling :meth:`io.TextIOBase.detach` in +:term:`free-threaded build`. diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 8d59bda5f74b386..3c682cb2f271aef 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -1331,4 +1331,29 @@ _io_TextIOWrapper__CHUNK_SIZE_set(PyObject *self, PyObject *value, void *Py_UNUS return return_value; } -/*[clinic end generated code: output=8c571c9dba87d2b1 input=a9049054013a1b77]*/ + +#if !defined(_io_TextIOWrapper_buffer_DOCSTR) +# define _io_TextIOWrapper_buffer_DOCSTR NULL +#endif +#if defined(_IO_TEXTIOWRAPPER_BUFFER_GETSETDEF) +# undef _IO_TEXTIOWRAPPER_BUFFER_GETSETDEF +# define _IO_TEXTIOWRAPPER_BUFFER_GETSETDEF {"buffer", (getter)_io_TextIOWrapper_buffer_get, (setter)_io_TextIOWrapper_buffer_set, _io_TextIOWrapper_buffer_DOCSTR}, +#else +# define _IO_TEXTIOWRAPPER_BUFFER_GETSETDEF {"buffer", (getter)_io_TextIOWrapper_buffer_get, NULL, _io_TextIOWrapper_buffer_DOCSTR}, +#endif + +static PyObject * +_io_TextIOWrapper_buffer_get_impl(textio *self); + +static PyObject * +_io_TextIOWrapper_buffer_get(PyObject *self, void *Py_UNUSED(context)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_TextIOWrapper_buffer_get_impl((textio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} +/*[clinic end generated code: output=e34c75e1d2a12084 input=a9049054013a1b77]*/ diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 5b2a20a30c28cb2..ea8ed2713d8a146 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -3424,6 +3424,19 @@ _io_TextIOWrapper__CHUNK_SIZE_set_impl(textio *self, PyObject *value) return 0; } +/*[clinic input] +@critical_section +@getter +_io.TextIOWrapper.buffer +[clinic start generated code]*/ + +static PyObject * +_io_TextIOWrapper_buffer_get_impl(textio *self) +/*[clinic end generated code: output=d265a34555aa5d4b input=5951cfa148f7350a]*/ +{ + return Py_XNewRef(buffer_access_safe(self)); +} + static PyMethodDef incrementalnewlinedecoder_methods[] = { _IO_INCREMENTALNEWLINEDECODER_DECODE_METHODDEF _IO_INCREMENTALNEWLINEDECODER_GETSTATE_METHODDEF @@ -3482,7 +3495,6 @@ static PyMethodDef textiowrapper_methods[] = { static PyMemberDef textiowrapper_members[] = { {"encoding", _Py_T_OBJECT, offsetof(textio, encoding), Py_READONLY}, - {"buffer", _Py_T_OBJECT, offsetof(textio, buffer), Py_READONLY}, {"line_buffering", Py_T_BOOL, offsetof(textio, line_buffering), Py_READONLY}, {"write_through", Py_T_BOOL, offsetof(textio, write_through), Py_READONLY}, {"_finalizing", Py_T_BOOL, offsetof(textio, finalizing), 0}, @@ -3497,6 +3509,7 @@ static PyGetSetDef textiowrapper_getset[] = { _IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF _IO_TEXTIOWRAPPER_ERRORS_GETSETDEF _IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF + _IO_TEXTIOWRAPPER_BUFFER_GETSETDEF {NULL} };