From 7da9d35c70b667397677ef0d6bbfdc127b39a746 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 17:40:32 +0800 Subject: [PATCH 01/21] perf: accelerate numeric match expansion --- README.md | 3 +- benchmarks/api_hotpaths.py | 3 +- pcre_ext/pcre2.c | 194 +++++++++++++++++++++++++++++++++- tests/test_expand_fastpath.py | 100 ++++++++++++++++++ 4 files changed, 295 insertions(+), 5 deletions(-) create mode 100644 tests/test_expand_fastpath.py diff --git a/README.md b/README.md index c3cec99..01e6e4f 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 +* 08/10/2026 **Call-local `Match.expand` fast path**: exact text and bytes templates containing one unambiguous numeric capture now render directly from the immutable C Match snapshot, without importing the compatibility parser or retaining a parsed-template cache entry. Pinned A/B measurements against merged `main` improve matched `[\\1]` expansion by **20.8x** on Python 3.10 and **16.7x** on free-threaded Python 3.14t/GIL=0; unmatched captures reach **25.5x/20.7x**, and bytes reach **37.4x/31.7x**. Named, escaped, multi-reference, ambiguous two-digit, subclass, and invalid templates continue through the fully compatible parser. ⚡🛡️ * 08/10/2026 **Literal split/substitution/findall fast paths**: exact plain-literal `Pattern.split` calls now use the immutable built-in splitter after construction-time validation, measuring **2.1x** faster than the prior C dispatch on Python 3.10 and **1.7x** faster on free-threaded Python 3.14t/GIL=0; delimiter-heavy multi-character literals reach roughly **4.8x**. Literal `Pattern.subn` and module-level `sub`/`subn` now use native replace/count primitives, reaching about **15x** on short repeated tokens and **3x** on delimiter-heavy text. Literal `findall` uses non-overlapping native count/list construction, reaching about **9x** on short repeated tokens and **8x** on delimiter-heavy text. Regex metacharacters, explicit flags, subclasses, and buffer subjects remain on the compatibility-safe PCRE2 path. ⚡ * 08/09/2026 **API hot-path update**: large `parallel_map(findall)` workloads now reach **11.5x** speedup on Python 3.10 and **11.25x** on free-threaded Python 3.14t/GIL=0 with 12 performance-tier workers. Ordered `parallel_map(search)` reaches **8.57x** and **7.85x**, respectively; one-item and up to eight tiny explicit `parallel_map` subjects now avoid executor setup (the one-item case measures **13.3x** faster on Python 3.10 and **27.7x** on 3.14t), default bound `Pattern.split` is another **1.6x/1.5x** faster on 3.10/3.14t, and default bound literal `Pattern.subn` is about **1.5x** faster on Python 3.10. Canonical module helpers retain their optimized dispatch while their wrapper/template caches are thread-scoped, size-bounded, and invalidated across live workers. Repeated backreference `Match.expand()` avoids reparsing within the active cache context, while captured values returned by `Match.groups()` remain call-local so a long-lived Match does not retain an additional copy of large captures. 🧵⚡ * 08/08/2026 **0.6.0**: `findall`, `finditer`, `sub`/`subn`, `split`, and `match`/`search`/`fullmatch` are now up to **46x faster** than `stdlib.re` and **48x faster** than `regex` on `finditer`/`findall` workloads, **13x** on `split`, and **2–9x** on `sub`/`subn` backref workloads, with full `re` semantics. Free-threaded `findall` reaches **13.8x** vs `re` on 8 threads. 🚀⚡ @@ -82,7 +83,7 @@ hard CPU affinity. | Bound one-character literal `Pattern.split` | **2.1x** | **1.7x** | | Bound backreference `sub` hot path | **1.38 μs** | **1.14 μs** | | Repeated call-local `Match.groups()` | **~0.05 μs** | **~0.05 μs** | -| Repeated `Match.expand(r"[\\1]")` | **5.23 μs** | **1.10 μs** | +| Call-local `Match.expand(r"[\\1]")` | **0.07 μs** | **0.07 μs** | | Repeated default `compile("(x)")` | **0.49 μs** | **0.38 μs** | | Repeated integer-flagged `compile("x", CASELESS)` | **1.16 μs** | **0.81 μs** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index 9e87c12..19e57c5 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -22,7 +22,6 @@ import pcre - RUNS = int(os.getenv("PYPCRE_BENCH_RUNS", "50000")) @@ -50,6 +49,8 @@ def main() -> int: ("bound.sub.literal", lambda: pattern.sub("[X]", short_subject)), ("bound.sub.backref", lambda: pattern.sub(r"[\1]", short_subject)), ("match.groups", captured.groups), + ("match.expand.numeric", lambda: captured.expand(r"[\1]")), + ("match.expand.named", lambda: captured.expand(r"[\g<1>]")), ("module.match", lambda: pcre.match("(x)", subject)), ("module.search", lambda: pcre.search("(x)", subject)), ("module.fullmatch", lambda: pcre.fullmatch("(x)", subject)), diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index a0dcd90..b64d40f 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -1064,6 +1064,169 @@ Match_get_regs(MatchObject *self, void *closure) return cached; } +static PyObject * +match_expand_simple_numeric(MatchObject *self, + PyObject *template_obj, + Py_ssize_t slash_index, + Py_ssize_t template_length, + int *handled) +{ + *handled = 0; + if (slash_index < 0 || slash_index + 1 >= template_length) { + return NULL; + } + + int digit = -1; + if (!self->subject_is_bytes && PyUnicode_CheckExact(template_obj)) { + Py_UCS4 character = PyUnicode_ReadChar(template_obj, slash_index + 1); + if (character == (Py_UCS4)-1 && PyErr_Occurred()) { + return NULL; + } + if (character < '1' || character > '9') { + return NULL; + } + if (slash_index + 2 < template_length) { + Py_UCS4 following = PyUnicode_ReadChar(template_obj, slash_index + 2); + if (following == (Py_UCS4)-1 && PyErr_Occurred()) { + return NULL; + } + if (following >= '0' && following <= '9') { + return NULL; + } + Py_ssize_t next_slash = PyUnicode_FindChar( + template_obj, + '\\', + slash_index + 2, + template_length, + 1 + ); + if (next_slash >= 0) { + return NULL; + } + if (PyErr_Occurred()) { + return NULL; + } + } + digit = (int)(character - '0'); + } else if (self->subject_is_bytes && PyBytes_CheckExact(template_obj)) { + const unsigned char *template_data = (const unsigned char *)PyBytes_AS_STRING(template_obj); + unsigned char character = template_data[slash_index + 1]; + if (character < '1' || character > '9') { + return NULL; + } + if (slash_index + 2 < template_length) { + unsigned char following = template_data[slash_index + 2]; + if (following >= '0' && following <= '9') { + return NULL; + } + if (memchr(template_data + slash_index + 2, + '\\', + (size_t)(template_length - slash_index - 2)) != NULL) { + return NULL; + } + } + digit = (int)(character - '0'); + } else { + return NULL; + } + + if (digit <= 0 || (size_t)digit >= self->ovec_count) { + return NULL; + } + + PyObject *group = match_get_group_value(self, (Py_ssize_t)digit); + if (group == NULL) { + return NULL; + } + *handled = 1; + + if (group == Py_None) { + Py_DECREF(group); + group = NULL; + } + + Py_ssize_t suffix_start = slash_index + 2; + Py_ssize_t suffix_length = template_length - suffix_start; + if (slash_index == 0 && suffix_length == 0) { + if (group != NULL) { + return group; + } + return self->subject_is_bytes + ? PyBytes_FromStringAndSize("", 0) + : PyUnicode_New(0, 127); + } + + Py_ssize_t group_length = group == NULL ? 0 : PyObject_Length(group); + if (group_length < 0) { + Py_XDECREF(group); + return NULL; + } + if (group_length > PY_SSIZE_T_MAX - slash_index - suffix_length) { + Py_XDECREF(group); + PyErr_NoMemory(); + return NULL; + } + Py_ssize_t result_length = slash_index + group_length + suffix_length; + + if (self->subject_is_bytes) { + PyObject *result = PyBytes_FromStringAndSize(NULL, result_length); + if (result == NULL) { + Py_XDECREF(group); + return NULL; + } + char *output = PyBytes_AS_STRING(result); + const char *template_data = PyBytes_AS_STRING(template_obj); + memcpy(output, template_data, (size_t)slash_index); + if (group != NULL && group_length > 0) { + memcpy(output + slash_index, + PyBytes_AS_STRING(group), + (size_t)group_length); + } + memcpy(output + slash_index + group_length, + template_data + suffix_start, + (size_t)suffix_length); + Py_XDECREF(group); + return result; + } + + Py_UCS4 max_character = PyUnicode_MAX_CHAR_VALUE(template_obj); + if (group != NULL) { + Py_UCS4 group_max = PyUnicode_MAX_CHAR_VALUE(group); + if (group_max > max_character) { + max_character = group_max; + } + } + PyObject *result = PyUnicode_New(result_length, max_character); + if (result == NULL) { + Py_XDECREF(group); + return NULL; + } + if (slash_index > 0 && + PyUnicode_CopyCharacters(result, 0, template_obj, 0, slash_index) < 0) { + Py_DECREF(result); + Py_XDECREF(group); + return NULL; + } + if (group != NULL && group_length > 0 && + PyUnicode_CopyCharacters(result, slash_index, group, 0, group_length) < 0) { + Py_DECREF(result); + Py_DECREF(group); + return NULL; + } + if (suffix_length > 0 && + PyUnicode_CopyCharacters(result, + slash_index + group_length, + template_obj, + suffix_start, + suffix_length) < 0) { + Py_DECREF(result); + Py_XDECREF(group); + return NULL; + } + Py_XDECREF(group); + return result; +} + static PyObject * Match_expand(MatchObject *self, PyObject *template_obj) { @@ -1074,10 +1237,21 @@ Match_expand(MatchObject *self, PyObject *template_obj) in PyUnicode_FromObject. */ if (!self->subject_is_bytes && PyUnicode_Check(template_obj)) { Py_ssize_t template_length = PyUnicode_GET_LENGTH(template_obj); - if (PyUnicode_FindChar(template_obj, '\\', 0, template_length, 1) < 0 && - !PyErr_Occurred()) { + Py_ssize_t slash_index = PyUnicode_FindChar( + template_obj, '\\', 0, template_length, 1 + ); + if (slash_index < 0 && !PyErr_Occurred()) { return PyUnicode_FromObject(template_obj); } + if (slash_index >= 0 && PyUnicode_CheckExact(template_obj)) { + int handled = 0; + PyObject *result = match_expand_simple_numeric( + self, template_obj, slash_index, template_length, &handled + ); + if (handled || result != NULL || PyErr_Occurred()) { + return result; + } + } PyErr_Clear(); } if (self->subject_is_bytes && @@ -1088,9 +1262,23 @@ Match_expand(MatchObject *self, PyObject *template_obj) Py_ssize_t template_length = PyBytes_Check(template_obj) ? PyBytes_GET_SIZE(template_obj) : PyByteArray_GET_SIZE(template_obj); - if (memchr(template_data, '\\', (size_t)template_length) == NULL) { + const char *slash = memchr(template_data, '\\', (size_t)template_length); + if (slash == NULL) { return PyBytes_FromObject(template_obj); } + if (PyBytes_CheckExact(template_obj)) { + int handled = 0; + PyObject *result = match_expand_simple_numeric( + self, + template_obj, + (Py_ssize_t)(slash - template_data), + template_length, + &handled + ); + if (handled || result != NULL || PyErr_Occurred()) { + return result; + } + } } /* Delegate template parsing to the Python compatibility helper. */ diff --git a/tests/test_expand_fastpath.py b/tests/test_expand_fastpath.py new file mode 100644 index 0000000..30f55b3 --- /dev/null +++ b/tests/test_expand_fastpath.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: 2025 ModelCloud.ai +# SPDX-FileCopyrightText: 2025 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +from __future__ import annotations + +import concurrent.futures +import re + +import pytest + +import pcre +from pcre import re_compat + + +@pytest.mark.parametrize( + ("pattern", "subject", "template", "expected"), + [ + (r"(a)", "a", r"\1", "a"), + (r"(a)", "a", r"[\1]", "[a]"), + (r"(é)", "é", "前\\1後", "前é後"), + (r"(a)?(b)?", "a", r"[\2]", "[]"), + (b"(a)", b"a", rb"\1", b"a"), + (b"(a)", b"a", rb"[\1]", b"[a]"), + (b"(a)?(b)?", b"a", rb"[\2]", b"[]"), + ], +) +def test_single_numeric_expand_is_exact_and_call_local( + pattern, + subject, + template, + expected, + monkeypatch: pytest.MonkeyPatch, +) -> None: + match = pcre.compile(pattern).fullmatch(subject) + assert match is not None + re_compat._cached_expand_template.cache_clear() + monkeypatch.setattr( + re_compat, + "expand_match_template", + lambda *args: pytest.fail("simple numeric expansion reached Python parser"), + ) + + assert match.expand(template) == expected + assert re_compat._expand_template_cache_size() == 0 + + +@pytest.mark.parametrize("template", [r"\12", r"\\1", r"\g<1>"]) +def test_ambiguous_or_extended_expand_stays_on_compatibility_parser( + template: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + match = pcre.compile("(a)" * 12).fullmatch("a" * 12) + assert match is not None + sentinel = object() + monkeypatch.setattr( + re_compat, + "expand_match_template", + lambda *args: sentinel, + ) + + assert match.expand(template) is sentinel + + +def test_single_numeric_expand_is_safe_on_one_match_across_threads() -> None: + match = pcre.compile(r"(é)").fullmatch("é") + assert match is not None + + def expand_many() -> None: + for _ in range(10_000): + assert match.expand("前\\1後") == "前é後" + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(lambda _: expand_many(), range(8))) + + +@pytest.mark.parametrize( + ("pattern", "subject"), + [ + (r"(a)?(b)?", "a"), + (r"(é)?(β)?", "éβ"), + (b"(a)?(b)?", b"a"), + ], +) +def test_numeric_expand_differential_matrix(pattern, subject) -> None: + expected_match = re.fullmatch(pattern, subject) + actual_match = pcre.fullmatch(pattern, subject) + assert expected_match is not None + assert actual_match is not None + + prefixes = ("", "[", "前", "$", "12") + suffixes = ("", "]", "後", "$", r"\g<2>") + for group in (1, 2): + for prefix in prefixes: + for suffix in suffixes: + template = f"{prefix}\\{group}{suffix}" + if isinstance(pattern, bytes): + template = template.encode() + assert actual_match.expand(template) == expected_match.expand(template) From 4173b788ea23943bf690ee474ac3da3a4a78afe6 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 17:46:36 +0800 Subject: [PATCH 02/21] perf: accelerate explicit numeric expansion --- README.md | 3 +- benchmarks/api_hotpaths.py | 2 +- pcre_ext/pcre2.c | 276 ++++++++++++++++++++++++---------- tests/test_expand_fastpath.py | 23 ++- 4 files changed, 225 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index 01e6e4f..76d51c8 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 -* 08/10/2026 **Call-local `Match.expand` fast path**: exact text and bytes templates containing one unambiguous numeric capture now render directly from the immutable C Match snapshot, without importing the compatibility parser or retaining a parsed-template cache entry. Pinned A/B measurements against merged `main` improve matched `[\\1]` expansion by **20.8x** on Python 3.10 and **16.7x** on free-threaded Python 3.14t/GIL=0; unmatched captures reach **25.5x/20.7x**, and bytes reach **37.4x/31.7x**. Named, escaped, multi-reference, ambiguous two-digit, subclass, and invalid templates continue through the fully compatible parser. ⚡🛡️ +* 08/10/2026 **Call-local `Match.expand` fast paths**: exact text and bytes templates containing one unambiguous numeric capture now render directly from the immutable C Match snapshot, without importing the compatibility parser or retaining a parsed-template cache entry. Pinned A/B measurements against merged `main` improve matched `[\\1]` expansion by **20.8x** on Python 3.10 and **16.7x** on free-threaded Python 3.14t/GIL=0; unmatched captures reach **25.5x/20.7x**, and bytes reach **37.4x/31.7x**. Explicit numeric `[\\g<1>]` improves by **14.3x/13.8x**, with checked multi-digit references such as `\\g<12>` reaching **63.3x/13.1x**. Named, escaped, multi-reference, ambiguous two-digit, subclass, and invalid templates continue through the fully compatible parser. ⚡🛡️ * 08/10/2026 **Literal split/substitution/findall fast paths**: exact plain-literal `Pattern.split` calls now use the immutable built-in splitter after construction-time validation, measuring **2.1x** faster than the prior C dispatch on Python 3.10 and **1.7x** faster on free-threaded Python 3.14t/GIL=0; delimiter-heavy multi-character literals reach roughly **4.8x**. Literal `Pattern.subn` and module-level `sub`/`subn` now use native replace/count primitives, reaching about **15x** on short repeated tokens and **3x** on delimiter-heavy text. Literal `findall` uses non-overlapping native count/list construction, reaching about **9x** on short repeated tokens and **8x** on delimiter-heavy text. Regex metacharacters, explicit flags, subclasses, and buffer subjects remain on the compatibility-safe PCRE2 path. ⚡ * 08/09/2026 **API hot-path update**: large `parallel_map(findall)` workloads now reach **11.5x** speedup on Python 3.10 and **11.25x** on free-threaded Python 3.14t/GIL=0 with 12 performance-tier workers. Ordered `parallel_map(search)` reaches **8.57x** and **7.85x**, respectively; one-item and up to eight tiny explicit `parallel_map` subjects now avoid executor setup (the one-item case measures **13.3x** faster on Python 3.10 and **27.7x** on 3.14t), default bound `Pattern.split` is another **1.6x/1.5x** faster on 3.10/3.14t, and default bound literal `Pattern.subn` is about **1.5x** faster on Python 3.10. Canonical module helpers retain their optimized dispatch while their wrapper/template caches are thread-scoped, size-bounded, and invalidated across live workers. Repeated backreference `Match.expand()` avoids reparsing within the active cache context, while captured values returned by `Match.groups()` remain call-local so a long-lived Match does not retain an additional copy of large captures. 🧵⚡ * 08/08/2026 **0.6.0**: `findall`, `finditer`, `sub`/`subn`, `split`, and `match`/`search`/`fullmatch` are now up to **46x faster** than `stdlib.re` and **48x faster** than `regex` on `finditer`/`findall` workloads, **13x** on `split`, and **2–9x** on `sub`/`subn` backref workloads, with full `re` semantics. Free-threaded `findall` reaches **13.8x** vs `re` on 8 threads. 🚀⚡ @@ -84,6 +84,7 @@ hard CPU affinity. | Bound backreference `sub` hot path | **1.38 μs** | **1.14 μs** | | Repeated call-local `Match.groups()` | **~0.05 μs** | **~0.05 μs** | | Call-local `Match.expand(r"[\\1]")` | **0.07 μs** | **0.07 μs** | +| Call-local `Match.expand(r"[\\g<1>]")` | **0.11 μs** | **0.08 μs** | | Repeated default `compile("(x)")` | **0.49 μs** | **0.38 μs** | | Repeated integer-flagged `compile("x", CASELESS)` | **1.16 μs** | **0.81 μs** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index 19e57c5..9dca1a6 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -50,7 +50,7 @@ def main() -> int: ("bound.sub.backref", lambda: pattern.sub(r"[\1]", short_subject)), ("match.groups", captured.groups), ("match.expand.numeric", lambda: captured.expand(r"[\1]")), - ("match.expand.named", lambda: captured.expand(r"[\g<1>]")), + ("match.expand.explicit", lambda: captured.expand(r"[\g<1>]")), ("module.match", lambda: pcre.match("(x)", subject)), ("module.search", lambda: pcre.search("(x)", subject)), ("module.fullmatch", lambda: pcre.fullmatch("(x)", subject)), diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index b64d40f..f4b91d4 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -1065,76 +1065,24 @@ Match_get_regs(MatchObject *self, void *closure) } static PyObject * -match_expand_simple_numeric(MatchObject *self, - PyObject *template_obj, - Py_ssize_t slash_index, - Py_ssize_t template_length, - int *handled) +match_expand_render_reference(MatchObject *self, + PyObject *template_obj, + Py_ssize_t prefix_length, + Py_ssize_t suffix_start, + Py_ssize_t group_index, + int *handled) { *handled = 0; - if (slash_index < 0 || slash_index + 1 >= template_length) { + Py_ssize_t template_length = self->subject_is_bytes + ? PyBytes_GET_SIZE(template_obj) + : PyUnicode_GET_LENGTH(template_obj); + if (prefix_length < 0 || suffix_start < prefix_length || + suffix_start > template_length || group_index < 0 || + (size_t)group_index >= self->ovec_count) { return NULL; } - int digit = -1; - if (!self->subject_is_bytes && PyUnicode_CheckExact(template_obj)) { - Py_UCS4 character = PyUnicode_ReadChar(template_obj, slash_index + 1); - if (character == (Py_UCS4)-1 && PyErr_Occurred()) { - return NULL; - } - if (character < '1' || character > '9') { - return NULL; - } - if (slash_index + 2 < template_length) { - Py_UCS4 following = PyUnicode_ReadChar(template_obj, slash_index + 2); - if (following == (Py_UCS4)-1 && PyErr_Occurred()) { - return NULL; - } - if (following >= '0' && following <= '9') { - return NULL; - } - Py_ssize_t next_slash = PyUnicode_FindChar( - template_obj, - '\\', - slash_index + 2, - template_length, - 1 - ); - if (next_slash >= 0) { - return NULL; - } - if (PyErr_Occurred()) { - return NULL; - } - } - digit = (int)(character - '0'); - } else if (self->subject_is_bytes && PyBytes_CheckExact(template_obj)) { - const unsigned char *template_data = (const unsigned char *)PyBytes_AS_STRING(template_obj); - unsigned char character = template_data[slash_index + 1]; - if (character < '1' || character > '9') { - return NULL; - } - if (slash_index + 2 < template_length) { - unsigned char following = template_data[slash_index + 2]; - if (following >= '0' && following <= '9') { - return NULL; - } - if (memchr(template_data + slash_index + 2, - '\\', - (size_t)(template_length - slash_index - 2)) != NULL) { - return NULL; - } - } - digit = (int)(character - '0'); - } else { - return NULL; - } - - if (digit <= 0 || (size_t)digit >= self->ovec_count) { - return NULL; - } - - PyObject *group = match_get_group_value(self, (Py_ssize_t)digit); + PyObject *group = match_get_group_value(self, group_index); if (group == NULL) { return NULL; } @@ -1145,9 +1093,8 @@ match_expand_simple_numeric(MatchObject *self, group = NULL; } - Py_ssize_t suffix_start = slash_index + 2; Py_ssize_t suffix_length = template_length - suffix_start; - if (slash_index == 0 && suffix_length == 0) { + if (prefix_length == 0 && suffix_length == 0) { if (group != NULL) { return group; } @@ -1161,12 +1108,12 @@ match_expand_simple_numeric(MatchObject *self, Py_XDECREF(group); return NULL; } - if (group_length > PY_SSIZE_T_MAX - slash_index - suffix_length) { + if (group_length > PY_SSIZE_T_MAX - prefix_length - suffix_length) { Py_XDECREF(group); PyErr_NoMemory(); return NULL; } - Py_ssize_t result_length = slash_index + group_length + suffix_length; + Py_ssize_t result_length = prefix_length + group_length + suffix_length; if (self->subject_is_bytes) { PyObject *result = PyBytes_FromStringAndSize(NULL, result_length); @@ -1176,13 +1123,13 @@ match_expand_simple_numeric(MatchObject *self, } char *output = PyBytes_AS_STRING(result); const char *template_data = PyBytes_AS_STRING(template_obj); - memcpy(output, template_data, (size_t)slash_index); + memcpy(output, template_data, (size_t)prefix_length); if (group != NULL && group_length > 0) { - memcpy(output + slash_index, + memcpy(output + prefix_length, PyBytes_AS_STRING(group), (size_t)group_length); } - memcpy(output + slash_index + group_length, + memcpy(output + prefix_length + group_length, template_data + suffix_start, (size_t)suffix_length); Py_XDECREF(group); @@ -1201,21 +1148,21 @@ match_expand_simple_numeric(MatchObject *self, Py_XDECREF(group); return NULL; } - if (slash_index > 0 && - PyUnicode_CopyCharacters(result, 0, template_obj, 0, slash_index) < 0) { + if (prefix_length > 0 && + PyUnicode_CopyCharacters(result, 0, template_obj, 0, prefix_length) < 0) { Py_DECREF(result); Py_XDECREF(group); return NULL; } if (group != NULL && group_length > 0 && - PyUnicode_CopyCharacters(result, slash_index, group, 0, group_length) < 0) { + PyUnicode_CopyCharacters(result, prefix_length, group, 0, group_length) < 0) { Py_DECREF(result); Py_DECREF(group); return NULL; } if (suffix_length > 0 && PyUnicode_CopyCharacters(result, - slash_index + group_length, + prefix_length + group_length, template_obj, suffix_start, suffix_length) < 0) { @@ -1227,6 +1174,167 @@ match_expand_simple_numeric(MatchObject *self, return result; } +static PyObject * +match_expand_simple_numeric(MatchObject *self, + PyObject *template_obj, + Py_ssize_t slash_index, + Py_ssize_t template_length, + int *handled) +{ + *handled = 0; + if (slash_index < 0 || slash_index + 1 >= template_length) { + return NULL; + } + + int digit = -1; + if (!self->subject_is_bytes && PyUnicode_CheckExact(template_obj)) { + Py_UCS4 character = PyUnicode_ReadChar(template_obj, slash_index + 1); + if (character == (Py_UCS4)-1 && PyErr_Occurred()) { + return NULL; + } + if (character < '1' || character > '9') { + return NULL; + } + if (slash_index + 2 < template_length) { + Py_UCS4 following = PyUnicode_ReadChar(template_obj, slash_index + 2); + if (following == (Py_UCS4)-1 && PyErr_Occurred()) { + return NULL; + } + if (following >= '0' && following <= '9') { + return NULL; + } + Py_ssize_t next_slash = PyUnicode_FindChar( + template_obj, + '\\', + slash_index + 2, + template_length, + 1 + ); + if (next_slash >= 0) { + return NULL; + } + if (PyErr_Occurred()) { + return NULL; + } + } + digit = (int)(character - '0'); + } else if (self->subject_is_bytes && PyBytes_CheckExact(template_obj)) { + const unsigned char *template_data = (const unsigned char *)PyBytes_AS_STRING(template_obj); + unsigned char character = template_data[slash_index + 1]; + if (character < '1' || character > '9') { + return NULL; + } + if (slash_index + 2 < template_length) { + unsigned char following = template_data[slash_index + 2]; + if (following >= '0' && following <= '9') { + return NULL; + } + if (memchr(template_data + slash_index + 2, + '\\', + (size_t)(template_length - slash_index - 2)) != NULL) { + return NULL; + } + } + digit = (int)(character - '0'); + } else { + return NULL; + } + + if (digit <= 0 || (size_t)digit >= self->ovec_count) { + return NULL; + } + return match_expand_render_reference( + self, template_obj, slash_index, slash_index + 2, digit, handled + ); +} + +static PyObject * +match_expand_explicit_numeric(MatchObject *self, + PyObject *template_obj, + Py_ssize_t slash_index, + Py_ssize_t template_length, + int *handled) +{ + *handled = 0; + if (slash_index < 0 || template_length - slash_index < 5) { + return NULL; + } + + Py_ssize_t cursor = slash_index + 1; + Py_ssize_t group_index = 0; + if (!self->subject_is_bytes && PyUnicode_CheckExact(template_obj)) { + if (PyUnicode_ReadChar(template_obj, cursor) != 'g' || + PyUnicode_ReadChar(template_obj, cursor + 1) != '<') { + return NULL; + } + cursor += 2; + Py_ssize_t digit_start = cursor; + while (cursor < template_length) { + Py_UCS4 character = PyUnicode_ReadChar(template_obj, cursor); + if (character == '>') { + break; + } + if (character < '0' || character > '9' || + group_index > (PY_SSIZE_T_MAX - (character - '0')) / 10) { + return NULL; + } + group_index = group_index * 10 + (Py_ssize_t)(character - '0'); + cursor += 1; + } + if (cursor == digit_start || cursor >= template_length) { + return NULL; + } + if (cursor + 1 < template_length) { + Py_ssize_t next_slash = PyUnicode_FindChar( + template_obj, '\\', cursor + 1, template_length, 1 + ); + if (next_slash >= 0) { + return NULL; + } + if (PyErr_Occurred()) { + return NULL; + } + } + } else if (self->subject_is_bytes && PyBytes_CheckExact(template_obj)) { + const unsigned char *template_data = (const unsigned char *)PyBytes_AS_STRING(template_obj); + if (template_data[cursor] != 'g' || template_data[cursor + 1] != '<') { + return NULL; + } + cursor += 2; + Py_ssize_t digit_start = cursor; + while (cursor < template_length) { + unsigned char character = template_data[cursor]; + if (character == '>') { + break; + } + if (character < '0' || character > '9' || + group_index > (PY_SSIZE_T_MAX - (character - '0')) / 10) { + return NULL; + } + group_index = group_index * 10 + (Py_ssize_t)(character - '0'); + cursor += 1; + } + if (cursor == digit_start || cursor >= template_length) { + return NULL; + } + if (cursor + 1 < template_length && + memchr(template_data + cursor + 1, + '\\', + (size_t)(template_length - cursor - 1)) != NULL) { + return NULL; + } + } else { + return NULL; + } + + if ((size_t)group_index >= self->ovec_count) { + return NULL; + } + return match_expand_render_reference( + self, template_obj, slash_index, cursor + 1, group_index, handled + ); +} + static PyObject * Match_expand(MatchObject *self, PyObject *template_obj) { @@ -1251,6 +1359,12 @@ Match_expand(MatchObject *self, PyObject *template_obj) if (handled || result != NULL || PyErr_Occurred()) { return result; } + result = match_expand_explicit_numeric( + self, template_obj, slash_index, template_length, &handled + ); + if (handled || result != NULL || PyErr_Occurred()) { + return result; + } } PyErr_Clear(); } @@ -1278,6 +1392,16 @@ Match_expand(MatchObject *self, PyObject *template_obj) if (handled || result != NULL || PyErr_Occurred()) { return result; } + result = match_expand_explicit_numeric( + self, + template_obj, + (Py_ssize_t)(slash - template_data), + template_length, + &handled + ); + if (handled || result != NULL || PyErr_Occurred()) { + return result; + } } } diff --git a/tests/test_expand_fastpath.py b/tests/test_expand_fastpath.py index 30f55b3..3e8e2a4 100644 --- a/tests/test_expand_fastpath.py +++ b/tests/test_expand_fastpath.py @@ -24,6 +24,18 @@ (b"(a)", b"a", rb"\1", b"a"), (b"(a)", b"a", rb"[\1]", b"[a]"), (b"(a)?(b)?", b"a", rb"[\2]", b"[]"), + (r"(a)", "a", r"\g<0>", "a"), + (r"(a)", "a", r"[\g<1>]", "[a]"), + (r"(é)", "é", "前\\g<1>後", "前é後"), + (b"(a)", b"a", rb"\g<0>", b"a"), + (b"(a)", b"a", rb"[\g<1>]", b"[a]"), + ( + "(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)", + "abcdefghijkl", + r"[\g<12>]", + "[l]", + ), + (r"(a)", "a", r"[\g<001>]", "[a]"), ], ) def test_single_numeric_expand_is_exact_and_call_local( @@ -46,7 +58,16 @@ def test_single_numeric_expand_is_exact_and_call_local( assert re_compat._expand_template_cache_size() == 0 -@pytest.mark.parametrize("template", [r"\12", r"\\1", r"\g<1>"]) +@pytest.mark.parametrize( + "template", + [ + r"\12", + r"\\1", + r"\g", + r"\g<999999999999999999999999999>", + r"\g<13>", + ], +) def test_ambiguous_or_extended_expand_stays_on_compatibility_parser( template: str, monkeypatch: pytest.MonkeyPatch, From 5b3b109e98e154788df321195d4b37956a156891 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 17:50:02 +0800 Subject: [PATCH 03/21] perf: accelerate named match expansion --- README.md | 3 +- benchmarks/api_hotpaths.py | 4 + pcre_ext/pcre2.c | 159 ++++++++++++++++++++++++++++ tests/test_expand_fastpath.py | 37 ++++++- tests/test_python_coverage_audit.py | 6 +- 5 files changed, 204 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 76d51c8..a6b25e6 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 -* 08/10/2026 **Call-local `Match.expand` fast paths**: exact text and bytes templates containing one unambiguous numeric capture now render directly from the immutable C Match snapshot, without importing the compatibility parser or retaining a parsed-template cache entry. Pinned A/B measurements against merged `main` improve matched `[\\1]` expansion by **20.8x** on Python 3.10 and **16.7x** on free-threaded Python 3.14t/GIL=0; unmatched captures reach **25.5x/20.7x**, and bytes reach **37.4x/31.7x**. Explicit numeric `[\\g<1>]` improves by **14.3x/13.8x**, with checked multi-digit references such as `\\g<12>` reaching **63.3x/13.1x**. Named, escaped, multi-reference, ambiguous two-digit, subclass, and invalid templates continue through the fully compatible parser. ⚡🛡️ +* 08/10/2026 **Call-local `Match.expand` fast paths**: exact text and bytes templates containing one unambiguous capture now render directly from the immutable C Match snapshot, without importing the compatibility parser or retaining a parsed-template cache entry. Pinned A/B measurements against merged `main` improve matched `[\\1]` expansion by **20.8x** on Python 3.10 and **16.7x** on free-threaded Python 3.14t/GIL=0; unmatched captures reach **25.5x/20.7x**, and bytes reach **37.4x/31.7x**. Explicit numeric `[\\g<1>]` improves by **14.3x/13.8x**, checked multi-digit references such as `\\g<12>` reach **63.3x/13.1x**, and named `[\\g]` improves by **28.7x/10.6x**. Duplicate-name alternatives select the participating capture. Escaped, multi-reference, ambiguous two-digit, subclass, invalid, and non-ASCII-name templates continue through the fully compatible parser. ⚡🛡️ * 08/10/2026 **Literal split/substitution/findall fast paths**: exact plain-literal `Pattern.split` calls now use the immutable built-in splitter after construction-time validation, measuring **2.1x** faster than the prior C dispatch on Python 3.10 and **1.7x** faster on free-threaded Python 3.14t/GIL=0; delimiter-heavy multi-character literals reach roughly **4.8x**. Literal `Pattern.subn` and module-level `sub`/`subn` now use native replace/count primitives, reaching about **15x** on short repeated tokens and **3x** on delimiter-heavy text. Literal `findall` uses non-overlapping native count/list construction, reaching about **9x** on short repeated tokens and **8x** on delimiter-heavy text. Regex metacharacters, explicit flags, subclasses, and buffer subjects remain on the compatibility-safe PCRE2 path. ⚡ * 08/09/2026 **API hot-path update**: large `parallel_map(findall)` workloads now reach **11.5x** speedup on Python 3.10 and **11.25x** on free-threaded Python 3.14t/GIL=0 with 12 performance-tier workers. Ordered `parallel_map(search)` reaches **8.57x** and **7.85x**, respectively; one-item and up to eight tiny explicit `parallel_map` subjects now avoid executor setup (the one-item case measures **13.3x** faster on Python 3.10 and **27.7x** on 3.14t), default bound `Pattern.split` is another **1.6x/1.5x** faster on 3.10/3.14t, and default bound literal `Pattern.subn` is about **1.5x** faster on Python 3.10. Canonical module helpers retain their optimized dispatch while their wrapper/template caches are thread-scoped, size-bounded, and invalidated across live workers. Repeated backreference `Match.expand()` avoids reparsing within the active cache context, while captured values returned by `Match.groups()` remain call-local so a long-lived Match does not retain an additional copy of large captures. 🧵⚡ * 08/08/2026 **0.6.0**: `findall`, `finditer`, `sub`/`subn`, `split`, and `match`/`search`/`fullmatch` are now up to **46x faster** than `stdlib.re` and **48x faster** than `regex` on `finditer`/`findall` workloads, **13x** on `split`, and **2–9x** on `sub`/`subn` backref workloads, with full `re` semantics. Free-threaded `findall` reaches **13.8x** vs `re` on 8 threads. 🚀⚡ @@ -85,6 +85,7 @@ hard CPU affinity. | Repeated call-local `Match.groups()` | **~0.05 μs** | **~0.05 μs** | | Call-local `Match.expand(r"[\\1]")` | **0.07 μs** | **0.07 μs** | | Call-local `Match.expand(r"[\\g<1>]")` | **0.11 μs** | **0.08 μs** | +| Call-local `Match.expand(r"[\\g]")` | **0.13 μs** | **0.11 μs** | | Repeated default `compile("(x)")` | **0.49 μs** | **0.38 μs** | | Repeated integer-flagged `compile("x", CASELESS)` | **1.16 μs** | **0.81 μs** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index 9dca1a6..d8811d0 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -37,8 +37,11 @@ def main() -> int: short_subject = "x" * 10 pattern = pcre.compile("(x)") captured = pattern.match(short_subject) + named_captured = pcre.compile("(?Px)").match(short_subject) if captured is None: raise AssertionError("benchmark pattern failed to produce a match") + if named_captured is None: + raise AssertionError("benchmark named pattern failed to produce a match") operations: list[tuple[str, Callable[[], object]]] = [ ("bound.match", lambda: pattern.match(subject)), ("bound.search", lambda: pattern.search(subject)), @@ -51,6 +54,7 @@ def main() -> int: ("match.groups", captured.groups), ("match.expand.numeric", lambda: captured.expand(r"[\1]")), ("match.expand.explicit", lambda: captured.expand(r"[\g<1>]")), + ("match.expand.named", lambda: named_captured.expand(r"[\g]")), ("module.match", lambda: pcre.match("(x)", subject)), ("module.search", lambda: pcre.search("(x)", subject)), ("module.fullmatch", lambda: pcre.fullmatch("(x)", subject)), diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index f4b91d4..c8edc51 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -1335,6 +1335,149 @@ match_expand_explicit_numeric(MatchObject *self, ); } +static int +match_expand_resolve_ascii_name(MatchObject *self, + const char *name, + Py_ssize_t name_length, + Py_ssize_t *group_index) +{ + uint32_t name_count = 0; + uint32_t entry_size = 0; + PCRE2_SPTR name_table = NULL; + if (name_length <= 0 || + pcre2_pattern_info(self->pattern->code, + PCRE2_INFO_NAMECOUNT, + &name_count) != 0 || + name_count == 0 || + pcre2_pattern_info(self->pattern->code, + PCRE2_INFO_NAMEENTRYSIZE, + &entry_size) != 0 || + pcre2_pattern_info(self->pattern->code, + PCRE2_INFO_NAMETABLE, + &name_table) != 0 || + name_table == NULL || entry_size < 3) { + return 0; + } + + Py_ssize_t first_match = -1; + size_t name_max = (size_t)entry_size - 2; + for (uint32_t i = 0; i < name_count; ++i) { + const unsigned char *entry = (const unsigned char *)( + name_table + (size_t)i * entry_size + ); + const char *entry_name = (const char *)(entry + 2); + size_t entry_length = strnlen(entry_name, name_max); + if (entry_length != (size_t)name_length || + memcmp(entry_name, name, entry_length) != 0) { + continue; + } + + Py_ssize_t candidate = (Py_ssize_t)((entry[0] << 8) | entry[1]); + if (first_match < 0) { + first_match = candidate; + } + if (candidate >= 0 && (size_t)candidate < self->ovec_count) { + Py_ssize_t start = self->ovector[(size_t)candidate * 2]; + Py_ssize_t end = self->ovector[(size_t)candidate * 2 + 1]; + if (start >= 0 && end >= 0) { + *group_index = candidate; + return 1; + } + } + } + if (first_match >= 0) { + *group_index = first_match; + return 1; + } + return 0; +} + +static PyObject * +match_expand_explicit_named(MatchObject *self, + PyObject *template_obj, + Py_ssize_t slash_index, + Py_ssize_t template_length, + int *handled) +{ + *handled = 0; + if (slash_index < 0 || template_length - slash_index < 5) { + return NULL; + } + + char name[129]; + Py_ssize_t cursor = slash_index + 1; + Py_ssize_t name_length = 0; + if (!self->subject_is_bytes && PyUnicode_CheckExact(template_obj)) { + if (PyUnicode_ReadChar(template_obj, cursor) != 'g' || + PyUnicode_ReadChar(template_obj, cursor + 1) != '<') { + return NULL; + } + cursor += 2; + while (cursor < template_length) { + Py_UCS4 character = PyUnicode_ReadChar(template_obj, cursor); + if (character == '>') { + break; + } + if (character > 0x7f || name_length >= 128) { + return NULL; + } + name[name_length++] = (char)character; + cursor += 1; + } + if (name_length == 0 || cursor >= template_length) { + return NULL; + } + if (cursor + 1 < template_length) { + Py_ssize_t next_slash = PyUnicode_FindChar( + template_obj, '\\', cursor + 1, template_length, 1 + ); + if (next_slash >= 0) { + return NULL; + } + if (PyErr_Occurred()) { + return NULL; + } + } + } else if (self->subject_is_bytes && PyBytes_CheckExact(template_obj)) { + const unsigned char *template_data = (const unsigned char *)PyBytes_AS_STRING(template_obj); + if (template_data[cursor] != 'g' || template_data[cursor + 1] != '<') { + return NULL; + } + cursor += 2; + while (cursor < template_length) { + unsigned char character = template_data[cursor]; + if (character == '>') { + break; + } + if (character > 0x7f || name_length >= 128) { + return NULL; + } + name[name_length++] = (char)character; + cursor += 1; + } + if (name_length == 0 || cursor >= template_length) { + return NULL; + } + if (cursor + 1 < template_length && + memchr(template_data + cursor + 1, + '\\', + (size_t)(template_length - cursor - 1)) != NULL) { + return NULL; + } + } else { + return NULL; + } + + Py_ssize_t group_index = -1; + if (!match_expand_resolve_ascii_name( + self, name, name_length, &group_index)) { + return NULL; + } + return match_expand_render_reference( + self, template_obj, slash_index, cursor + 1, group_index, handled + ); +} + static PyObject * Match_expand(MatchObject *self, PyObject *template_obj) { @@ -1365,6 +1508,12 @@ Match_expand(MatchObject *self, PyObject *template_obj) if (handled || result != NULL || PyErr_Occurred()) { return result; } + result = match_expand_explicit_named( + self, template_obj, slash_index, template_length, &handled + ); + if (handled || result != NULL || PyErr_Occurred()) { + return result; + } } PyErr_Clear(); } @@ -1402,6 +1551,16 @@ Match_expand(MatchObject *self, PyObject *template_obj) if (handled || result != NULL || PyErr_Occurred()) { return result; } + result = match_expand_explicit_named( + self, + template_obj, + (Py_ssize_t)(slash - template_data), + template_length, + &handled + ); + if (handled || result != NULL || PyErr_Occurred()) { + return result; + } } } diff --git a/tests/test_expand_fastpath.py b/tests/test_expand_fastpath.py index 3e8e2a4..f09ca75 100644 --- a/tests/test_expand_fastpath.py +++ b/tests/test_expand_fastpath.py @@ -36,6 +36,10 @@ "[l]", ), (r"(a)", "a", r"[\g<001>]", "[a]"), + (r"(?Pa)", "a", r"[\g]", "[a]"), + (r"(?Pé)", "é", "前\\g後", "前é後"), + (r"(?Pa)?(?Pb)?", "a", r"[\g]", "[]"), + (b"(?Pa)", b"a", rb"[\g]", b"[a]"), ], ) def test_single_numeric_expand_is_exact_and_call_local( @@ -85,12 +89,13 @@ def test_ambiguous_or_extended_expand_stays_on_compatibility_parser( def test_single_numeric_expand_is_safe_on_one_match_across_threads() -> None: - match = pcre.compile(r"(é)").fullmatch("é") + match = pcre.compile(r"(?Pé)").fullmatch("é") assert match is not None def expand_many() -> None: for _ in range(10_000): assert match.expand("前\\1後") == "前é後" + assert match.expand("前\\g後") == "前é後" with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: list(executor.map(lambda _: expand_many(), range(8))) @@ -119,3 +124,33 @@ def test_numeric_expand_differential_matrix(pattern, subject) -> None: if isinstance(pattern, bytes): template = template.encode() assert actual_match.expand(template) == expected_match.expand(template) + + +@pytest.mark.parametrize("subject", ["a", "b"]) +def test_duplicate_named_expand_selects_the_participating_capture(subject: str) -> None: + match = pcre.fullmatch(r"(?J)(?Pa)|(?Pb)", subject) + assert match is not None + + assert match.expand(r"[\g]") == f"[{subject}]" + + +@pytest.mark.parametrize( + ("pattern", "subject"), + [ + (r"(?Pa)?(?Pb)?", "a"), + (r"(?Pé)?(?Pβ)?", "éβ"), + (b"(?Pa)?(?Pb)?", b"a"), + ], +) +def test_named_expand_differential_matrix(pattern, subject) -> None: + expected_match = re.fullmatch(pattern, subject) + actual_match = pcre.fullmatch(pattern, subject) + assert expected_match is not None + assert actual_match is not None + + for name in ("first", "second"): + for prefix, suffix in (("", ""), ("[", "]"), ("前", "後")): + template = f"{prefix}\\g<{name}>{suffix}" + if isinstance(pattern, bytes): + template = template.encode() + assert actual_match.expand(template) == expected_match.expand(template) diff --git a/tests/test_python_coverage_audit.py b/tests/test_python_coverage_audit.py index cbaf817..32fb412 100644 --- a/tests/test_python_coverage_audit.py +++ b/tests/test_python_coverage_audit.py @@ -101,12 +101,12 @@ def counted_parse(template: Any, state: Any) -> Any: return original(template, state) monkeypatch.setattr(compat._parser, "parse_template", counted_parse) - assert match.expand(r"[\g]") == "[x]" - assert match.expand(r"[\g]") == "[x]" + assert match.expand(r"[\g]-\g") == "[x]-x" + assert match.expand(r"[\g]-\g") == "[x]-x" assert calls == 1 pcre.clear_cache() - assert match.expand(r"[\g]") == "[x]" + assert match.expand(r"[\g]-\g") == "[x]-x" assert calls == 2 From 8812d056aff884cbafe69adbd48171fbb4775164 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 17:54:33 +0800 Subject: [PATCH 04/21] perf: accelerate two-reference match expansion --- README.md | 3 +- benchmarks/api_hotpaths.py | 7 + pcre_ext/pcre2.c | 287 ++++++++++++++++++++++++++++ tests/test_expand_fastpath.py | 74 +++++++ tests/test_python_coverage_audit.py | 6 +- 5 files changed, 373 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a6b25e6..38b0f5f 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 -* 08/10/2026 **Call-local `Match.expand` fast paths**: exact text and bytes templates containing one unambiguous capture now render directly from the immutable C Match snapshot, without importing the compatibility parser or retaining a parsed-template cache entry. Pinned A/B measurements against merged `main` improve matched `[\\1]` expansion by **20.8x** on Python 3.10 and **16.7x** on free-threaded Python 3.14t/GIL=0; unmatched captures reach **25.5x/20.7x**, and bytes reach **37.4x/31.7x**. Explicit numeric `[\\g<1>]` improves by **14.3x/13.8x**, checked multi-digit references such as `\\g<12>` reach **63.3x/13.1x**, and named `[\\g]` improves by **28.7x/10.6x**. Duplicate-name alternatives select the participating capture. Escaped, multi-reference, ambiguous two-digit, subclass, invalid, and non-ASCII-name templates continue through the fully compatible parser. ⚡🛡️ +* 08/10/2026 **Call-local `Match.expand` fast paths**: exact text and bytes templates containing one or two unambiguous captures now render directly from the immutable C Match snapshot, without importing the compatibility parser or retaining a parsed-template cache entry. Pinned A/B measurements against merged `main` improve matched `[\\1]` expansion by **20.8x** on Python 3.10 and **16.7x** on free-threaded Python 3.14t/GIL=0; unmatched captures reach **25.5x/20.7x**, and bytes reach **37.4x/31.7x**. Explicit numeric `[\\g<1>]` improves by **14.3x/13.8x**, checked multi-digit references such as `\\g<12>` reach **63.3x/13.1x**, named `[\\g]` improves by **28.7x/10.6x**, and a two-name template reaches **38.7x/8.1x**. Duplicate-name alternatives select the participating capture. Escaped, three-or-more-reference, ambiguous two-digit, subclass, invalid, and non-ASCII-name templates continue through the fully compatible parser. ⚡🛡️ * 08/10/2026 **Literal split/substitution/findall fast paths**: exact plain-literal `Pattern.split` calls now use the immutable built-in splitter after construction-time validation, measuring **2.1x** faster than the prior C dispatch on Python 3.10 and **1.7x** faster on free-threaded Python 3.14t/GIL=0; delimiter-heavy multi-character literals reach roughly **4.8x**. Literal `Pattern.subn` and module-level `sub`/`subn` now use native replace/count primitives, reaching about **15x** on short repeated tokens and **3x** on delimiter-heavy text. Literal `findall` uses non-overlapping native count/list construction, reaching about **9x** on short repeated tokens and **8x** on delimiter-heavy text. Regex metacharacters, explicit flags, subclasses, and buffer subjects remain on the compatibility-safe PCRE2 path. ⚡ * 08/09/2026 **API hot-path update**: large `parallel_map(findall)` workloads now reach **11.5x** speedup on Python 3.10 and **11.25x** on free-threaded Python 3.14t/GIL=0 with 12 performance-tier workers. Ordered `parallel_map(search)` reaches **8.57x** and **7.85x**, respectively; one-item and up to eight tiny explicit `parallel_map` subjects now avoid executor setup (the one-item case measures **13.3x** faster on Python 3.10 and **27.7x** on 3.14t), default bound `Pattern.split` is another **1.6x/1.5x** faster on 3.10/3.14t, and default bound literal `Pattern.subn` is about **1.5x** faster on Python 3.10. Canonical module helpers retain their optimized dispatch while their wrapper/template caches are thread-scoped, size-bounded, and invalidated across live workers. Repeated backreference `Match.expand()` avoids reparsing within the active cache context, while captured values returned by `Match.groups()` remain call-local so a long-lived Match does not retain an additional copy of large captures. 🧵⚡ * 08/08/2026 **0.6.0**: `findall`, `finditer`, `sub`/`subn`, `split`, and `match`/`search`/`fullmatch` are now up to **46x faster** than `stdlib.re` and **48x faster** than `regex` on `finditer`/`findall` workloads, **13x** on `split`, and **2–9x** on `sub`/`subn` backref workloads, with full `re` semantics. Free-threaded `findall` reaches **13.8x** vs `re` on 8 threads. 🚀⚡ @@ -86,6 +86,7 @@ hard CPU affinity. | Call-local `Match.expand(r"[\\1]")` | **0.07 μs** | **0.07 μs** | | Call-local `Match.expand(r"[\\g<1>]")` | **0.11 μs** | **0.08 μs** | | Call-local `Match.expand(r"[\\g]")` | **0.13 μs** | **0.11 μs** | +| Call-local two-name `Match.expand` | **0.21 μs** | **0.18 μs** | | Repeated default `compile("(x)")` | **0.49 μs** | **0.38 μs** | | Repeated integer-flagged `compile("x", CASELESS)` | **1.16 μs** | **0.81 μs** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index d8811d0..6137834 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -38,10 +38,13 @@ def main() -> int: pattern = pcre.compile("(x)") captured = pattern.match(short_subject) named_captured = pcre.compile("(?Px)").match(short_subject) + multi_captured = pcre.compile("(?Px)(?Px)").match(short_subject) if captured is None: raise AssertionError("benchmark pattern failed to produce a match") if named_captured is None: raise AssertionError("benchmark named pattern failed to produce a match") + if multi_captured is None: + raise AssertionError("benchmark multi pattern failed to produce a match") operations: list[tuple[str, Callable[[], object]]] = [ ("bound.match", lambda: pattern.match(subject)), ("bound.search", lambda: pattern.search(subject)), @@ -55,6 +58,10 @@ def main() -> int: ("match.expand.numeric", lambda: captured.expand(r"[\1]")), ("match.expand.explicit", lambda: captured.expand(r"[\g<1>]")), ("match.expand.named", lambda: named_captured.expand(r"[\g]")), + ( + "match.expand.multi", + lambda: multi_captured.expand(r"[\g]-\g"), + ), ("module.match", lambda: pcre.match("(x)", subject)), ("module.search", lambda: pcre.search("(x)", subject)), ("module.fullmatch", lambda: pcre.fullmatch("(x)", subject)), diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index c8edc51..f48ddd1 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -1478,6 +1478,277 @@ match_expand_explicit_named(MatchObject *self, ); } +typedef struct { + Py_ssize_t slash_index; + Py_ssize_t reference_end; + Py_ssize_t group_index; +} MatchExpandReference; + +static int +match_expand_parse_reference(MatchObject *self, + PyObject *template_obj, + Py_ssize_t slash_index, + Py_ssize_t template_length, + MatchExpandReference *reference) +{ + if (slash_index < 0 || slash_index + 1 >= template_length) { + return 0; + } + + Py_UCS4 following = self->subject_is_bytes + ? (unsigned char)PyBytes_AS_STRING(template_obj)[slash_index + 1] + : PyUnicode_ReadChar(template_obj, slash_index + 1); + if (following >= '1' && following <= '9') { + if (slash_index + 2 < template_length) { + Py_UCS4 next = self->subject_is_bytes + ? (unsigned char)PyBytes_AS_STRING(template_obj)[slash_index + 2] + : PyUnicode_ReadChar(template_obj, slash_index + 2); + if (next >= '0' && next <= '9') { + return 0; + } + } + Py_ssize_t group_index = (Py_ssize_t)(following - '0'); + if ((size_t)group_index >= self->ovec_count) { + return 0; + } + reference->slash_index = slash_index; + reference->reference_end = slash_index + 2; + reference->group_index = group_index; + return 1; + } + + if (following != 'g' || slash_index + 4 >= template_length) { + return 0; + } + Py_UCS4 opener = self->subject_is_bytes + ? (unsigned char)PyBytes_AS_STRING(template_obj)[slash_index + 2] + : PyUnicode_ReadChar(template_obj, slash_index + 2); + if (opener != '<') { + return 0; + } + + char name[129]; + Py_ssize_t name_length = 0; + Py_ssize_t numeric_index = 0; + int is_numeric = 1; + Py_ssize_t cursor = slash_index + 3; + while (cursor < template_length) { + Py_UCS4 character = self->subject_is_bytes + ? (unsigned char)PyBytes_AS_STRING(template_obj)[cursor] + : PyUnicode_ReadChar(template_obj, cursor); + if (character == '>') { + break; + } + if (character > 0x7f || name_length >= 128) { + return 0; + } + name[name_length++] = (char)character; + if (character < '0' || character > '9') { + is_numeric = 0; + } else if (is_numeric) { + if (numeric_index > + (PY_SSIZE_T_MAX - (Py_ssize_t)(character - '0')) / 10) { + return 0; + } + numeric_index = numeric_index * 10 + + (Py_ssize_t)(character - '0'); + } + cursor += 1; + } + if (name_length == 0 || cursor >= template_length) { + return 0; + } + + Py_ssize_t group_index = -1; + if (is_numeric) { + if ((size_t)numeric_index >= self->ovec_count) { + return 0; + } + group_index = numeric_index; + } else if (!match_expand_resolve_ascii_name( + self, name, name_length, &group_index)) { + return 0; + } + reference->slash_index = slash_index; + reference->reference_end = cursor + 1; + reference->group_index = group_index; + return 1; +} + +static int +match_expand_checked_add(Py_ssize_t *total, Py_ssize_t value) +{ + if (value < 0 || *total > PY_SSIZE_T_MAX - value) { + PyErr_NoMemory(); + return -1; + } + *total += value; + return 0; +} + +static PyObject * +match_expand_two_references(MatchObject *self, + PyObject *template_obj, + Py_ssize_t first_slash, + Py_ssize_t template_length, + int *handled) +{ + *handled = 0; + Py_ssize_t second_slash; + if (self->subject_is_bytes) { + const char *template_data = PyBytes_AS_STRING(template_obj); + const char *found = memchr(template_data + first_slash + 1, + '\\', + (size_t)(template_length - first_slash - 1)); + if (found == NULL) { + return NULL; + } + second_slash = (Py_ssize_t)(found - template_data); + if (memchr(found + 1, + '\\', + (size_t)(template_length - second_slash - 1)) != NULL) { + return NULL; + } + } else { + second_slash = PyUnicode_FindChar( + template_obj, '\\', first_slash + 1, template_length, 1 + ); + if (second_slash < 0) { + return NULL; + } + Py_ssize_t third_slash = PyUnicode_FindChar( + template_obj, '\\', second_slash + 1, template_length, 1 + ); + if (third_slash >= 0 || PyErr_Occurred()) { + return NULL; + } + } + + MatchExpandReference first; + MatchExpandReference second; + if (!match_expand_parse_reference( + self, template_obj, first_slash, template_length, &first) || + first.reference_end > second_slash || + !match_expand_parse_reference( + self, template_obj, second_slash, template_length, &second)) { + return NULL; + } + + PyObject *first_group = match_get_group_value(self, first.group_index); + if (first_group == NULL) { + return NULL; + } + PyObject *second_group = match_get_group_value(self, second.group_index); + if (second_group == NULL) { + Py_DECREF(first_group); + return NULL; + } + *handled = 1; + if (first_group == Py_None) { + Py_DECREF(first_group); + first_group = NULL; + } + if (second_group == Py_None) { + Py_DECREF(second_group); + second_group = NULL; + } + + Py_ssize_t first_length = first_group == NULL + ? 0 : PyObject_Length(first_group); + Py_ssize_t second_length = second_group == NULL + ? 0 : PyObject_Length(second_group); + if (first_length < 0 || second_length < 0) { + Py_XDECREF(first_group); + Py_XDECREF(second_group); + return NULL; + } + Py_ssize_t middle_start = first.reference_end; + Py_ssize_t middle_length = second.slash_index - middle_start; + Py_ssize_t suffix_length = template_length - second.reference_end; + Py_ssize_t result_length = first.slash_index; + if (match_expand_checked_add(&result_length, first_length) < 0 || + match_expand_checked_add(&result_length, middle_length) < 0 || + match_expand_checked_add(&result_length, second_length) < 0 || + match_expand_checked_add(&result_length, suffix_length) < 0) { + Py_XDECREF(first_group); + Py_XDECREF(second_group); + return NULL; + } + + if (self->subject_is_bytes) { + PyObject *result = PyBytes_FromStringAndSize(NULL, result_length); + if (result == NULL) { + Py_XDECREF(first_group); + Py_XDECREF(second_group); + return NULL; + } + char *output = PyBytes_AS_STRING(result); + const char *input = PyBytes_AS_STRING(template_obj); + Py_ssize_t output_offset = 0; +#define COPY_EXPAND_BYTES(source, length) do { \ + if ((length) > 0) { \ + memcpy(output + output_offset, (source), (size_t)(length)); \ + output_offset += (length); \ + } \ + } while (0) + COPY_EXPAND_BYTES(input, first.slash_index); + if (first_group != NULL) { + COPY_EXPAND_BYTES(PyBytes_AS_STRING(first_group), first_length); + } + COPY_EXPAND_BYTES(input + middle_start, middle_length); + if (second_group != NULL) { + COPY_EXPAND_BYTES(PyBytes_AS_STRING(second_group), second_length); + } + COPY_EXPAND_BYTES(input + second.reference_end, suffix_length); +#undef COPY_EXPAND_BYTES + Py_XDECREF(first_group); + Py_XDECREF(second_group); + return result; + } + + Py_UCS4 max_character = PyUnicode_MAX_CHAR_VALUE(template_obj); + if (first_group != NULL && + PyUnicode_MAX_CHAR_VALUE(first_group) > max_character) { + max_character = PyUnicode_MAX_CHAR_VALUE(first_group); + } + if (second_group != NULL && + PyUnicode_MAX_CHAR_VALUE(second_group) > max_character) { + max_character = PyUnicode_MAX_CHAR_VALUE(second_group); + } + PyObject *result = PyUnicode_New(result_length, max_character); + if (result == NULL) { + Py_XDECREF(first_group); + Py_XDECREF(second_group); + return NULL; + } + Py_ssize_t output_offset = 0; +#define COPY_EXPAND_UNICODE(source, start, length) do { \ + if ((length) > 0) { \ + if (PyUnicode_CopyCharacters(result, output_offset, \ + (source), (start), (length)) < 0) { \ + Py_DECREF(result); \ + Py_XDECREF(first_group); \ + Py_XDECREF(second_group); \ + return NULL; \ + } \ + output_offset += (length); \ + } \ + } while (0) + COPY_EXPAND_UNICODE(template_obj, 0, first.slash_index); + if (first_group != NULL) { + COPY_EXPAND_UNICODE(first_group, 0, first_length); + } + COPY_EXPAND_UNICODE(template_obj, middle_start, middle_length); + if (second_group != NULL) { + COPY_EXPAND_UNICODE(second_group, 0, second_length); + } + COPY_EXPAND_UNICODE(template_obj, second.reference_end, suffix_length); +#undef COPY_EXPAND_UNICODE + Py_XDECREF(first_group); + Py_XDECREF(second_group); + return result; +} + static PyObject * Match_expand(MatchObject *self, PyObject *template_obj) { @@ -1514,6 +1785,12 @@ Match_expand(MatchObject *self, PyObject *template_obj) if (handled || result != NULL || PyErr_Occurred()) { return result; } + result = match_expand_two_references( + self, template_obj, slash_index, template_length, &handled + ); + if (handled || result != NULL || PyErr_Occurred()) { + return result; + } } PyErr_Clear(); } @@ -1561,6 +1838,16 @@ Match_expand(MatchObject *self, PyObject *template_obj) if (handled || result != NULL || PyErr_Occurred()) { return result; } + result = match_expand_two_references( + self, + template_obj, + (Py_ssize_t)(slash - template_data), + template_length, + &handled + ); + if (handled || result != NULL || PyErr_Occurred()) { + return result; + } } } diff --git a/tests/test_expand_fastpath.py b/tests/test_expand_fastpath.py index f09ca75..ce70c28 100644 --- a/tests/test_expand_fastpath.py +++ b/tests/test_expand_fastpath.py @@ -96,6 +96,7 @@ def expand_many() -> None: for _ in range(10_000): assert match.expand("前\\1後") == "前é後" assert match.expand("前\\g後") == "前é後" + assert match.expand("前\\1中\\g後") == "前é中é後" with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: list(executor.map(lambda _: expand_many(), range(8))) @@ -154,3 +155,76 @@ def test_named_expand_differential_matrix(pattern, subject) -> None: if isinstance(pattern, bytes): template = template.encode() assert actual_match.expand(template) == expected_match.expand(template) + + +@pytest.mark.parametrize( + ("pattern", "subject", "template", "expected"), + [ + (r"(a)(b)", "ab", r"[\1]-\2", "[a]-b"), + (r"(?Pa)(?Pb)", "ab", r"[\g]-\g", "[a]-b"), + (r"(?Pa)", "a", r"\g<0>:\g", "a:a"), + (r"(é)(β)", "éβ", "前\\1中\\g<2>後", "前é中β後"), + (r"(a)?(b)?", "a", r"[\1]-[\2]", "[a]-[]"), + (b"(a)(b)", b"ab", rb"[\1]-\g<2>", b"[a]-b"), + ( + b"(?Pa)(?Pb)", + b"ab", + rb"[\g]-\g", + b"[a]-b", + ), + ], +) +def test_two_reference_expand_is_exact_and_call_local( + pattern, + subject, + template, + expected, + monkeypatch: pytest.MonkeyPatch, +) -> None: + match = pcre.fullmatch(pattern, subject) + assert match is not None + re_compat._cached_expand_template.cache_clear() + monkeypatch.setattr( + re_compat, + "expand_match_template", + lambda *args: pytest.fail("two-reference expansion reached Python parser"), + ) + + assert match.expand(template) == expected + assert re_compat._expand_template_cache_size() == 0 + + +def test_three_references_stay_on_compatibility_parser( + monkeypatch: pytest.MonkeyPatch, +) -> None: + match = pcre.fullmatch(r"(a)(b)(c)", "abc") + assert match is not None + sentinel = object() + monkeypatch.setattr(re_compat, "expand_match_template", lambda *args: sentinel) + + assert match.expand(r"\1\2\3") is sentinel + + +@pytest.mark.parametrize( + ("pattern", "subject"), + [ + (r"(?Pa)?(?Pb)?", "a"), + (r"(?Pé)?(?Pβ)?", "éβ"), + (b"(?Pa)?(?Pb)?", b"a"), + ], +) +def test_two_reference_expand_differential_matrix(pattern, subject) -> None: + expected_match = re.fullmatch(pattern, subject) + actual_match = pcre.fullmatch(pattern, subject) + assert expected_match is not None + assert actual_match is not None + + references = (r"\1", r"\2", r"\g<0>", r"\g<01>", r"\g", r"\g") + literals = (("", "", ""), ("[", "]: [", "]"), ("前", "中", "後")) + for first in references: + for second in references: + for prefix, middle, suffix in literals: + template = f"{prefix}{first}{middle}{second}{suffix}" + if isinstance(pattern, bytes): + template = template.encode() + assert actual_match.expand(template) == expected_match.expand(template) diff --git a/tests/test_python_coverage_audit.py b/tests/test_python_coverage_audit.py index 32fb412..dd46e20 100644 --- a/tests/test_python_coverage_audit.py +++ b/tests/test_python_coverage_audit.py @@ -101,12 +101,12 @@ def counted_parse(template: Any, state: Any) -> Any: return original(template, state) monkeypatch.setattr(compat._parser, "parse_template", counted_parse) - assert match.expand(r"[\g]-\g") == "[x]-x" - assert match.expand(r"[\g]-\g") == "[x]-x" + assert match.expand(r"[\g]-\g-\g") == "[x]-x-x" + assert match.expand(r"[\g]-\g-\g") == "[x]-x-x" assert calls == 1 pcre.clear_cache() - assert match.expand(r"[\g]-\g") == "[x]-x" + assert match.expand(r"[\g]-\g-\g") == "[x]-x-x" assert calls == 2 From 5737ff10fd246f521e029de0db2b04fefb07a9d6 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 18:00:07 +0800 Subject: [PATCH 05/21] perf: accelerate bounded multi-reference expansion --- README.md | 6 +- benchmarks/api_hotpaths.py | 4 + pcre_ext/pcre2.c | 298 +++++++++++++++------------- tests/test_expand_fastpath.py | 26 ++- tests/test_python_coverage_audit.py | 7 +- 5 files changed, 197 insertions(+), 144 deletions(-) diff --git a/README.md b/README.md index 38b0f5f..d13b82e 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 -* 08/10/2026 **Call-local `Match.expand` fast paths**: exact text and bytes templates containing one or two unambiguous captures now render directly from the immutable C Match snapshot, without importing the compatibility parser or retaining a parsed-template cache entry. Pinned A/B measurements against merged `main` improve matched `[\\1]` expansion by **20.8x** on Python 3.10 and **16.7x** on free-threaded Python 3.14t/GIL=0; unmatched captures reach **25.5x/20.7x**, and bytes reach **37.4x/31.7x**. Explicit numeric `[\\g<1>]` improves by **14.3x/13.8x**, checked multi-digit references such as `\\g<12>` reach **63.3x/13.1x**, named `[\\g]` improves by **28.7x/10.6x**, and a two-name template reaches **38.7x/8.1x**. Duplicate-name alternatives select the participating capture. Escaped, three-or-more-reference, ambiguous two-digit, subclass, invalid, and non-ASCII-name templates continue through the fully compatible parser. ⚡🛡️ +* 08/10/2026 **Call-local `Match.expand` fast paths**: exact text and bytes templates containing one to eight unambiguous captures now render directly from the immutable C Match snapshot, without importing the compatibility parser or retaining a parsed-template cache entry. Pinned A/B measurements against merged `main` improve matched `[\\1]` expansion by **20.8x** on Python 3.10 and **16.7x** on free-threaded Python 3.14t/GIL=0; unmatched captures reach **25.5x/20.7x**, and bytes reach **37.4x/31.7x**. Explicit numeric `[\\g<1>]` improves by **14.3x/13.8x**, checked multi-digit references such as `\\g<12>` reach **63.3x/13.1x**, named `[\\g]` improves by **28.7x/10.6x**, and a two-name template reaches **45.3x/9.6x**. Three references reach **9.2x/8.2x**, while eight reach **6.3x/6.3x**. Duplicate-name alternatives select the participating capture. Escaped, nine-or-more-reference, ambiguous two-digit, subclass, invalid, and non-ASCII-name templates continue through the fully compatible parser. ⚡🛡️ * 08/10/2026 **Literal split/substitution/findall fast paths**: exact plain-literal `Pattern.split` calls now use the immutable built-in splitter after construction-time validation, measuring **2.1x** faster than the prior C dispatch on Python 3.10 and **1.7x** faster on free-threaded Python 3.14t/GIL=0; delimiter-heavy multi-character literals reach roughly **4.8x**. Literal `Pattern.subn` and module-level `sub`/`subn` now use native replace/count primitives, reaching about **15x** on short repeated tokens and **3x** on delimiter-heavy text. Literal `findall` uses non-overlapping native count/list construction, reaching about **9x** on short repeated tokens and **8x** on delimiter-heavy text. Regex metacharacters, explicit flags, subclasses, and buffer subjects remain on the compatibility-safe PCRE2 path. ⚡ * 08/09/2026 **API hot-path update**: large `parallel_map(findall)` workloads now reach **11.5x** speedup on Python 3.10 and **11.25x** on free-threaded Python 3.14t/GIL=0 with 12 performance-tier workers. Ordered `parallel_map(search)` reaches **8.57x** and **7.85x**, respectively; one-item and up to eight tiny explicit `parallel_map` subjects now avoid executor setup (the one-item case measures **13.3x** faster on Python 3.10 and **27.7x** on 3.14t), default bound `Pattern.split` is another **1.6x/1.5x** faster on 3.10/3.14t, and default bound literal `Pattern.subn` is about **1.5x** faster on Python 3.10. Canonical module helpers retain their optimized dispatch while their wrapper/template caches are thread-scoped, size-bounded, and invalidated across live workers. Repeated backreference `Match.expand()` avoids reparsing within the active cache context, while captured values returned by `Match.groups()` remain call-local so a long-lived Match does not retain an additional copy of large captures. 🧵⚡ * 08/08/2026 **0.6.0**: `findall`, `finditer`, `sub`/`subn`, `split`, and `match`/`search`/`fullmatch` are now up to **46x faster** than `stdlib.re` and **48x faster** than `regex` on `finditer`/`findall` workloads, **13x** on `split`, and **2–9x** on `sub`/`subn` backref workloads, with full `re` semantics. Free-threaded `findall` reaches **13.8x** vs `re` on 8 threads. 🚀⚡ @@ -86,7 +86,9 @@ hard CPU affinity. | Call-local `Match.expand(r"[\\1]")` | **0.07 μs** | **0.07 μs** | | Call-local `Match.expand(r"[\\g<1>]")` | **0.11 μs** | **0.08 μs** | | Call-local `Match.expand(r"[\\g]")` | **0.13 μs** | **0.11 μs** | -| Call-local two-name `Match.expand` | **0.21 μs** | **0.18 μs** | +| Call-local two-name `Match.expand` | **0.18 μs** | **0.15 μs** | +| Call-local three-name `Match.expand` | **0.23 μs** | **0.20 μs** | +| Call-local eight-name `Match.expand` | **0.44 μs** | **0.39 μs** | | Repeated default `compile("(x)")` | **0.49 μs** | **0.38 μs** | | Repeated integer-flagged `compile("x", CASELESS)` | **1.16 μs** | **0.81 μs** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index 6137834..bd092ae 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -62,6 +62,10 @@ def main() -> int: "match.expand.multi", lambda: multi_captured.expand(r"[\g]-\g"), ), + ( + "match.expand.three", + lambda: multi_captured.expand(r"[\g]-\g-\g"), + ), ("module.match", lambda: pcre.match("(x)", subject)), ("module.search", lambda: pcre.search("(x)", subject)), ("module.fullmatch", lambda: pcre.fullmatch("(x)", subject)), diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index f48ddd1..ca2b23b 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -1586,167 +1586,197 @@ match_expand_checked_add(Py_ssize_t *total, Py_ssize_t value) return 0; } +#define MATCH_EXPAND_MAX_REFERENCES 8 + static PyObject * -match_expand_two_references(MatchObject *self, - PyObject *template_obj, - Py_ssize_t first_slash, - Py_ssize_t template_length, - int *handled) +match_expand_multiple_references(MatchObject *self, + PyObject *template_obj, + Py_ssize_t first_slash, + Py_ssize_t template_length, + int *handled) { *handled = 0; - Py_ssize_t second_slash; - if (self->subject_is_bytes) { - const char *template_data = PyBytes_AS_STRING(template_obj); - const char *found = memchr(template_data + first_slash + 1, - '\\', - (size_t)(template_length - first_slash - 1)); - if (found == NULL) { - return NULL; - } - second_slash = (Py_ssize_t)(found - template_data); - if (memchr(found + 1, - '\\', - (size_t)(template_length - second_slash - 1)) != NULL) { - return NULL; - } - } else { - second_slash = PyUnicode_FindChar( - template_obj, '\\', first_slash + 1, template_length, 1 - ); - if (second_slash < 0) { + MatchExpandReference references[MATCH_EXPAND_MAX_REFERENCES]; + PyObject *groups[MATCH_EXPAND_MAX_REFERENCES] = {NULL}; + Py_ssize_t group_offsets[MATCH_EXPAND_MAX_REFERENCES] = {0}; + Py_ssize_t group_lengths[MATCH_EXPAND_MAX_REFERENCES] = {0}; + Py_ssize_t reference_count = 0; + Py_ssize_t slash_index = first_slash; + + while (slash_index >= 0) { + if (reference_count >= MATCH_EXPAND_MAX_REFERENCES || + !match_expand_parse_reference( + self, + template_obj, + slash_index, + template_length, + &references[reference_count])) { return NULL; } - Py_ssize_t third_slash = PyUnicode_FindChar( - template_obj, '\\', second_slash + 1, template_length, 1 - ); - if (third_slash >= 0 || PyErr_Occurred()) { - return NULL; + Py_ssize_t search_start = + references[reference_count].reference_end; + reference_count += 1; + if (self->subject_is_bytes) { + const char *input = PyBytes_AS_STRING(template_obj); + const char *found = memchr( + input + search_start, + '\\', + (size_t)(template_length - search_start) + ); + slash_index = found == NULL + ? -1 : (Py_ssize_t)(found - input); + } else { + slash_index = PyUnicode_FindChar( + template_obj, '\\', search_start, template_length, 1 + ); + if (slash_index < 0 && PyErr_Occurred()) { + return NULL; + } } } - - MatchExpandReference first; - MatchExpandReference second; - if (!match_expand_parse_reference( - self, template_obj, first_slash, template_length, &first) || - first.reference_end > second_slash || - !match_expand_parse_reference( - self, template_obj, second_slash, template_length, &second)) { + if (reference_count < 2) { return NULL; } - PyObject *first_group = match_get_group_value(self, first.group_index); - if (first_group == NULL) { - return NULL; + Py_ssize_t result_length = 0; + Py_ssize_t literal_start = 0; + int direct_groups = self->subject_is_bytes || PyUnicode_IS_ASCII(self->subject); + Py_UCS4 max_character = self->subject_is_bytes + ? 0 : PyUnicode_MAX_CHAR_VALUE(template_obj); + for (Py_ssize_t i = 0; i < reference_count; ++i) { + size_t offset_index = (size_t)references[i].group_index * 2; + Py_ssize_t group_start = self->ovector[offset_index]; + Py_ssize_t group_end = self->ovector[offset_index + 1]; + if (group_start < 0 || group_end < 0) { + group_offsets[i] = 0; + group_lengths[i] = 0; + } else if (group_end < group_start || group_end > self->utf8_length) { + PyErr_SetString(PyExc_RuntimeError, "invalid capture offsets"); + goto error; + } else if (direct_groups) { + group_offsets[i] = group_start; + group_lengths[i] = group_end - group_start; + } else { + groups[i] = match_get_group_value(self, references[i].group_index); + if (groups[i] == NULL) { + goto error; + } + group_lengths[i] = PyObject_Length(groups[i]); + if (group_lengths[i] < 0) { + goto error; + } + if (!self->subject_is_bytes && + PyUnicode_MAX_CHAR_VALUE(groups[i]) > max_character) { + max_character = PyUnicode_MAX_CHAR_VALUE(groups[i]); + } + } + Py_ssize_t literal_length = references[i].slash_index - literal_start; + if (match_expand_checked_add(&result_length, literal_length) < 0 || + match_expand_checked_add(&result_length, group_lengths[i]) < 0) { + goto error; + } + literal_start = references[i].reference_end; } - PyObject *second_group = match_get_group_value(self, second.group_index); - if (second_group == NULL) { - Py_DECREF(first_group); - return NULL; + if (match_expand_checked_add( + &result_length, template_length - literal_start) < 0) { + goto error; } *handled = 1; - if (first_group == Py_None) { - Py_DECREF(first_group); - first_group = NULL; - } - if (second_group == Py_None) { - Py_DECREF(second_group); - second_group = NULL; - } - - Py_ssize_t first_length = first_group == NULL - ? 0 : PyObject_Length(first_group); - Py_ssize_t second_length = second_group == NULL - ? 0 : PyObject_Length(second_group); - if (first_length < 0 || second_length < 0) { - Py_XDECREF(first_group); - Py_XDECREF(second_group); - return NULL; - } - Py_ssize_t middle_start = first.reference_end; - Py_ssize_t middle_length = second.slash_index - middle_start; - Py_ssize_t suffix_length = template_length - second.reference_end; - Py_ssize_t result_length = first.slash_index; - if (match_expand_checked_add(&result_length, first_length) < 0 || - match_expand_checked_add(&result_length, middle_length) < 0 || - match_expand_checked_add(&result_length, second_length) < 0 || - match_expand_checked_add(&result_length, suffix_length) < 0) { - Py_XDECREF(first_group); - Py_XDECREF(second_group); - return NULL; - } if (self->subject_is_bytes) { PyObject *result = PyBytes_FromStringAndSize(NULL, result_length); if (result == NULL) { - Py_XDECREF(first_group); - Py_XDECREF(second_group); - return NULL; + goto error; } char *output = PyBytes_AS_STRING(result); const char *input = PyBytes_AS_STRING(template_obj); Py_ssize_t output_offset = 0; -#define COPY_EXPAND_BYTES(source, length) do { \ - if ((length) > 0) { \ - memcpy(output + output_offset, (source), (size_t)(length)); \ - output_offset += (length); \ - } \ - } while (0) - COPY_EXPAND_BYTES(input, first.slash_index); - if (first_group != NULL) { - COPY_EXPAND_BYTES(PyBytes_AS_STRING(first_group), first_length); - } - COPY_EXPAND_BYTES(input + middle_start, middle_length); - if (second_group != NULL) { - COPY_EXPAND_BYTES(PyBytes_AS_STRING(second_group), second_length); - } - COPY_EXPAND_BYTES(input + second.reference_end, suffix_length); -#undef COPY_EXPAND_BYTES - Py_XDECREF(first_group); - Py_XDECREF(second_group); + literal_start = 0; + for (Py_ssize_t i = 0; i < reference_count; ++i) { + Py_ssize_t literal_length = + references[i].slash_index - literal_start; + if (literal_length > 0) { + memcpy(output + output_offset, + input + literal_start, + (size_t)literal_length); + output_offset += literal_length; + } + if (group_lengths[i] > 0) { + const char *group_data = direct_groups + ? self->utf8_data + group_offsets[i] + : PyBytes_AS_STRING(groups[i]); + memcpy(output + output_offset, + group_data, + (size_t)group_lengths[i]); + output_offset += group_lengths[i]; + } + literal_start = references[i].reference_end; + } + Py_ssize_t suffix_length = template_length - literal_start; + if (suffix_length > 0) { + memcpy(output + output_offset, + input + literal_start, + (size_t)suffix_length); + } + for (Py_ssize_t i = 0; i < reference_count; ++i) { + Py_XDECREF(groups[i]); + } return result; } - Py_UCS4 max_character = PyUnicode_MAX_CHAR_VALUE(template_obj); - if (first_group != NULL && - PyUnicode_MAX_CHAR_VALUE(first_group) > max_character) { - max_character = PyUnicode_MAX_CHAR_VALUE(first_group); - } - if (second_group != NULL && - PyUnicode_MAX_CHAR_VALUE(second_group) > max_character) { - max_character = PyUnicode_MAX_CHAR_VALUE(second_group); - } PyObject *result = PyUnicode_New(result_length, max_character); if (result == NULL) { - Py_XDECREF(first_group); - Py_XDECREF(second_group); - return NULL; + goto error; } Py_ssize_t output_offset = 0; -#define COPY_EXPAND_UNICODE(source, start, length) do { \ - if ((length) > 0) { \ - if (PyUnicode_CopyCharacters(result, output_offset, \ - (source), (start), (length)) < 0) { \ - Py_DECREF(result); \ - Py_XDECREF(first_group); \ - Py_XDECREF(second_group); \ - return NULL; \ - } \ - output_offset += (length); \ - } \ - } while (0) - COPY_EXPAND_UNICODE(template_obj, 0, first.slash_index); - if (first_group != NULL) { - COPY_EXPAND_UNICODE(first_group, 0, first_length); - } - COPY_EXPAND_UNICODE(template_obj, middle_start, middle_length); - if (second_group != NULL) { - COPY_EXPAND_UNICODE(second_group, 0, second_length); - } - COPY_EXPAND_UNICODE(template_obj, second.reference_end, suffix_length); -#undef COPY_EXPAND_UNICODE - Py_XDECREF(first_group); - Py_XDECREF(second_group); + literal_start = 0; + for (Py_ssize_t i = 0; i < reference_count; ++i) { + Py_ssize_t literal_length = references[i].slash_index - literal_start; + if (literal_length > 0) { + if (PyUnicode_CopyCharacters(result, + output_offset, + template_obj, + literal_start, + literal_length) < 0) { + Py_DECREF(result); + goto error; + } + output_offset += literal_length; + } + if (group_lengths[i] > 0) { + PyObject *group_source = direct_groups ? self->subject : groups[i]; + if (PyUnicode_CopyCharacters(result, + output_offset, + group_source, + group_offsets[i], + group_lengths[i]) < 0) { + Py_DECREF(result); + goto error; + } + output_offset += group_lengths[i]; + } + literal_start = references[i].reference_end; + } + Py_ssize_t suffix_length = template_length - literal_start; + if (suffix_length > 0 && + PyUnicode_CopyCharacters(result, + output_offset, + template_obj, + literal_start, + suffix_length) < 0) { + Py_DECREF(result); + goto error; + } + for (Py_ssize_t i = 0; i < reference_count; ++i) { + Py_XDECREF(groups[i]); + } return result; + +error: + for (Py_ssize_t i = 0; i < reference_count; ++i) { + Py_XDECREF(groups[i]); + } + return NULL; } static PyObject * @@ -1785,7 +1815,7 @@ Match_expand(MatchObject *self, PyObject *template_obj) if (handled || result != NULL || PyErr_Occurred()) { return result; } - result = match_expand_two_references( + result = match_expand_multiple_references( self, template_obj, slash_index, template_length, &handled ); if (handled || result != NULL || PyErr_Occurred()) { @@ -1838,7 +1868,7 @@ Match_expand(MatchObject *self, PyObject *template_obj) if (handled || result != NULL || PyErr_Occurred()) { return result; } - result = match_expand_two_references( + result = match_expand_multiple_references( self, template_obj, (Py_ssize_t)(slash - template_data), diff --git a/tests/test_expand_fastpath.py b/tests/test_expand_fastpath.py index ce70c28..9124ef8 100644 --- a/tests/test_expand_fastpath.py +++ b/tests/test_expand_fastpath.py @@ -172,9 +172,16 @@ def test_named_expand_differential_matrix(pattern, subject) -> None: rb"[\g]-\g", b"[a]-b", ), + (r"(a)(b)(c)", "abc", r"\1-\2-\3", "a-b-c"), + ( + r"(a)(b)(c)(d)(e)(f)(g)(h)", + "abcdefgh", + r"\1\2\3\4\5\6\7\8", + "abcdefgh", + ), ], ) -def test_two_reference_expand_is_exact_and_call_local( +def test_bounded_multiple_reference_expand_is_exact_and_call_local( pattern, subject, template, @@ -187,22 +194,22 @@ def test_two_reference_expand_is_exact_and_call_local( monkeypatch.setattr( re_compat, "expand_match_template", - lambda *args: pytest.fail("two-reference expansion reached Python parser"), + lambda *args: pytest.fail("bounded-reference expansion reached Python parser"), ) assert match.expand(template) == expected assert re_compat._expand_template_cache_size() == 0 -def test_three_references_stay_on_compatibility_parser( +def test_nine_references_stay_on_compatibility_parser( monkeypatch: pytest.MonkeyPatch, ) -> None: - match = pcre.fullmatch(r"(a)(b)(c)", "abc") + match = pcre.fullmatch(r"(a)(b)(c)(d)(e)(f)(g)(h)(i)", "abcdefghi") assert match is not None sentinel = object() monkeypatch.setattr(re_compat, "expand_match_template", lambda *args: sentinel) - assert match.expand(r"\1\2\3") is sentinel + assert match.expand(r"\1\2\3\4\5\6\7\8\9") is sentinel @pytest.mark.parametrize( @@ -228,3 +235,12 @@ def test_two_reference_expand_differential_matrix(pattern, subject) -> None: if isinstance(pattern, bytes): template = template.encode() assert actual_match.expand(template) == expected_match.expand(template) + + +def test_multiple_reference_expand_uses_immutable_buffer_snapshot() -> None: + subject = bytearray(b"ab") + match = pcre.fullmatch(b"(a)(b)", subject) + assert match is not None + subject[:] = b"zz" + + assert match.expand(rb"[\1]-\2") == b"[a]-b" diff --git a/tests/test_python_coverage_audit.py b/tests/test_python_coverage_audit.py index dd46e20..540b4ff 100644 --- a/tests/test_python_coverage_audit.py +++ b/tests/test_python_coverage_audit.py @@ -101,12 +101,13 @@ def counted_parse(template: Any, state: Any) -> Any: return original(template, state) monkeypatch.setattr(compat._parser, "parse_template", counted_parse) - assert match.expand(r"[\g]-\g-\g") == "[x]-x-x" - assert match.expand(r"[\g]-\g-\g") == "[x]-x-x" + template = r"[\g]" + r"-\g" * 8 + assert match.expand(template) == "[x]" + "-x" * 8 + assert match.expand(template) == "[x]" + "-x" * 8 assert calls == 1 pcre.clear_cache() - assert match.expand(r"[\g]-\g-\g") == "[x]-x-x" + assert match.expand(template) == "[x]" + "-x" * 8 assert calls == 2 From 9a55f29314dd62c8965481adcb1c511c58107ebc Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 18:03:08 +0800 Subject: [PATCH 06/21] perf: accelerate literal backslash expansion --- README.md | 4 +- benchmarks/api_hotpaths.py | 1 + pcre_ext/pcre2.c | 132 +++++++++++++++++++++------------- tests/test_expand_fastpath.py | 29 +++++++- 4 files changed, 113 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index d13b82e..2f1b4ca 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 -* 08/10/2026 **Call-local `Match.expand` fast paths**: exact text and bytes templates containing one to eight unambiguous captures now render directly from the immutable C Match snapshot, without importing the compatibility parser or retaining a parsed-template cache entry. Pinned A/B measurements against merged `main` improve matched `[\\1]` expansion by **20.8x** on Python 3.10 and **16.7x** on free-threaded Python 3.14t/GIL=0; unmatched captures reach **25.5x/20.7x**, and bytes reach **37.4x/31.7x**. Explicit numeric `[\\g<1>]` improves by **14.3x/13.8x**, checked multi-digit references such as `\\g<12>` reach **63.3x/13.1x**, named `[\\g]` improves by **28.7x/10.6x**, and a two-name template reaches **45.3x/9.6x**. Three references reach **9.2x/8.2x**, while eight reach **6.3x/6.3x**. Duplicate-name alternatives select the participating capture. Escaped, nine-or-more-reference, ambiguous two-digit, subclass, invalid, and non-ASCII-name templates continue through the fully compatible parser. ⚡🛡️ +* 08/10/2026 **Call-local `Match.expand` fast paths**: exact text and bytes templates containing up to eight unambiguous capture/backslash tokens now render directly from the immutable C Match snapshot, without importing the compatibility parser or retaining a parsed-template cache entry. Pinned A/B measurements against merged `main` improve matched `[\\1]` expansion by **20.8x** on Python 3.10 and **16.7x** on free-threaded Python 3.14t/GIL=0; unmatched captures reach **25.5x/20.7x**, and bytes reach **37.4x/31.7x**. Explicit numeric `[\\g<1>]` improves by **14.3x/13.8x**, checked multi-digit references such as `\\g<12>` reach **63.3x/13.1x**, named `[\\g]` improves by **28.7x/10.6x**, and a two-name template reaches **45.3x/9.6x**. Three references reach **9.2x/8.2x**, while eight reach **6.3x/6.3x**. A literal backslash plus named capture reaches **13.9x/11.7x**, and a named capture with a backslash suffix reaches **42.9x/9.2x**. Duplicate-name alternatives select the participating capture. Nine-or-more tokens, non-backslash escapes, ambiguous two-digit, subclass, invalid, and non-ASCII-name templates continue through the fully compatible parser. ⚡🛡️ * 08/10/2026 **Literal split/substitution/findall fast paths**: exact plain-literal `Pattern.split` calls now use the immutable built-in splitter after construction-time validation, measuring **2.1x** faster than the prior C dispatch on Python 3.10 and **1.7x** faster on free-threaded Python 3.14t/GIL=0; delimiter-heavy multi-character literals reach roughly **4.8x**. Literal `Pattern.subn` and module-level `sub`/`subn` now use native replace/count primitives, reaching about **15x** on short repeated tokens and **3x** on delimiter-heavy text. Literal `findall` uses non-overlapping native count/list construction, reaching about **9x** on short repeated tokens and **8x** on delimiter-heavy text. Regex metacharacters, explicit flags, subclasses, and buffer subjects remain on the compatibility-safe PCRE2 path. ⚡ * 08/09/2026 **API hot-path update**: large `parallel_map(findall)` workloads now reach **11.5x** speedup on Python 3.10 and **11.25x** on free-threaded Python 3.14t/GIL=0 with 12 performance-tier workers. Ordered `parallel_map(search)` reaches **8.57x** and **7.85x**, respectively; one-item and up to eight tiny explicit `parallel_map` subjects now avoid executor setup (the one-item case measures **13.3x** faster on Python 3.10 and **27.7x** on 3.14t), default bound `Pattern.split` is another **1.6x/1.5x** faster on 3.10/3.14t, and default bound literal `Pattern.subn` is about **1.5x** faster on Python 3.10. Canonical module helpers retain their optimized dispatch while their wrapper/template caches are thread-scoped, size-bounded, and invalidated across live workers. Repeated backreference `Match.expand()` avoids reparsing within the active cache context, while captured values returned by `Match.groups()` remain call-local so a long-lived Match does not retain an additional copy of large captures. 🧵⚡ * 08/08/2026 **0.6.0**: `findall`, `finditer`, `sub`/`subn`, `split`, and `match`/`search`/`fullmatch` are now up to **46x faster** than `stdlib.re` and **48x faster** than `regex` on `finditer`/`findall` workloads, **13x** on `split`, and **2–9x** on `sub`/`subn` backref workloads, with full `re` semantics. Free-threaded `findall` reaches **13.8x** vs `re` on 8 threads. 🚀⚡ @@ -89,6 +89,8 @@ hard CPU affinity. | Call-local two-name `Match.expand` | **0.18 μs** | **0.15 μs** | | Call-local three-name `Match.expand` | **0.23 μs** | **0.20 μs** | | Call-local eight-name `Match.expand` | **0.44 μs** | **0.39 μs** | +| Literal-backslash + named `Match.expand` | **0.12 μs** | **0.10 μs** | +| Named + backslash-suffix `Match.expand` | **0.16 μs** | **0.13 μs** | | Repeated default `compile("(x)")` | **0.49 μs** | **0.38 μs** | | Repeated integer-flagged `compile("x", CASELESS)` | **1.16 μs** | **0.81 μs** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index bd092ae..c9e1627 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -58,6 +58,7 @@ def main() -> int: ("match.expand.numeric", lambda: captured.expand(r"[\1]")), ("match.expand.explicit", lambda: captured.expand(r"[\g<1>]")), ("match.expand.named", lambda: named_captured.expand(r"[\g]")), + ("match.expand.escaped", lambda: named_captured.expand(r"\\\g")), ( "match.expand.multi", lambda: multi_captured.expand(r"[\g]-\g"), diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index ca2b23b..02062b6 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -1482,14 +1482,15 @@ typedef struct { Py_ssize_t slash_index; Py_ssize_t reference_end; Py_ssize_t group_index; -} MatchExpandReference; + int literal_backslash; +} MatchExpandToken; static int match_expand_parse_reference(MatchObject *self, PyObject *template_obj, Py_ssize_t slash_index, Py_ssize_t template_length, - MatchExpandReference *reference) + MatchExpandToken *reference) { if (slash_index < 0 || slash_index + 1 >= template_length) { return 0; @@ -1498,6 +1499,13 @@ match_expand_parse_reference(MatchObject *self, Py_UCS4 following = self->subject_is_bytes ? (unsigned char)PyBytes_AS_STRING(template_obj)[slash_index + 1] : PyUnicode_ReadChar(template_obj, slash_index + 1); + if (following == '\\') { + reference->slash_index = slash_index; + reference->reference_end = slash_index + 2; + reference->group_index = -1; + reference->literal_backslash = 1; + return 1; + } if (following >= '1' && following <= '9') { if (slash_index + 2 < template_length) { Py_UCS4 next = self->subject_is_bytes @@ -1514,6 +1522,7 @@ match_expand_parse_reference(MatchObject *self, reference->slash_index = slash_index; reference->reference_end = slash_index + 2; reference->group_index = group_index; + reference->literal_backslash = 0; return 1; } @@ -1572,6 +1581,7 @@ match_expand_parse_reference(MatchObject *self, reference->slash_index = slash_index; reference->reference_end = cursor + 1; reference->group_index = group_index; + reference->literal_backslash = 0; return 1; } @@ -1586,25 +1596,25 @@ match_expand_checked_add(Py_ssize_t *total, Py_ssize_t value) return 0; } -#define MATCH_EXPAND_MAX_REFERENCES 8 +#define MATCH_EXPAND_MAX_TOKENS 8 static PyObject * -match_expand_multiple_references(MatchObject *self, - PyObject *template_obj, - Py_ssize_t first_slash, - Py_ssize_t template_length, - int *handled) +match_expand_multiple_tokens(MatchObject *self, + PyObject *template_obj, + Py_ssize_t first_slash, + Py_ssize_t template_length, + int *handled) { *handled = 0; - MatchExpandReference references[MATCH_EXPAND_MAX_REFERENCES]; - PyObject *groups[MATCH_EXPAND_MAX_REFERENCES] = {NULL}; - Py_ssize_t group_offsets[MATCH_EXPAND_MAX_REFERENCES] = {0}; - Py_ssize_t group_lengths[MATCH_EXPAND_MAX_REFERENCES] = {0}; + MatchExpandToken references[MATCH_EXPAND_MAX_TOKENS]; + PyObject *groups[MATCH_EXPAND_MAX_TOKENS] = {NULL}; + Py_ssize_t group_offsets[MATCH_EXPAND_MAX_TOKENS] = {0}; + Py_ssize_t group_lengths[MATCH_EXPAND_MAX_TOKENS] = {0}; Py_ssize_t reference_count = 0; Py_ssize_t slash_index = first_slash; while (slash_index >= 0) { - if (reference_count >= MATCH_EXPAND_MAX_REFERENCES || + if (reference_count >= MATCH_EXPAND_MAX_TOKENS || !match_expand_parse_reference( self, template_obj, @@ -1634,7 +1644,7 @@ match_expand_multiple_references(MatchObject *self, } } } - if (reference_count < 2) { + if (reference_count < 2 && !references[0].literal_backslash) { return NULL; } @@ -1644,30 +1654,38 @@ match_expand_multiple_references(MatchObject *self, Py_UCS4 max_character = self->subject_is_bytes ? 0 : PyUnicode_MAX_CHAR_VALUE(template_obj); for (Py_ssize_t i = 0; i < reference_count; ++i) { - size_t offset_index = (size_t)references[i].group_index * 2; - Py_ssize_t group_start = self->ovector[offset_index]; - Py_ssize_t group_end = self->ovector[offset_index + 1]; - if (group_start < 0 || group_end < 0) { + if (references[i].literal_backslash) { group_offsets[i] = 0; - group_lengths[i] = 0; - } else if (group_end < group_start || group_end > self->utf8_length) { - PyErr_SetString(PyExc_RuntimeError, "invalid capture offsets"); - goto error; - } else if (direct_groups) { - group_offsets[i] = group_start; - group_lengths[i] = group_end - group_start; + group_lengths[i] = 1; } else { - groups[i] = match_get_group_value(self, references[i].group_index); - if (groups[i] == NULL) { + size_t offset_index = (size_t)references[i].group_index * 2; + Py_ssize_t group_start = self->ovector[offset_index]; + Py_ssize_t group_end = self->ovector[offset_index + 1]; + if (group_start < 0 || group_end < 0) { + group_offsets[i] = 0; + group_lengths[i] = 0; + } else if (group_end < group_start || + group_end > self->utf8_length) { + PyErr_SetString(PyExc_RuntimeError, "invalid capture offsets"); goto error; - } - group_lengths[i] = PyObject_Length(groups[i]); - if (group_lengths[i] < 0) { - goto error; - } - if (!self->subject_is_bytes && - PyUnicode_MAX_CHAR_VALUE(groups[i]) > max_character) { - max_character = PyUnicode_MAX_CHAR_VALUE(groups[i]); + } else if (direct_groups) { + group_offsets[i] = group_start; + group_lengths[i] = group_end - group_start; + } else { + groups[i] = match_get_group_value( + self, references[i].group_index + ); + if (groups[i] == NULL) { + goto error; + } + group_lengths[i] = PyObject_Length(groups[i]); + if (group_lengths[i] < 0) { + goto error; + } + if (!self->subject_is_bytes && + PyUnicode_MAX_CHAR_VALUE(groups[i]) > max_character) { + max_character = PyUnicode_MAX_CHAR_VALUE(groups[i]); + } } } Py_ssize_t literal_length = references[i].slash_index - literal_start; @@ -1702,12 +1720,16 @@ match_expand_multiple_references(MatchObject *self, output_offset += literal_length; } if (group_lengths[i] > 0) { - const char *group_data = direct_groups - ? self->utf8_data + group_offsets[i] - : PyBytes_AS_STRING(groups[i]); - memcpy(output + output_offset, - group_data, - (size_t)group_lengths[i]); + if (references[i].literal_backslash) { + output[output_offset] = '\\'; + } else { + const char *group_data = direct_groups + ? self->utf8_data + group_offsets[i] + : PyBytes_AS_STRING(groups[i]); + memcpy(output + output_offset, + group_data, + (size_t)group_lengths[i]); + } output_offset += group_lengths[i]; } literal_start = references[i].reference_end; @@ -1744,14 +1766,22 @@ match_expand_multiple_references(MatchObject *self, output_offset += literal_length; } if (group_lengths[i] > 0) { - PyObject *group_source = direct_groups ? self->subject : groups[i]; - if (PyUnicode_CopyCharacters(result, - output_offset, - group_source, - group_offsets[i], - group_lengths[i]) < 0) { - Py_DECREF(result); - goto error; + if (references[i].literal_backslash) { + if (PyUnicode_WriteChar(result, output_offset, '\\') < 0) { + Py_DECREF(result); + goto error; + } + } else { + PyObject *group_source = direct_groups + ? self->subject : groups[i]; + if (PyUnicode_CopyCharacters(result, + output_offset, + group_source, + group_offsets[i], + group_lengths[i]) < 0) { + Py_DECREF(result); + goto error; + } } output_offset += group_lengths[i]; } @@ -1815,7 +1845,7 @@ Match_expand(MatchObject *self, PyObject *template_obj) if (handled || result != NULL || PyErr_Occurred()) { return result; } - result = match_expand_multiple_references( + result = match_expand_multiple_tokens( self, template_obj, slash_index, template_length, &handled ); if (handled || result != NULL || PyErr_Occurred()) { @@ -1868,7 +1898,7 @@ Match_expand(MatchObject *self, PyObject *template_obj) if (handled || result != NULL || PyErr_Occurred()) { return result; } - result = match_expand_multiple_references( + result = match_expand_multiple_tokens( self, template_obj, (Py_ssize_t)(slash - template_data), diff --git a/tests/test_expand_fastpath.py b/tests/test_expand_fastpath.py index 9124ef8..92caf65 100644 --- a/tests/test_expand_fastpath.py +++ b/tests/test_expand_fastpath.py @@ -40,6 +40,11 @@ (r"(?Pé)", "é", "前\\g後", "前é後"), (r"(?Pa)?(?Pb)?", "a", r"[\g]", "[]"), (b"(?Pa)", b"a", rb"[\g]", b"[a]"), + (r"(a)", "a", r"\\", "\\"), + (r"(a)", "a", r"\\1", r"\1"), + (r"(?Pa)", "a", r"\\\g", r"\a"), + (r"(?Pa)", "a", r"[\g]\\tail", r"[a]\tail"), + (b"(?Pa)", b"a", rb"\\\g", rb"\a"), ], ) def test_single_numeric_expand_is_exact_and_call_local( @@ -66,7 +71,7 @@ def test_single_numeric_expand_is_exact_and_call_local( "template", [ r"\12", - r"\\1", + r"\n", r"\g", r"\g<999999999999999999999999999>", r"\g<13>", @@ -97,6 +102,7 @@ def expand_many() -> None: assert match.expand("前\\1後") == "前é後" assert match.expand("前\\g後") == "前é後" assert match.expand("前\\1中\\g後") == "前é中é後" + assert match.expand("\\\\\\g") == "\\é" with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: list(executor.map(lambda _: expand_many(), range(8))) @@ -244,3 +250,24 @@ def test_multiple_reference_expand_uses_immutable_buffer_snapshot() -> None: subject[:] = b"zz" assert match.expand(rb"[\1]-\2") == b"[a]-b" + + +@pytest.mark.parametrize( + ("pattern", "subject"), + [ + (r"(?Pa)", "a"), + (r"(?Pé)", "é"), + (b"(?Pa)", b"a"), + ], +) +def test_literal_backslash_expand_differential_matrix(pattern, subject) -> None: + expected_match = re.fullmatch(pattern, subject) + actual_match = pcre.fullmatch(pattern, subject) + assert expected_match is not None + assert actual_match is not None + + templates = (r"\\", r"\\1", r"\\\g", r"[\g]\\tail") + for template in templates: + if isinstance(pattern, bytes): + template = template.encode() + assert actual_match.expand(template) == expected_match.expand(template) From c66dd4cb99724e87e5881cbbbfcbb5298a7e8780 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 18:51:46 +0800 Subject: [PATCH 07/21] perf: accelerate single-reference substitution --- README.md | 4 + benchmarks/api_hotpaths.py | 6 + pcre/pcre.py | 17 ++ pcre_ext/pcre2.c | 237 +++++++++++++++++++++++++++ tests/test_python_coverage_audit.py | 35 +++- tests/test_sub_reference_fastpath.py | 140 ++++++++++++++++ 6 files changed, 436 insertions(+), 3 deletions(-) create mode 100644 tests/test_sub_reference_fastpath.py diff --git a/README.md b/README.md index 2f1b4ca..60f5c9f 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 +* 08/10/2026 **Call-local single-reference substitution**: exact default-count `Pattern.sub`/`subn` replacements containing one valid numeric, explicit numeric, or named capture now translate and execute entirely in C without entering or growing the thread-local replacement-template cache. Pinned A/B measurements improve short numeric and explicit forms by roughly **1.9–2.1x** on Python 3.10 and free-threaded Python 3.14t/GIL=0; named replacement improves by **10.9x/1.9x**. Duplicate PCRE names select the participating capture, while `$`, multiple/ambiguous references, subclasses, and bounded counts retain the compatibility parser. ⚡🛡️ * 08/10/2026 **Call-local `Match.expand` fast paths**: exact text and bytes templates containing up to eight unambiguous capture/backslash tokens now render directly from the immutable C Match snapshot, without importing the compatibility parser or retaining a parsed-template cache entry. Pinned A/B measurements against merged `main` improve matched `[\\1]` expansion by **20.8x** on Python 3.10 and **16.7x** on free-threaded Python 3.14t/GIL=0; unmatched captures reach **25.5x/20.7x**, and bytes reach **37.4x/31.7x**. Explicit numeric `[\\g<1>]` improves by **14.3x/13.8x**, checked multi-digit references such as `\\g<12>` reach **63.3x/13.1x**, named `[\\g]` improves by **28.7x/10.6x**, and a two-name template reaches **45.3x/9.6x**. Three references reach **9.2x/8.2x**, while eight reach **6.3x/6.3x**. A literal backslash plus named capture reaches **13.9x/11.7x**, and a named capture with a backslash suffix reaches **42.9x/9.2x**. Duplicate-name alternatives select the participating capture. Nine-or-more tokens, non-backslash escapes, ambiguous two-digit, subclass, invalid, and non-ASCII-name templates continue through the fully compatible parser. ⚡🛡️ * 08/10/2026 **Literal split/substitution/findall fast paths**: exact plain-literal `Pattern.split` calls now use the immutable built-in splitter after construction-time validation, measuring **2.1x** faster than the prior C dispatch on Python 3.10 and **1.7x** faster on free-threaded Python 3.14t/GIL=0; delimiter-heavy multi-character literals reach roughly **4.8x**. Literal `Pattern.subn` and module-level `sub`/`subn` now use native replace/count primitives, reaching about **15x** on short repeated tokens and **3x** on delimiter-heavy text. Literal `findall` uses non-overlapping native count/list construction, reaching about **9x** on short repeated tokens and **8x** on delimiter-heavy text. Regex metacharacters, explicit flags, subclasses, and buffer subjects remain on the compatibility-safe PCRE2 path. ⚡ * 08/09/2026 **API hot-path update**: large `parallel_map(findall)` workloads now reach **11.5x** speedup on Python 3.10 and **11.25x** on free-threaded Python 3.14t/GIL=0 with 12 performance-tier workers. Ordered `parallel_map(search)` reaches **8.57x** and **7.85x**, respectively; one-item and up to eight tiny explicit `parallel_map` subjects now avoid executor setup (the one-item case measures **13.3x** faster on Python 3.10 and **27.7x** on 3.14t), default bound `Pattern.split` is another **1.6x/1.5x** faster on 3.10/3.14t, and default bound literal `Pattern.subn` is about **1.5x** faster on Python 3.10. Canonical module helpers retain their optimized dispatch while their wrapper/template caches are thread-scoped, size-bounded, and invalidated across live workers. Repeated backreference `Match.expand()` avoids reparsing within the active cache context, while captured values returned by `Match.groups()` remain call-local so a long-lived Match does not retain an additional copy of large captures. 🧵⚡ @@ -82,6 +83,9 @@ hard CPU affinity. | `parallel_map(findall)`, 48 × 1 MiB subjects, 12 workers | **11.51x** | **11.25x** | | Bound one-character literal `Pattern.split` | **2.1x** | **1.7x** | | Bound backreference `sub` hot path | **1.38 μs** | **1.14 μs** | +| One-match numeric-reference `Pattern.sub` | **0.45 μs** | **0.34 μs** | +| One-match explicit-reference `Pattern.sub` | **0.45 μs** | **0.31 μs** | +| One-match named-reference `Pattern.sub` | **0.46 μs** | **0.33 μs** | | Repeated call-local `Match.groups()` | **~0.05 μs** | **~0.05 μs** | | Call-local `Match.expand(r"[\\1]")` | **0.07 μs** | **0.07 μs** | | Call-local `Match.expand(r"[\\g<1>]")` | **0.11 μs** | **0.08 μs** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index c9e1627..b72f092 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -36,6 +36,7 @@ def main() -> int: subject = "x" * 1000 short_subject = "x" * 10 pattern = pcre.compile("(x)") + named_pattern = pcre.compile("(?Px)") captured = pattern.match(short_subject) named_captured = pcre.compile("(?Px)").match(short_subject) multi_captured = pcre.compile("(?Px)(?Px)").match(short_subject) @@ -54,6 +55,11 @@ def main() -> int: ("bound.split", lambda: pattern.split("x " * 8)), ("bound.sub.literal", lambda: pattern.sub("[X]", short_subject)), ("bound.sub.backref", lambda: pattern.sub(r"[\1]", short_subject)), + ("bound.sub.explicit", lambda: pattern.sub(r"[\g<1>]", short_subject)), + ( + "bound.sub.named", + lambda: named_pattern.sub(r"[\g]", short_subject), + ), ("match.groups", captured.groups), ("match.expand.numeric", lambda: captured.expand(r"[\1]")), ("match.expand.explicit", lambda: captured.expand(r"[\g<1>]")), diff --git a/pcre/pcre.py b/pcre/pcre.py index 2520645..305ff02 100644 --- a/pcre/pcre.py +++ b/pcre/pcre.py @@ -779,6 +779,23 @@ def subn(self, repl: Any, subject: Any, count: Any = 0) -> tuple[Any, int]: if fast_substitute is not None: return fast_substitute(subject, repl) + # One exact, valid Python capture reference can be translated to + # PCRE2's equivalent replacement syntax with call-local state only. + # Keep every ambiguous or extended form on the compatibility parser. + if ( + self._is_c_pattern + and type(self) is Pattern + and type(subject) in (str, bytes) + and type(repl) is type(subject) + and type(count) is int + and count == 0 + ): + fast_substitute = getattr(self._pattern, "_substitute_python_fast", None) + if fast_substitute is not None: + direct_result = fast_substitute(subject, repl) + if direct_result is not NotImplemented: + return direct_result + subject = prepare_subject(subject) subject_is_bytes = is_bytes_like(subject) empty = b"" if subject_is_bytes else "" diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index 02062b6..c86f9d2 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -4545,6 +4545,242 @@ Pattern_substitute_fast(PatternObject *self, PyObject *const *args, Py_ssize_t n return Pattern_substitute(self, args[0], args[1], 0); } +static int +pattern_has_ascii_group_name(PatternObject *self, + const char *name, + Py_ssize_t name_length) +{ + uint32_t name_count = 0; + uint32_t entry_size = 0; + PCRE2_SPTR name_table = NULL; + if (name_length <= 0 || + pcre2_pattern_info(self->code, PCRE2_INFO_NAMECOUNT, &name_count) != 0 || + pcre2_pattern_info(self->code, + PCRE2_INFO_NAMEENTRYSIZE, + &entry_size) != 0 || + pcre2_pattern_info(self->code, + PCRE2_INFO_NAMETABLE, + &name_table) != 0 || + name_table == NULL || entry_size < 3) { + return 0; + } + size_t name_max = (size_t)entry_size - 2; + for (uint32_t i = 0; i < name_count; ++i) { + const char *entry_name = (const char *)( + name_table + (size_t)i * entry_size + 2 + ); + size_t entry_length = strnlen(entry_name, name_max); + if (entry_length == (size_t)name_length && + memcmp(entry_name, name, entry_length) == 0) { + return 1; + } + } + return 0; +} + +static Py_UCS4 +replacement_character_at(PyObject *replacement, + int replacement_is_bytes, + Py_ssize_t index) +{ + return replacement_is_bytes + ? (unsigned char)PyBytes_AS_STRING(replacement)[index] + : PyUnicode_ReadChar(replacement, index); +} + +static Py_ssize_t +replacement_find_character(PyObject *replacement, + int replacement_is_bytes, + Py_UCS4 character, + Py_ssize_t start, + Py_ssize_t length) +{ + if (!replacement_is_bytes) { + return PyUnicode_FindChar(replacement, character, start, length, 1); + } + const char *data = PyBytes_AS_STRING(replacement); + const char *found = memchr( + data + start, (unsigned char)character, (size_t)(length - start) + ); + return found == NULL ? -1 : (Py_ssize_t)(found - data); +} + +static PyObject * +pattern_translate_single_replacement(PatternObject *self, + PyObject *replacement, + int *handled) +{ + *handled = 0; + int replacement_is_bytes = PyBytes_CheckExact(replacement); + if (!replacement_is_bytes && !PyUnicode_CheckExact(replacement)) { + return NULL; + } + if (!replacement_is_bytes && PyUnicode_READY(replacement) < 0) { + return NULL; + } + Py_ssize_t length = replacement_is_bytes + ? PyBytes_GET_SIZE(replacement) : PyUnicode_GET_LENGTH(replacement); + Py_ssize_t slash_index = replacement_find_character( + replacement, replacement_is_bytes, '\\', 0, length + ); + if (slash_index < 0 || + replacement_find_character( + replacement, replacement_is_bytes, '$', 0, length + ) >= 0 || + slash_index + 1 >= length) { + return NULL; + } + + Py_UCS4 following = replacement_character_at( + replacement, replacement_is_bytes, slash_index + 1 + ); + if (following >= '1' && following <= '9') { + if ((uint32_t)(following - '0') > self->capture_count || + (slash_index + 2 < length && + replacement_character_at( + replacement, replacement_is_bytes, slash_index + 2 + ) >= '0' && + replacement_character_at( + replacement, replacement_is_bytes, slash_index + 2 + ) <= '9') || + replacement_find_character( + replacement, + replacement_is_bytes, + '\\', + slash_index + 2, + length + ) >= 0) { + return NULL; + } + if (length > PY_SSIZE_T_MAX - 3) { + PyErr_NoMemory(); + return NULL; + } + *handled = 1; + Py_ssize_t result_length = length + 3; + if (replacement_is_bytes) { + PyObject *result = PyBytes_FromStringAndSize(NULL, result_length); + if (result == NULL) { + return NULL; + } + char *output = PyBytes_AS_STRING(result); + const char *input = PyBytes_AS_STRING(replacement); + memcpy(output, input, (size_t)slash_index); + output[slash_index] = '\\'; + output[slash_index + 1] = 'g'; + output[slash_index + 2] = '<'; + output[slash_index + 3] = (char)following; + output[slash_index + 4] = '>'; + memcpy(output + slash_index + 5, + input + slash_index + 2, + (size_t)(length - slash_index - 2)); + return result; + } + + PyObject *result = PyUnicode_New( + result_length, PyUnicode_MAX_CHAR_VALUE(replacement) + ); + if (result == NULL) { + return NULL; + } + if ((slash_index > 0 && + PyUnicode_CopyCharacters( + result, 0, replacement, 0, slash_index + ) < 0) || + PyUnicode_WriteChar(result, slash_index, '\\') < 0 || + PyUnicode_WriteChar(result, slash_index + 1, 'g') < 0 || + PyUnicode_WriteChar(result, slash_index + 2, '<') < 0 || + PyUnicode_WriteChar(result, slash_index + 3, following) < 0 || + PyUnicode_WriteChar(result, slash_index + 4, '>') < 0 || + (slash_index + 2 < length && + PyUnicode_CopyCharacters(result, + slash_index + 5, + replacement, + slash_index + 2, + length - slash_index - 2) < 0)) { + Py_DECREF(result); + return NULL; + } + return result; + } + + if (following != 'g' || slash_index + 4 >= length || + replacement_character_at( + replacement, replacement_is_bytes, slash_index + 2 + ) != '<') { + return NULL; + } + char name[129]; + Py_ssize_t name_length = 0; + uint32_t group_index = 0; + int numeric = 1; + Py_ssize_t cursor = slash_index + 3; + while (cursor < length) { + Py_UCS4 character = replacement_character_at( + replacement, replacement_is_bytes, cursor + ); + if (character == '>') { + break; + } + if (character > 0x7f || name_length >= 128) { + return NULL; + } + name[name_length++] = (char)character; + if (character < '0' || character > '9') { + numeric = 0; + } else if (numeric) { + uint32_t digit = (uint32_t)(character - '0'); + if (group_index > (UINT32_MAX - digit) / 10) { + return NULL; + } + group_index = group_index * 10 + digit; + } + cursor += 1; + } + if (name_length == 0 || cursor >= length || + replacement_find_character( + replacement, + replacement_is_bytes, + '\\', + cursor + 1, + length + ) >= 0 || + (numeric + ? group_index > self->capture_count + : !pattern_has_ascii_group_name(self, name, name_length))) { + return NULL; + } + *handled = 1; + Py_INCREF(replacement); + return replacement; +} + +static PyObject * +Pattern_substitute_python_fast(PatternObject *self, + PyObject *const *args, + Py_ssize_t nargs) +{ + if (nargs != 2) { + PyErr_Format(PyExc_TypeError, + "_substitute_python_fast() takes exactly 2 positional arguments (%zd given)", + nargs); + return NULL; + } + int handled = 0; + PyObject *replacement = pattern_translate_single_replacement( + self, args[1], &handled + ); + if (replacement == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NOTIMPLEMENTED; + } + PyObject *result = Pattern_substitute(self, args[0], replacement, 0); + Py_DECREF(replacement); + return result; +} + /* * These lookup helpers are intentionally private. They are used only by the * high-level parallel fan-out when every optional argument has its default @@ -4636,6 +4872,7 @@ static PyMethodDef Pattern_methods[] = { {"fullmatch", (PyCFunction)Pattern_fullmatch_method, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Require the pattern to match the entire subject." )}, {"_findall_fast", (PyCFunction)(void(*)(void))Pattern_findall_fast, METH_FASTCALL, NULL}, {"_substitute_fast", (PyCFunction)(void(*)(void))Pattern_substitute_fast, METH_FASTCALL, NULL}, + {"_substitute_python_fast", (PyCFunction)(void(*)(void))Pattern_substitute_python_fast, METH_FASTCALL, NULL}, {"_match_fast", (PyCFunction)(void(*)(void))Pattern_match_fast, METH_FASTCALL, NULL}, {"_search_fast", (PyCFunction)(void(*)(void))Pattern_search_fast, METH_FASTCALL, NULL}, {"_fullmatch_fast", (PyCFunction)(void(*)(void))Pattern_fullmatch_fast, METH_FASTCALL, NULL}, diff --git a/tests/test_python_coverage_audit.py b/tests/test_python_coverage_audit.py index 540b4ff..fe02968 100644 --- a/tests/test_python_coverage_audit.py +++ b/tests/test_python_coverage_audit.py @@ -59,18 +59,36 @@ def counted_parse(template: Any, state: Any) -> Any: return original(template, state) monkeypatch.setattr(pcre_mod._parser, "parse_template", counted_parse) - assert pattern.sub(r"[\1]", "x") == "[x]" - assert pattern.sub(r"[\1]", "x") == "[x]" + assert pattern.sub(r"[\1]-\1", "x") == "[x]-x" + assert pattern.sub(r"[\1]-\1", "x") == "[x]-x" assert calls == 1 pcre.clear_cache() - assert pattern.sub(r"[\1]", "x") == "[x]" + assert pattern.sub(r"[\1]-\1", "x") == "[x]-x" assert calls == 2 with pytest.raises(pcre.PcreError): pattern.sub(r"\2", "x", count=1) +def test_replacement_template_cache_disabled_and_explicit_clear() -> None: + pattern = pcre.compile(r"(x)") + original_limit = cache_mod.get_cache_limit() + try: + cache_mod.set_cache_limit(0) + pcre_mod._cached_replacement_parts(pattern, r"\1-\1", False) + + cache_mod.set_cache_limit(2) + pcre_mod._cached_replacement_parts.cache_clear() + pcre_mod._cached_replacement_parts(pattern, r"\1-\1", False) + assert pcre_mod._replacement_cache_size() == 1 + pcre_mod._cached_replacement_parts.cache_clear() + assert pcre_mod._replacement_cache_size() == 0 + finally: + cache_mod.set_cache_limit(original_limit) + pcre.clear_cache() + + def test_local_cache_lazy_initializers_and_module_lru_clear() -> None: pcre.clear_cache() pcre_mod._DEFAULT_COMPILE_LOCAL.flagged_cache = None @@ -237,11 +255,22 @@ def test_expand_template_cache_limit_and_trim_branches() -> None: long_name = "n" * (compat._MAX_GROUPINDEX_CACHE_UNITS + 1) assert compat._cached_expand_template("name-not-cached", 1, ((long_name, 1),)) assert compat._expand_template_cache_size() == 0 + + compat._EXPAND_TEMPLATE_LOCAL.epoch = cache_mod.get_cache_epoch() - 1 + compat._EXPAND_TEMPLATE_LOCAL.lru = None + assert compat._cached_expand_template("stale-without-lru", 0, ()) finally: cache_mod.set_cache_limit(original_limit) pcre.clear_cache() +def test_bytes_match_expand_coerces_unsupported_bytearray_template() -> None: + match = pcre.fullmatch(b"(a)", b"a") + assert match is not None + + assert match.expand(bytearray(rb"\n")) == b"\n" + + def test_default_compile_cache_is_thread_local_and_tracks_thread_mode( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/test_sub_reference_fastpath.py b/tests/test_sub_reference_fastpath.py new file mode 100644 index 0000000..2a5b0c9 --- /dev/null +++ b/tests/test_sub_reference_fastpath.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: 2025 ModelCloud.ai +# SPDX-FileCopyrightText: 2025 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +from __future__ import annotations + +import concurrent.futures +import re + +import pytest + +import pcre +from pcre import pcre as pcre_mod + + +@pytest.mark.parametrize( + ("pattern", "subject", "replacement"), + [ + (r"(a)(b)?", "a ab", r"[\1]"), + (r"(a)(b)?", "a ab", r"[\g<0>]"), + (r"(a)(b)?", "a ab", r"[\g<02>]"), + (r"(?Pa)(?Pb)?", "a ab", r"[\g]"), + (r"(?Pé)", "é é", "前\\g後"), + (b"(a)(b)?", b"a ab", rb"[\1]"), + (b"(?Pa)", b"a a", rb"[\g]"), + ], +) +def test_single_reference_subn_is_exact_and_call_local( + pattern, + subject, + replacement, + monkeypatch: pytest.MonkeyPatch, +) -> None: + compiled = pcre.compile(pattern) + expected = re.compile(pattern).subn(replacement, subject) + pcre.clear_cache() + monkeypatch.setattr( + pcre_mod, + "_cached_replacement_parts", + lambda *args: pytest.fail("single replacement reached template cache"), + ) + + assert compiled.subn(replacement, subject) == expected + assert pcre_mod._replacement_cache_size() == 0 + + +@pytest.mark.parametrize( + "replacement", + [r"\1-\2", r"[\1]$", r"\n\1"], +) +def test_extended_replacements_stay_on_compatibility_parser( + replacement: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + compiled = pcre.compile(r"(a)(b)") + original = pcre_mod._cached_replacement_parts + calls = 0 + + def counted(*args): + nonlocal calls + calls += 1 + return original(*args) + + monkeypatch.setattr(pcre_mod, "_cached_replacement_parts", counted) + assert compiled.sub(replacement, "ab") == re.sub(r"(a)(b)", replacement, "ab") + assert calls == 1 + + +def test_bounded_count_and_replacement_subclass_stay_compatible( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class Replacement(str): + pass + + compiled = pcre.compile(r"(a)") + original = pcre_mod._cached_replacement_parts + calls = 0 + + def counted(*args): + nonlocal calls + calls += 1 + return original(*args) + + monkeypatch.setattr(pcre_mod, "_cached_replacement_parts", counted) + assert compiled.sub(r"[\1]", "aa", count=1) == "[a]a" + assert compiled.sub(Replacement(r"[\1]"), "a") == "[a]" + assert calls == 1 + + +def test_single_reference_subn_is_safe_on_shared_pattern() -> None: + compiled = pcre.compile(r"(?Pé)") + + def replace_many() -> None: + for _ in range(5_000): + assert compiled.subn("前\\g後", "é é") == ("前é後 前é後", 2) + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(lambda _: replace_many(), range(8))) + + +@pytest.mark.parametrize("subject", ["a", "b"]) +def test_duplicate_named_substitution_selects_participating_capture( + subject: str, +) -> None: + compiled = pcre.compile(r"(?J)(?Pa)|(?Pb)") + + assert compiled.subn(r"[\g]", subject) == (f"[{subject}]", 1) + + +def test_single_reference_translator_rejects_ambiguous_inputs() -> None: + compiled = pcre.compile(r"(?Pa)(b)") + rejected = [ + "literal", + "$\\1", + "\\", + "\\3", + "\\12", + "\\1\\2", + "\\q", + "\\g\\1", + "\\g<>", + "\\g<99999999999>", + "\\g<3>", + "\\g<未知>", + "\\g", + ] + for replacement in rejected: + assert ( + compiled._pattern._substitute_python_fast("ab", replacement) + is NotImplemented + ) + + assert ( + compiled._pattern._substitute_python_fast(b"ab", b"\\g<\xff>") is NotImplemented + ) + + with pytest.raises(TypeError): + compiled._pattern._substitute_python_fast("ab") From 63e7edcdaa6f9696125dde13dfab188aeb868aed Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 18:55:27 +0800 Subject: [PATCH 08/21] bench: stabilize cross-api hot-path timings --- benchmarks/api_hotpaths.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index b72f092..ff943bc 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -7,7 +7,8 @@ The benchmark intentionally uses short subjects so Python dispatch, template parsing, and object-wrapper costs are visible instead of being hidden by a -large PCRE2 scan. Set ``PYPCRE_BENCH_RUNS`` to change the iteration count. +large PCRE2 scan. Set ``PYPCRE_BENCH_RUNS`` and ``PYPCRE_BENCH_REPEATS`` to +change the sample size. When running on a free-threaded build, an additional shared-pattern workload checks the concurrent execution path. """ @@ -16,20 +17,25 @@ import concurrent.futures import os +import statistics import sys import time +import timeit from collections.abc import Callable import pcre -RUNS = int(os.getenv("PYPCRE_BENCH_RUNS", "50000")) +RUNS = int(os.getenv("PYPCRE_BENCH_RUNS", "10000")) +REPEATS = int(os.getenv("PYPCRE_BENCH_REPEATS", "5")) def _time(fn: Callable[[], object]) -> float: - started = time.perf_counter() - for _ in range(RUNS): - fn() - return (time.perf_counter() - started) * 1_000_000.0 / RUNS + # A separate Timer gives every operation a monomorphic call site. Keep the + # default whole-suite duration short as well: on asymmetric macOS hosts a + # sustained microbenchmark can migrate to efficiency cores despite its + # performance-tier task policy, creating a visible step in later rows. + samples = timeit.Timer(fn).repeat(repeat=REPEATS, number=RUNS) + return statistics.median(samples) * 1_000_000.0 / RUNS def main() -> int: @@ -83,7 +89,10 @@ def main() -> int: ] gil_enabled = getattr(sys, "_is_gil_enabled", lambda: True)() - print(f"runtime={sys.version.split()[0]} gil_enabled={gil_enabled} runs={RUNS}") + print( + f"runtime={sys.version.split()[0]} gil_enabled={gil_enabled} " + f"runs={RUNS} repeats={REPEATS}" + ) for name, operation in operations: print(f"{name:22s} {_time(operation):8.3f} us") From 6af31163937da9fc14702bae873cb1269bf80b18 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 19:10:24 +0800 Subject: [PATCH 09/21] Accelerate stdlib-compatible escape --- README.md | 3 + benchmarks/api_hotpaths.py | 3 + pcre/__init__.py | 8 +- pcre_ext/pcre2.c | 1 + pcre_ext/pcre2_module.h | 4 + pcre_ext/string_helpers.c | 163 ++++++++++++++++++++++++++++++++++ tests/test_escape_fastpath.py | 66 ++++++++++++++ 7 files changed, 241 insertions(+), 7 deletions(-) create mode 100644 tests/test_escape_fastpath.py diff --git a/README.md b/README.md index 60f5c9f..3868c69 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 +* 08/10/2026 **Stateless `escape` fast path**: exact immutable text and bytes now use a native `re.escape`-compatible scanner with no cache, retained parsing state, or cross-thread ownership. Pinned A/B measurements against the previous Python wrapper improve short no-op text by **5.0x/3.6x** on Python 3.10/3.14t and no-op bytes by **6.9x/6.5x**; short escaped punctuation improves by **3.6x/3.1x**. Mutable buffers and subclasses continue through stdlib coercion/dynamic dispatch, and exhaustive byte plus randomized Unicode parity checks cover the native path. ⚡🛡️ * 08/10/2026 **Call-local single-reference substitution**: exact default-count `Pattern.sub`/`subn` replacements containing one valid numeric, explicit numeric, or named capture now translate and execute entirely in C without entering or growing the thread-local replacement-template cache. Pinned A/B measurements improve short numeric and explicit forms by roughly **1.9–2.1x** on Python 3.10 and free-threaded Python 3.14t/GIL=0; named replacement improves by **10.9x/1.9x**. Duplicate PCRE names select the participating capture, while `$`, multiple/ambiguous references, subclasses, and bounded counts retain the compatibility parser. ⚡🛡️ * 08/10/2026 **Call-local `Match.expand` fast paths**: exact text and bytes templates containing up to eight unambiguous capture/backslash tokens now render directly from the immutable C Match snapshot, without importing the compatibility parser or retaining a parsed-template cache entry. Pinned A/B measurements against merged `main` improve matched `[\\1]` expansion by **20.8x** on Python 3.10 and **16.7x** on free-threaded Python 3.14t/GIL=0; unmatched captures reach **25.5x/20.7x**, and bytes reach **37.4x/31.7x**. Explicit numeric `[\\g<1>]` improves by **14.3x/13.8x**, checked multi-digit references such as `\\g<12>` reach **63.3x/13.1x**, named `[\\g]` improves by **28.7x/10.6x**, and a two-name template reaches **45.3x/9.6x**. Three references reach **9.2x/8.2x**, while eight reach **6.3x/6.3x**. A literal backslash plus named capture reaches **13.9x/11.7x**, and a named capture with a backslash suffix reaches **42.9x/9.2x**. Duplicate-name alternatives select the participating capture. Nine-or-more tokens, non-backslash escapes, ambiguous two-digit, subclass, invalid, and non-ASCII-name templates continue through the fully compatible parser. ⚡🛡️ * 08/10/2026 **Literal split/substitution/findall fast paths**: exact plain-literal `Pattern.split` calls now use the immutable built-in splitter after construction-time validation, measuring **2.1x** faster than the prior C dispatch on Python 3.10 and **1.7x** faster on free-threaded Python 3.14t/GIL=0; delimiter-heavy multi-character literals reach roughly **4.8x**. Literal `Pattern.subn` and module-level `sub`/`subn` now use native replace/count primitives, reaching about **15x** on short repeated tokens and **3x** on delimiter-heavy text. Literal `findall` uses non-overlapping native count/list construction, reaching about **9x** on short repeated tokens and **8x** on delimiter-heavy text. Regex metacharacters, explicit flags, subclasses, and buffer subjects remain on the compatibility-safe PCRE2 path. ⚡ @@ -81,6 +82,8 @@ hard CPU affinity. | --- | ---: | ---: | | `parallel_map(search)`, 16 × 1 MiB subjects, 12 workers | **8.57x** | **7.85x** | | `parallel_map(findall)`, 48 × 1 MiB subjects, 12 workers | **11.51x** | **11.25x** | +| No-op `escape("literal")` | **5.0x** | **3.6x** | +| No-op `escape(b"literal")` | **6.9x** | **6.5x** | | Bound one-character literal `Pattern.split` | **2.1x** | **1.7x** | | Bound backreference `sub` hot path | **1.38 μs** | **1.14 μs** | | One-match numeric-reference `Pattern.sub` | **0.45 μs** | **0.34 μs** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index ff943bc..083c288 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -67,6 +67,9 @@ def main() -> int: lambda: named_pattern.sub(r"[\g]", short_subject), ), ("match.groups", captured.groups), + ("module.escape.text", lambda: pcre.escape("identifier_123")), + ("module.escape.bytes", lambda: pcre.escape(b"identifier123")), + ("module.escape.special", lambda: pcre.escape("a+b [c]")), ("match.expand.numeric", lambda: captured.expand(r"[\1]")), ("match.expand.explicit", lambda: captured.expand(r"[\g<1>]")), ("match.expand.named", lambda: named_captured.expand(r"[\g]")), diff --git a/pcre/__init__.py b/pcre/__init__.py index c7f0e5a..3ea7af8 100644 --- a/pcre/__init__.py +++ b/pcre/__init__.py @@ -13,8 +13,6 @@ from __future__ import annotations import importlib as _importlib -import re as _std_re -from typing import Any import pcre_ext_c as _backend @@ -42,7 +40,6 @@ ) from .threads import configure_thread_pool, configure_threads, shutdown_thread_pool - _error_module = _importlib.import_module(".error", __name__) pcre_ext_c = _backend @@ -76,10 +73,7 @@ def _error_code_property(self) -> PcreErrorCode | None: PatternError = PcreError -def escape(pattern: Any) -> Any: - """Escape special characters in *pattern* using :mod:`re` semantics.""" - - return _std_re.escape(pattern) +escape = _backend.escape # Compat: expose stdlib-style flag constants so migrating `re` users can diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index c86f9d2..08b7d42 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -5398,6 +5398,7 @@ static PyMethodDef module_methods[] = { {"_cpu_ascii_vector_mode", (PyCFunction)module_cpu_ascii_vector_mode, METH_NOARGS, PyDoc_STR("Return the active ASCII vector width (0=scalar,1=SSE2,2=AVX2,3=AVX512)." )}, {"_debug_thread_cache_count", (PyCFunction)module_debug_thread_cache_count, METH_NOARGS, PyDoc_STR("Return the number of live thread cache states (requires PYPCRE_DEBUG=1)." )}, {"translate_unicode_escapes", (PyCFunction)module_translate_unicode_escapes, METH_O, PyDoc_STR("Translate literal \\uXXXX/\\UXXXXXXXX escapes to PCRE2-compatible \\x{...} sequences." )}, + {"escape", (PyCFunction)(void(*)(void))module_escape, METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("escape($module, pattern)\n--\n\nEscape special characters using re.escape semantics.")}, {NULL, NULL, 0, NULL}, }; diff --git a/pcre_ext/pcre2_module.h b/pcre_ext/pcre2_module.h index 5a43e7e..db190a0 100644 --- a/pcre_ext/pcre2_module.h +++ b/pcre_ext/pcre2_module.h @@ -152,6 +152,10 @@ int ensure_valid_utf8_for_bytes_subject(PatternObject *pattern, int subject_is_b int ascii_vector_mode(void); PyObject *module_translate_unicode_escapes(PyObject *module, PyObject *arg); PyObject *module_cpu_ascii_vector_mode(PyObject *module, PyObject *args); +PyObject *module_escape(PyObject *module, + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwnames); /* Memory management */ int pcre_memory_initialize(void); diff --git a/pcre_ext/string_helpers.c b/pcre_ext/string_helpers.c index 08ac06a..b88076e 100644 --- a/pcre_ext/string_helpers.c +++ b/pcre_ext/string_helpers.c @@ -13,6 +13,169 @@ # include #endif +static inline int +escape_required(Py_UCS4 character) +{ + switch (character) { + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + case '?': + case '*': + case '+': + case '-': + case '|': + case '^': + case '$': + case '\\': + case '.': + case '&': + case '~': + case '#': + case ' ': + case '\t': + case '\n': + case '\v': + case '\f': + case '\r': + return 1; + default: + return 0; + } +} + +static PyObject * +escape_exact_bytes(PyObject *pattern) +{ + const unsigned char *input = (const unsigned char *)PyBytes_AS_STRING(pattern); + Py_ssize_t length = PyBytes_GET_SIZE(pattern); + Py_ssize_t escape_count = 0; + for (Py_ssize_t index = 0; index < length; ++index) { + escape_count += escape_required((Py_UCS4)input[index]); + } + + if (escape_count == 0) { + return Py_NewRef(pattern); + } + if (length > PY_SSIZE_T_MAX - escape_count) { + return PyErr_NoMemory(); + } + + PyObject *result = PyBytes_FromStringAndSize(NULL, length + escape_count); + if (result == NULL) { + return NULL; + } + char *output = PyBytes_AS_STRING(result); + for (Py_ssize_t index = 0; index < length; ++index) { + unsigned char character = input[index]; + if (escape_required((Py_UCS4)character)) { + *output++ = '\\'; + } + *output++ = (char)character; + } + return result; +} + +static PyObject * +escape_exact_unicode(PyObject *pattern) +{ + Py_ssize_t length = PyUnicode_GET_LENGTH(pattern); + int input_kind = PyUnicode_KIND(pattern); + void *input_data = PyUnicode_DATA(pattern); + Py_ssize_t escape_count = 0; + for (Py_ssize_t index = 0; index < length; ++index) { + escape_count += escape_required(PyUnicode_READ(input_kind, input_data, index)); + } + + if (escape_count == 0) { + return Py_NewRef(pattern); + } + if (length > PY_SSIZE_T_MAX - escape_count) { + return PyErr_NoMemory(); + } + + PyObject *result = PyUnicode_New( + length + escape_count, + PyUnicode_MAX_CHAR_VALUE(pattern) + ); + if (result == NULL) { + return NULL; + } + int output_kind = PyUnicode_KIND(result); + void *output_data = PyUnicode_DATA(result); + Py_ssize_t output_index = 0; + for (Py_ssize_t index = 0; index < length; ++index) { + Py_UCS4 character = PyUnicode_READ(input_kind, input_data, index); + if (escape_required(character)) { + PyUnicode_WRITE(output_kind, output_data, output_index++, '\\'); + } + PyUnicode_WRITE(output_kind, output_data, output_index++, character); + } + return result; +} + +static PyObject * +escape_stdlib_fallback(PyObject *pattern) +{ + /* Preserve dynamic ``str.translate`` overrides and the full stdlib buffer + * coercion/error behaviour for non-exact inputs. This is intentionally + * uncached: keeping a ``re`` module or callable in process-global extension + * state would create interpreter-lifetime and teardown hazards. */ + PyObject *re_module = PyImport_ImportModule("re"); + if (re_module == NULL) { + return NULL; + } + PyObject *escape_callable = PyObject_GetAttrString(re_module, "escape"); + Py_DECREF(re_module); + if (escape_callable == NULL) { + return NULL; + } + PyObject *result = PyObject_CallOneArg(escape_callable, pattern); + Py_DECREF(escape_callable); + return result; +} + +PyObject * +module_escape(PyObject *Py_UNUSED(module), + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwnames) +{ + Py_ssize_t keyword_count = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames); + PyObject *pattern = NULL; + + if (nargs == 1 && keyword_count == 0) { + pattern = args[0]; + } else if (nargs == 0 && keyword_count == 1) { + PyObject *keyword = PyTuple_GET_ITEM(kwnames, 0); + if (PyUnicode_Check(keyword) && + PyUnicode_CompareWithASCIIString(keyword, "pattern") == 0) { + pattern = args[0]; + } else { + PyErr_Format(PyExc_TypeError, + "escape() got an unexpected keyword argument '%U'", + keyword); + return NULL; + } + } else { + PyErr_Format(PyExc_TypeError, + "escape() takes 1 argument (%zd given)", + nargs + keyword_count); + return NULL; + } + + if (PyUnicode_CheckExact(pattern)) { + return escape_exact_unicode(pattern); + } + if (PyBytes_CheckExact(pattern)) { + return escape_exact_bytes(pattern); + } + return escape_stdlib_fallback(pattern); +} + static inline Py_ssize_t ascii_prefix_length_scalar(const char *data, Py_ssize_t max_len) { diff --git a/tests/test_escape_fastpath.py b/tests/test_escape_fastpath.py new file mode 100644 index 0000000..9daba51 --- /dev/null +++ b/tests/test_escape_fastpath.py @@ -0,0 +1,66 @@ +import inspect +import re + +import pytest + +import pcre + + +@pytest.mark.parametrize( + "value", + [ + "", + "identifier_123", + "éclair世界", + "()[]{}?*+-|^$\\.&~# \t\n\v\f\r", + b"", + b"identifier123", + bytes(range(256)), + ], +) +def test_escape_exact_builtins_match_stdlib(value): + assert pcre.escape(value) == re.escape(value) + + +def test_escape_noop_exact_builtins_reuse_immutable_input(): + text = "identifier_123" + data = b"identifier123" + assert pcre.escape(text) is text + assert pcre.escape(data) is data + + +@pytest.mark.parametrize("factory", [bytearray, memoryview]) +def test_escape_bytes_like_fallback_matches_stdlib(factory): + value = factory(b"a+b [c]") + assert pcre.escape(value) == re.escape(value) + + +def test_escape_preserves_str_subclass_translate_override(): + class CustomString(str): + def translate(self, table): + assert isinstance(table, dict) + return "custom-result" + + assert pcre.escape(CustomString("a+b")) == "custom-result" + + +def test_escape_bytes_subclass_matches_stdlib(): + class CustomBytes(bytes): + pass + + value = CustomBytes(b"a+b") + assert pcre.escape(value) == re.escape(value) + assert type(pcre.escape(value)) is bytes + + +def test_escape_keyword_and_signature_match_stdlib(): + assert pcre.escape(pattern="a+b") == re.escape(pattern="a+b") + assert inspect.signature(pcre.escape) == inspect.signature(re.escape) + + +@pytest.mark.parametrize("value", [None, 1, object()]) +def test_escape_invalid_input_matches_stdlib_exception(value): + with pytest.raises(TypeError): + re.escape(value) + with pytest.raises(TypeError): + pcre.escape(value) From 85abfe17dc5db889791e093b77316919e3fa0a48 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 19:23:01 +0800 Subject: [PATCH 10/21] Accelerate count-one substitutions --- README.md | 5 ++ benchmarks/api_hotpaths.py | 2 + pcre/pcre.py | 20 +++++--- pcre_ext/pcre2.c | 34 ++++++++---- tests/test_sub_count_one_fastpath.py | 77 ++++++++++++++++++++++++++++ tests/test_sub_reference_fastpath.py | 4 +- 6 files changed, 125 insertions(+), 17 deletions(-) create mode 100644 tests/test_sub_count_one_fastpath.py diff --git a/README.md b/README.md index 3868c69..fa36b6b 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 +* 08/10/2026 **Native count-one substitution**: exact `Pattern.sub`/`subn` calls with `count=1` now stay in PCRE2 for literal, numeric, explicit numeric, and named replacements instead of rebuilding the bounded result through a Python match loop. Pinned A/B measurements improve the four bound forms by **4.2–4.7x** on Python 3.10 and **4.1–4.3x** on free-threaded Python 3.14t/GIL=0; module-level forms improve by **2.4–3.2x**. Translation is call-local and never grows the replacement-template cache; `count>=2`, ambiguous templates, subclasses, mutable buffers, and callables retain the compatibility path. ⚡🛡️ * 08/10/2026 **Stateless `escape` fast path**: exact immutable text and bytes now use a native `re.escape`-compatible scanner with no cache, retained parsing state, or cross-thread ownership. Pinned A/B measurements against the previous Python wrapper improve short no-op text by **5.0x/3.6x** on Python 3.10/3.14t and no-op bytes by **6.9x/6.5x**; short escaped punctuation improves by **3.6x/3.1x**. Mutable buffers and subclasses continue through stdlib coercion/dynamic dispatch, and exhaustive byte plus randomized Unicode parity checks cover the native path. ⚡🛡️ * 08/10/2026 **Call-local single-reference substitution**: exact default-count `Pattern.sub`/`subn` replacements containing one valid numeric, explicit numeric, or named capture now translate and execute entirely in C without entering or growing the thread-local replacement-template cache. Pinned A/B measurements improve short numeric and explicit forms by roughly **1.9–2.1x** on Python 3.10 and free-threaded Python 3.14t/GIL=0; named replacement improves by **10.9x/1.9x**. Duplicate PCRE names select the participating capture, while `$`, multiple/ambiguous references, subclasses, and bounded counts retain the compatibility parser. ⚡🛡️ * 08/10/2026 **Call-local `Match.expand` fast paths**: exact text and bytes templates containing up to eight unambiguous capture/backslash tokens now render directly from the immutable C Match snapshot, without importing the compatibility parser or retaining a parsed-template cache entry. Pinned A/B measurements against merged `main` improve matched `[\\1]` expansion by **20.8x** on Python 3.10 and **16.7x** on free-threaded Python 3.14t/GIL=0; unmatched captures reach **25.5x/20.7x**, and bytes reach **37.4x/31.7x**. Explicit numeric `[\\g<1>]` improves by **14.3x/13.8x**, checked multi-digit references such as `\\g<12>` reach **63.3x/13.1x**, named `[\\g]` improves by **28.7x/10.6x**, and a two-name template reaches **45.3x/9.6x**. Three references reach **9.2x/8.2x**, while eight reach **6.3x/6.3x**. A literal backslash plus named capture reaches **13.9x/11.7x**, and a named capture with a backslash suffix reaches **42.9x/9.2x**. Duplicate-name alternatives select the participating capture. Nine-or-more tokens, non-backslash escapes, ambiguous two-digit, subclass, invalid, and non-ASCII-name templates continue through the fully compatible parser. ⚡🛡️ @@ -84,6 +85,10 @@ hard CPU affinity. | `parallel_map(findall)`, 48 × 1 MiB subjects, 12 workers | **11.51x** | **11.25x** | | No-op `escape("literal")` | **5.0x** | **3.6x** | | No-op `escape(b"literal")` | **6.9x** | **6.5x** | +| Bound literal `sub(..., count=1)` | **4.7x** | **4.3x** | +| Bound numeric-reference `sub(..., count=1)` | **4.3x** | **4.1x** | +| Bound explicit-reference `sub(..., count=1)` | **4.6x** | **4.3x** | +| Bound named-reference `sub(..., count=1)` | **4.2x** | **4.3x** | | Bound one-character literal `Pattern.split` | **2.1x** | **1.7x** | | Bound backreference `sub` hot path | **1.38 μs** | **1.14 μs** | | One-match numeric-reference `Pattern.sub` | **0.45 μs** | **0.34 μs** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index 083c288..8edafeb 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -60,7 +60,9 @@ def main() -> int: ("bound.finditer", lambda: list(pattern.finditer(short_subject))), ("bound.split", lambda: pattern.split("x " * 8)), ("bound.sub.literal", lambda: pattern.sub("[X]", short_subject)), + ("bound.sub.literal1", lambda: pattern.sub("[X]", short_subject, count=1)), ("bound.sub.backref", lambda: pattern.sub(r"[\1]", short_subject)), + ("bound.sub.backref1", lambda: pattern.sub(r"[\1]", short_subject, count=1)), ("bound.sub.explicit", lambda: pattern.sub(r"[\g<1>]", short_subject)), ( "bound.sub.named", diff --git a/pcre/pcre.py b/pcre/pcre.py index 305ff02..cace1aa 100644 --- a/pcre/pcre.py +++ b/pcre/pcre.py @@ -771,13 +771,15 @@ def subn(self, repl: Any, subject: Any, count: Any = 0) -> tuple[Any, int]: and type(subject) in (str, bytes) and type(repl) is type(subject) and type(count) is int - and count == 0 + and count in (0, 1) and ("\\" not in repl if type(repl) is str else b"\\" not in repl) and ("$" not in repl if type(repl) is str else b"$" not in repl) ): fast_substitute = getattr(self._pattern, "_substitute_fast", None) if fast_substitute is not None: - return fast_substitute(subject, repl) + if count == 0: + return fast_substitute(subject, repl) + return fast_substitute(subject, repl, count) # One exact, valid Python capture reference can be translated to # PCRE2's equivalent replacement syntax with call-local state only. @@ -788,11 +790,15 @@ def subn(self, repl: Any, subject: Any, count: Any = 0) -> tuple[Any, int]: and type(subject) in (str, bytes) and type(repl) is type(subject) and type(count) is int - and count == 0 + and count in (0, 1) ): fast_substitute = getattr(self._pattern, "_substitute_python_fast", None) if fast_substitute is not None: - direct_result = fast_substitute(subject, repl) + direct_result = ( + fast_substitute(subject, repl) + if count == 0 + else fast_substitute(subject, repl, count) + ) if direct_result is not NotImplemented: return direct_result @@ -1424,7 +1430,7 @@ def subn( and type(string) in (str, bytes) and type(repl) is type(string) and type(count) is int - and count == 0 + and count in (0, 1) and compiled._is_c_pattern and ( (type(repl) is str and "\\" not in repl and "$" not in repl) @@ -1433,7 +1439,9 @@ def subn( ): fast = getattr(compiled._pattern, "_substitute_fast", None) if fast is not None: - return fast(string, repl) + if count == 0: + return fast(string, repl) + return fast(string, repl, count) return compiled.subn(repl, string, count=count) diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index 08b7d42..15adcbe 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -3966,7 +3966,7 @@ Pattern_substitute(PatternObject *self, int match_data_from_pattern = 0; int match_context_from_pattern = 0; - if (count != 0) { + if (count != 0 && count != 1) { Py_RETURN_NOTIMPLEMENTED; } @@ -4072,10 +4072,12 @@ Pattern_substitute(PatternObject *self, pcre2_jit_stack_assign(match_context, NULL, jit_stack); } - uint32_t sub_options = PCRE2_SUBSTITUTE_GLOBAL - | PCRE2_SUBSTITUTE_EXTENDED + uint32_t sub_options = PCRE2_SUBSTITUTE_EXTENDED | PCRE2_SUBSTITUTE_UNSET_EMPTY | PCRE2_SUBSTITUTE_OVERFLOW_LENGTH; + if (count == 0) { + sub_options |= PCRE2_SUBSTITUTE_GLOBAL; + } if (!subject_is_bytes) { sub_options |= PCRE2_NO_UTF_CHECK; } @@ -4536,13 +4538,20 @@ Pattern_findall_fast(PatternObject *self, PyObject *const *args, Py_ssize_t narg static PyObject * Pattern_substitute_fast(PatternObject *self, PyObject *const *args, Py_ssize_t nargs) { - if (nargs != 2) { + if (nargs != 2 && nargs != 3) { PyErr_Format(PyExc_TypeError, - "_substitute_fast() takes exactly 2 positional arguments (%zd given)", + "_substitute_fast() takes 2 or 3 positional arguments (%zd given)", nargs); return NULL; } - return Pattern_substitute(self, args[0], args[1], 0); + Py_ssize_t count = 0; + if (nargs == 3) { + count = PyLong_AsSsize_t(args[2]); + if (count == -1 && PyErr_Occurred()) { + return NULL; + } + } + return Pattern_substitute(self, args[0], args[1], count); } static int @@ -4760,12 +4769,19 @@ Pattern_substitute_python_fast(PatternObject *self, PyObject *const *args, Py_ssize_t nargs) { - if (nargs != 2) { + if (nargs != 2 && nargs != 3) { PyErr_Format(PyExc_TypeError, - "_substitute_python_fast() takes exactly 2 positional arguments (%zd given)", + "_substitute_python_fast() takes 2 or 3 positional arguments (%zd given)", nargs); return NULL; } + Py_ssize_t count = 0; + if (nargs == 3) { + count = PyLong_AsSsize_t(args[2]); + if (count == -1 && PyErr_Occurred()) { + return NULL; + } + } int handled = 0; PyObject *replacement = pattern_translate_single_replacement( self, args[1], &handled @@ -4776,7 +4792,7 @@ Pattern_substitute_python_fast(PatternObject *self, } Py_RETURN_NOTIMPLEMENTED; } - PyObject *result = Pattern_substitute(self, args[0], replacement, 0); + PyObject *result = Pattern_substitute(self, args[0], replacement, count); Py_DECREF(replacement); return result; } diff --git a/tests/test_sub_count_one_fastpath.py b/tests/test_sub_count_one_fastpath.py new file mode 100644 index 0000000..bf82ca1 --- /dev/null +++ b/tests/test_sub_count_one_fastpath.py @@ -0,0 +1,77 @@ +import concurrent.futures +import re + +import pytest + +import pcre + + +@pytest.mark.parametrize( + "pattern,replacement,subject", + [ + (r"(x)", "[X]", "xxxx"), + (r"(x)", r"[\1]", "xxxx"), + (r"(x)", r"[\g<1>]", "xxxx"), + (r"(?Px)", r"[\g]", "xxxx"), + (rb"(x)", b"[X]", b"xxxx"), + (rb"(x)", rb"[\1]", b"xxxx"), + (rb"(x)", rb"[\g<1>]", b"xxxx"), + (rb"(?Px)", rb"[\g]", b"xxxx"), + (r"(x)", "replacement", "yyyy"), + (r"", "-", "ab"), + ], +) +def test_count_one_substitution_matches_stdlib(pattern, replacement, subject): + expected = re.compile(pattern) + actual = pcre.compile(pattern) + + assert actual.sub(replacement, subject, count=1) == expected.sub( + replacement, subject, count=1 + ) + assert actual.subn(replacement, subject, count=1) == expected.subn( + replacement, subject, count=1 + ) + assert pcre.sub(pattern, replacement, subject, count=1) == re.sub( + pattern, replacement, subject, count=1 + ) + assert pcre.subn(pattern, replacement, subject, count=1) == re.subn( + pattern, replacement, subject, count=1 + ) + + +def test_count_two_still_uses_compatible_bounded_path(): + pattern = pcre.compile(r"(x)") + assert pattern.sub(r"[\1]", "xxxx", count=2) == re.sub( + r"(x)", r"[\1]", "xxxx", count=2 + ) + + +def test_count_one_invalid_template_still_raises(): + pattern = pcre.compile(r"(x)") + with pytest.raises(pcre.PcreError): + pattern.sub(r"\q", "xxxx", count=1) + + +def test_count_one_preserves_duplicate_name_resolution(): + pattern = pcre.compile( + r"(?J)(?x)|(?y)", pcre.Flag.DUPNAMES | pcre.Flag.NO_JIT + ) + assert pattern.sub(r"[\g]", "yy", count=1) == "[y]y" + + +def test_low_level_single_substitution_is_bounded(): + pattern = pcre.compile(r"(x)")._pattern + assert pattern.substitute("xxxx", "[X]", 1) == ("[X]xxx", 1) + assert pattern.substitute("yyyy", "[X]", 1) == ("yyyy", 0) + + +def test_count_one_shared_pattern_is_thread_safe(): + pattern = pcre.compile(r"(?Px)") + + def exercise(_: int): + return pattern.subn(r"[\g]", "xxxx", count=1) + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(exercise, range(256))) + + assert results == [("[x]xxx", 1)] * 256 diff --git a/tests/test_sub_reference_fastpath.py b/tests/test_sub_reference_fastpath.py index 2a5b0c9..3cbf3df 100644 --- a/tests/test_sub_reference_fastpath.py +++ b/tests/test_sub_reference_fastpath.py @@ -67,7 +67,7 @@ def counted(*args): assert calls == 1 -def test_bounded_count_and_replacement_subclass_stay_compatible( +def test_single_count_is_call_local_and_replacement_subclass_stays_compatible( monkeypatch: pytest.MonkeyPatch, ) -> None: class Replacement(str): @@ -85,7 +85,7 @@ def counted(*args): monkeypatch.setattr(pcre_mod, "_cached_replacement_parts", counted) assert compiled.sub(r"[\1]", "aa", count=1) == "[a]a" assert compiled.sub(Replacement(r"[\1]"), "a") == "[a]" - assert calls == 1 + assert calls == 0 def test_single_reference_subn_is_safe_on_shared_pattern() -> None: From 9b4743b732353a228d01c31a58c04d856b61226e Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 19:39:29 +0800 Subject: [PATCH 11/21] Accelerate small bounded substitutions --- README.md | 8 +- benchmarks/api_hotpaths.py | 2 + pcre/pcre.py | 6 +- pcre_ext/pcre2.c | 155 +++++++++++++++++++++++++-- tests/test_clobber.py | 5 +- tests/test_sub_count_one_fastpath.py | 83 +++++++++++++- 6 files changed, 242 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index fa36b6b..add1a6a 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,10 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 -* 08/10/2026 **Native count-one substitution**: exact `Pattern.sub`/`subn` calls with `count=1` now stay in PCRE2 for literal, numeric, explicit numeric, and named replacements instead of rebuilding the bounded result through a Python match loop. Pinned A/B measurements improve the four bound forms by **4.2–4.7x** on Python 3.10 and **4.1–4.3x** on free-threaded Python 3.14t/GIL=0; module-level forms improve by **2.4–3.2x**. Translation is call-local and never grows the replacement-template cache; `count>=2`, ambiguous templates, subclasses, mutable buffers, and callables retain the compatibility path. ⚡🛡️ +* 08/10/2026 **Call-local bounded substitution**: exact `Pattern.sub`/`subn` calls with counts from 2 through 8 now stay in PCRE2 for literal, numeric, explicit numeric, and named replacements. A stack-local substitute callout stops after the requested accepted replacement, is cleared before its match context can be reused, and uses a compact output buffer that grows geometrically only toward a strict linear ceiling. Pinned A/B measurements improve these bound forms by **4.5–8.6x** on Python 3.10 and **4.1–6.5x** on free-threaded Python 3.14t/GIL=0. The path retains no callback/template state and does not grow the replacement cache; count 9+, multiple/ambiguous references, subclasses, buffers, and callables remain on the compatibility loop. ⚡🛡️ +* 08/10/2026 **Native count-one substitution**: exact `Pattern.sub`/`subn` calls with `count=1` now stay in PCRE2 for literal, numeric, explicit numeric, and named replacements instead of rebuilding the bounded result through a Python match loop. Pinned A/B measurements improve the four bound forms by **4.2–4.7x** on Python 3.10 and **4.1–4.3x** on free-threaded Python 3.14t/GIL=0; module-level forms improve by **2.4–3.2x**. Translation is call-local and never grows the replacement-template cache; count 9+, ambiguous templates, subclasses, mutable buffers, and callables retain the compatibility path. ⚡🛡️ * 08/10/2026 **Stateless `escape` fast path**: exact immutable text and bytes now use a native `re.escape`-compatible scanner with no cache, retained parsing state, or cross-thread ownership. Pinned A/B measurements against the previous Python wrapper improve short no-op text by **5.0x/3.6x** on Python 3.10/3.14t and no-op bytes by **6.9x/6.5x**; short escaped punctuation improves by **3.6x/3.1x**. Mutable buffers and subclasses continue through stdlib coercion/dynamic dispatch, and exhaustive byte plus randomized Unicode parity checks cover the native path. ⚡🛡️ -* 08/10/2026 **Call-local single-reference substitution**: exact default-count `Pattern.sub`/`subn` replacements containing one valid numeric, explicit numeric, or named capture now translate and execute entirely in C without entering or growing the thread-local replacement-template cache. Pinned A/B measurements improve short numeric and explicit forms by roughly **1.9–2.1x** on Python 3.10 and free-threaded Python 3.14t/GIL=0; named replacement improves by **10.9x/1.9x**. Duplicate PCRE names select the participating capture, while `$`, multiple/ambiguous references, subclasses, and bounded counts retain the compatibility parser. ⚡🛡️ +* 08/10/2026 **Call-local single-reference substitution**: exact default-count `Pattern.sub`/`subn` replacements containing one valid numeric, explicit numeric, or named capture now translate and execute entirely in C without entering or growing the thread-local replacement-template cache. Pinned A/B measurements improve short numeric and explicit forms by roughly **1.9–2.1x** on Python 3.10 and free-threaded Python 3.14t/GIL=0; named replacement improves by **10.9x/1.9x**. Duplicate PCRE names select the participating capture, while `$`, multiple/ambiguous references, subclasses, and counts above eight retain the compatibility parser. ⚡🛡️ * 08/10/2026 **Call-local `Match.expand` fast paths**: exact text and bytes templates containing up to eight unambiguous capture/backslash tokens now render directly from the immutable C Match snapshot, without importing the compatibility parser or retaining a parsed-template cache entry. Pinned A/B measurements against merged `main` improve matched `[\\1]` expansion by **20.8x** on Python 3.10 and **16.7x** on free-threaded Python 3.14t/GIL=0; unmatched captures reach **25.5x/20.7x**, and bytes reach **37.4x/31.7x**. Explicit numeric `[\\g<1>]` improves by **14.3x/13.8x**, checked multi-digit references such as `\\g<12>` reach **63.3x/13.1x**, named `[\\g]` improves by **28.7x/10.6x**, and a two-name template reaches **45.3x/9.6x**. Three references reach **9.2x/8.2x**, while eight reach **6.3x/6.3x**. A literal backslash plus named capture reaches **13.9x/11.7x**, and a named capture with a backslash suffix reaches **42.9x/9.2x**. Duplicate-name alternatives select the participating capture. Nine-or-more tokens, non-backslash escapes, ambiguous two-digit, subclass, invalid, and non-ASCII-name templates continue through the fully compatible parser. ⚡🛡️ * 08/10/2026 **Literal split/substitution/findall fast paths**: exact plain-literal `Pattern.split` calls now use the immutable built-in splitter after construction-time validation, measuring **2.1x** faster than the prior C dispatch on Python 3.10 and **1.7x** faster on free-threaded Python 3.14t/GIL=0; delimiter-heavy multi-character literals reach roughly **4.8x**. Literal `Pattern.subn` and module-level `sub`/`subn` now use native replace/count primitives, reaching about **15x** on short repeated tokens and **3x** on delimiter-heavy text. Literal `findall` uses non-overlapping native count/list construction, reaching about **9x** on short repeated tokens and **8x** on delimiter-heavy text. Regex metacharacters, explicit flags, subclasses, and buffer subjects remain on the compatibility-safe PCRE2 path. ⚡ * 08/09/2026 **API hot-path update**: large `parallel_map(findall)` workloads now reach **11.5x** speedup on Python 3.10 and **11.25x** on free-threaded Python 3.14t/GIL=0 with 12 performance-tier workers. Ordered `parallel_map(search)` reaches **8.57x** and **7.85x**, respectively; one-item and up to eight tiny explicit `parallel_map` subjects now avoid executor setup (the one-item case measures **13.3x** faster on Python 3.10 and **27.7x** on 3.14t), default bound `Pattern.split` is another **1.6x/1.5x** faster on 3.10/3.14t, and default bound literal `Pattern.subn` is about **1.5x** faster on Python 3.10. Canonical module helpers retain their optimized dispatch while their wrapper/template caches are thread-scoped, size-bounded, and invalidated across live workers. Repeated backreference `Match.expand()` avoids reparsing within the active cache context, while captured values returned by `Match.groups()` remain call-local so a long-lived Match does not retain an additional copy of large captures. 🧵⚡ @@ -89,6 +90,9 @@ hard CPU affinity. | Bound numeric-reference `sub(..., count=1)` | **4.3x** | **4.1x** | | Bound explicit-reference `sub(..., count=1)` | **4.6x** | **4.3x** | | Bound named-reference `sub(..., count=1)` | **4.2x** | **4.3x** | +| Bound numeric-reference `sub(..., count=2)` | **4.6x** | **4.2x** | +| Bound numeric-reference `sub(..., count=4)` | **6.0x** | **5.1x** | +| Bound numeric-reference `sub(..., count=8)` | **8.6x** | **6.1x** | | Bound one-character literal `Pattern.split` | **2.1x** | **1.7x** | | Bound backreference `sub` hot path | **1.38 μs** | **1.14 μs** | | One-match numeric-reference `Pattern.sub` | **0.45 μs** | **0.34 μs** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index 8edafeb..a292266 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -61,8 +61,10 @@ def main() -> int: ("bound.split", lambda: pattern.split("x " * 8)), ("bound.sub.literal", lambda: pattern.sub("[X]", short_subject)), ("bound.sub.literal1", lambda: pattern.sub("[X]", short_subject, count=1)), + ("bound.sub.literal4", lambda: pattern.sub("[X]", short_subject, count=4)), ("bound.sub.backref", lambda: pattern.sub(r"[\1]", short_subject)), ("bound.sub.backref1", lambda: pattern.sub(r"[\1]", short_subject, count=1)), + ("bound.sub.backref4", lambda: pattern.sub(r"[\1]", short_subject, count=4)), ("bound.sub.explicit", lambda: pattern.sub(r"[\g<1>]", short_subject)), ( "bound.sub.named", diff --git a/pcre/pcre.py b/pcre/pcre.py index cace1aa..f3d2731 100644 --- a/pcre/pcre.py +++ b/pcre/pcre.py @@ -771,7 +771,7 @@ def subn(self, repl: Any, subject: Any, count: Any = 0) -> tuple[Any, int]: and type(subject) in (str, bytes) and type(repl) is type(subject) and type(count) is int - and count in (0, 1) + and 0 <= count <= 8 and ("\\" not in repl if type(repl) is str else b"\\" not in repl) and ("$" not in repl if type(repl) is str else b"$" not in repl) ): @@ -790,7 +790,7 @@ def subn(self, repl: Any, subject: Any, count: Any = 0) -> tuple[Any, int]: and type(subject) in (str, bytes) and type(repl) is type(subject) and type(count) is int - and count in (0, 1) + and 0 <= count <= 8 ): fast_substitute = getattr(self._pattern, "_substitute_python_fast", None) if fast_substitute is not None: @@ -1430,7 +1430,7 @@ def subn( and type(string) in (str, bytes) and type(repl) is type(string) and type(count) is int - and count in (0, 1) + and 0 <= count <= 8 and compiled._is_c_pattern and ( (type(repl) is str and "\\" not in repl and "$" not in repl) diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index 15adcbe..a122080 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -3943,11 +3943,34 @@ Pattern_findall_method(PatternObject *self, PyObject *args, PyObject *kwargs) return Pattern_findall(self, subject, pos, endpos, options); } +typedef struct { + uint32_t limit; + int stopped; +} SubstituteLimitState; + +enum { + SUBSTITUTE_REPLACEMENT_GENERAL = 0, + SUBSTITUTE_REPLACEMENT_LITERAL = 1, + SUBSTITUTE_REPLACEMENT_SINGLE_REFERENCE = 2, +}; + +static int PCRE2_CALL_CONVENTION +bounded_substitute_callout(pcre2_substitute_callout_block *block, void *data) +{ + SubstituteLimitState *state = (SubstituteLimitState *)data; + if (block->subscount > state->limit) { + state->stopped = 1; + return -1; + } + return 0; +} + static PyObject * Pattern_substitute(PatternObject *self, PyObject *subject_obj, PyObject *repl_obj, - Py_ssize_t count) + Py_ssize_t count, + int replacement_shape) { PyObject *result = NULL; PyObject *result_tuple = NULL; @@ -3965,8 +3988,10 @@ Pattern_substitute(PatternObject *self, pcre2_jit_stack *jit_stack = NULL; int match_data_from_pattern = 0; int match_context_from_pattern = 0; + int substitute_callout_installed = 0; - if (count != 0 && count != 1) { + if (count < 0 || count > 8 || + (count > 1 && replacement_shape == SUBSTITUTE_REPLACEMENT_GENERAL)) { Py_RETURN_NOTIMPLEMENTED; } @@ -4058,12 +4083,30 @@ Pattern_substitute(PatternObject *self, goto error; } - if (pattern_jit_get(self)) { + if (pattern_jit_get(self) || count > 1) { match_context = pattern_match_context_acquire(self, 0, &match_context_from_pattern); if (match_context == NULL) { PyErr_NoMemory(); goto error; } + } + + SubstituteLimitState limit_state = {(uint32_t)count, 0}; + + if (count > 1) { + int callout_rc = pcre2_set_substitute_callout( + match_context, + bounded_substitute_callout, + &limit_state + ); + if (callout_rc < 0) { + raise_pcre_error("set_substitute_callout", callout_rc, 0); + goto error; + } + substitute_callout_installed = 1; + } + + if (pattern_jit_get(self)) { jit_stack = jit_stack_cache_acquire(); if (jit_stack == NULL) { PyErr_NoMemory(); @@ -4075,7 +4118,7 @@ Pattern_substitute(PatternObject *self, uint32_t sub_options = PCRE2_SUBSTITUTE_EXTENDED | PCRE2_SUBSTITUTE_UNSET_EMPTY | PCRE2_SUBSTITUTE_OVERFLOW_LENGTH; - if (count == 0) { + if (count != 1) { sub_options |= PCRE2_SUBSTITUTE_GLOBAL; } if (!subject_is_bytes) { @@ -4088,7 +4131,44 @@ Pattern_substitute(PatternObject *self, goto error; } PCRE2_SIZE initial_outlen = (PCRE2_SIZE)(subject_length + repl_length + 16); + PCRE2_SIZE bounded_max_outlen = 0; + if (count > 1) { + if (repl_length > (PY_SSIZE_T_MAX - subject_length - 16) / count) { + PyErr_NoMemory(); + goto error; + } + initial_outlen = (PCRE2_SIZE)( + subject_length + count * repl_length + 16 + ); + + if (replacement_shape == SUBSTITUTE_REPLACEMENT_LITERAL) { + bounded_max_outlen = initial_outlen; + } else { + /* Exactly one capture can contribute at most one whole subject + * per accepted replacement. The first allocation uses the usual + * compact bound; only a genuine expansion overflow grows + * geometrically toward this strict linear ceiling. */ + if (subject_length > PY_SSIZE_T_MAX - repl_length) { + PyErr_NoMemory(); + goto error; + } + Py_ssize_t per_replacement = subject_length + repl_length; + if (per_replacement > + (PY_SSIZE_T_MAX - subject_length - 16) / count) { + PyErr_NoMemory(); + goto error; + } + bounded_max_outlen = (PCRE2_SIZE)( + subject_length + count * per_replacement + 16 + ); + } + if (bounded_max_outlen < initial_outlen) { + PyErr_NoMemory(); + goto error; + } + } PCRE2_SIZE outlen = initial_outlen; + PCRE2_SIZE out_capacity = initial_outlen; PCRE2_UCHAR *out = (PCRE2_UCHAR *)PyMem_Malloc(outlen); if (out == NULL) { PyErr_NoMemory(); @@ -4096,6 +4176,7 @@ Pattern_substitute(PatternObject *self, } for (int attempts = 0; attempts < 5; ++attempts) { + limit_state.stopped = 0; int rc = pcre2_substitute(self->code, (PCRE2_SPTR)subject_data, (PCRE2_SIZE)subject_length, @@ -4109,7 +4190,16 @@ Pattern_substitute(PatternObject *self, &outlen); if (rc == PCRE2_ERROR_NOMEMORY) { PCRE2_SIZE required = outlen; - if (required == (PCRE2_SIZE)-1) { + if (count > 1) { + if (out_capacity >= bounded_max_outlen) { + PyMem_Free(out); + PyErr_NoMemory(); + goto error; + } + required = out_capacity <= bounded_max_outlen / 2 + ? out_capacity * 2 + : bounded_max_outlen; + } else if (required == (PCRE2_SIZE)-1) { if ((PCRE2_SIZE)subject_length > (PCRE2_SIZE)PY_SSIZE_T_MAX - initial_outlen) { PyMem_Free(out); PyErr_NoMemory(); @@ -4132,6 +4222,7 @@ Pattern_substitute(PatternObject *self, goto error; } out = (PCRE2_UCHAR *)new_out; + out_capacity = required; outlen = required; continue; } @@ -4141,6 +4232,11 @@ Pattern_substitute(PatternObject *self, raise_pcre_error("substitute", rc, error_offset); goto error; } + if (count > 1 && limit_state.stopped && rc > 0) { + /* PCRE2 includes the rejected stopping match in its return value; + * public subn() counts accepted replacements only. */ + rc -= 1; + } PyObject *out_obj = NULL; if (subject_is_bytes) { @@ -4184,6 +4280,16 @@ Pattern_substitute(PatternObject *self, result = NULL; cleanup: + if (substitute_callout_installed && match_context != NULL) { + int clear_rc = pcre2_set_substitute_callout(match_context, NULL, NULL); + if (clear_rc < 0) { + /* Never publish a context that could retain a pointer to the + * stack-local limit state, even if a future PCRE2 build reports a + * failure while clearing the callout. */ + pcre2_match_context_free(match_context); + match_context = NULL; + } + } if (jit_stack != NULL) { if (match_context != NULL) { pcre2_jit_stack_assign(match_context, NULL, NULL); @@ -4213,7 +4319,13 @@ Pattern_substitute_method(PatternObject *self, PyObject *args, PyObject *kwargs) return NULL; } - return Pattern_substitute(self, subject, replacement, count); + return Pattern_substitute( + self, + subject, + replacement, + count, + SUBSTITUTE_REPLACEMENT_GENERAL + ); } static PyObject * @@ -4551,7 +4663,28 @@ Pattern_substitute_fast(PatternObject *self, PyObject *const *args, Py_ssize_t n return NULL; } } - return Pattern_substitute(self, args[0], args[1], count); + int replacement_shape = SUBSTITUTE_REPLACEMENT_GENERAL; + if (count > 1) { + if (PyBytes_CheckExact(args[1])) { + const char *data = PyBytes_AS_STRING(args[1]); + Py_ssize_t length = PyBytes_GET_SIZE(args[1]); + if (memchr(data, '\\', (size_t)length) == NULL && + memchr(data, '$', (size_t)length) == NULL) { + replacement_shape = SUBSTITUTE_REPLACEMENT_LITERAL; + } + } else if (PyUnicode_CheckExact(args[1]) && + PyUnicode_FindChar( + args[1], '\\', 0, PyUnicode_GET_LENGTH(args[1]), 1 + ) < 0 && + PyUnicode_FindChar( + args[1], '$', 0, PyUnicode_GET_LENGTH(args[1]), 1 + ) < 0) { + replacement_shape = SUBSTITUTE_REPLACEMENT_LITERAL; + } + } + return Pattern_substitute( + self, args[0], args[1], count, replacement_shape + ); } static int @@ -4792,7 +4925,13 @@ Pattern_substitute_python_fast(PatternObject *self, } Py_RETURN_NOTIMPLEMENTED; } - PyObject *result = Pattern_substitute(self, args[0], replacement, count); + PyObject *result = Pattern_substitute( + self, + args[0], + replacement, + count, + SUBSTITUTE_REPLACEMENT_SINGLE_REFERENCE + ); Py_DECREF(replacement); return result; } diff --git a/tests/test_clobber.py b/tests/test_clobber.py index a6e4231..bbd7ee2 100644 --- a/tests/test_clobber.py +++ b/tests/test_clobber.py @@ -109,6 +109,9 @@ def _system_seed() -> int: + configured = os.getenv("PYPCRE_CLOBBER_SEED") + if configured is not None: + return int(configured, 0) return int.from_bytes(os.urandom(16), "little") @@ -299,7 +302,7 @@ def _exercise_pattern( def test_randomized_clobbering_ci_fuzz() -> None: seed = _system_seed() - print(f'[test_clobber] seed={seed}') + print(f"[test_clobber] seed={seed}", flush=True) rng = random.Random(seed) deadline = time.monotonic() + _RUN_DURATION_SECONDS iterations = 0 diff --git a/tests/test_sub_count_one_fastpath.py b/tests/test_sub_count_one_fastpath.py index bf82ca1..c4c2e47 100644 --- a/tests/test_sub_count_one_fastpath.py +++ b/tests/test_sub_count_one_fastpath.py @@ -39,13 +39,48 @@ def test_count_one_substitution_matches_stdlib(pattern, replacement, subject): ) -def test_count_two_still_uses_compatible_bounded_path(): +@pytest.mark.parametrize("count", [2, 4, 8]) +def test_small_bounded_counts_use_compatible_native_path(count): pattern = pcre.compile(r"(x)") - assert pattern.sub(r"[\1]", "xxxx", count=2) == re.sub( - r"(x)", r"[\1]", "xxxx", count=2 + assert pattern.sub(r"[\1]", "x" * 12, count=count) == re.sub( + r"(x)", r"[\1]", "x" * 12, count=count ) +@pytest.mark.parametrize( + "pattern,replacement,subject,count", + [ + (r"(?=(.*))", r"[\1]", "abcd", 2), + (r"(x)?", r"[\1]", "y", 4), + (r"(?Pé)", r"[\g]", "éééé", 8), + (rb"(?=(.*))", rb"[\1]", b"abcd", 4), + ], +) +def test_small_bounded_edge_cases_match_stdlib(pattern, replacement, subject, count): + assert pcre.subn(pattern, replacement, subject, count=count) == re.subn( + pattern, replacement, subject, count=count + ) + + +def test_count_nine_stays_on_compatible_python_path(monkeypatch): + from pcre import pcre as pcre_module + + pattern = pcre.compile(r"(x)") + original = pcre_module._cached_replacement_parts + calls = 0 + + def counted(*args): + nonlocal calls + calls += 1 + return original(*args) + + monkeypatch.setattr(pcre_module, "_cached_replacement_parts", counted) + assert pattern.sub(r"[\1]", "x" * 12, count=9) == re.sub( + r"(x)", r"[\1]", "x" * 12, count=9 + ) + assert calls == 1 + + def test_count_one_invalid_template_still_raises(): pattern = pcre.compile(r"(x)") with pytest.raises(pcre.PcreError): @@ -63,6 +98,34 @@ def test_low_level_single_substitution_is_bounded(): pattern = pcre.compile(r"(x)")._pattern assert pattern.substitute("xxxx", "[X]", 1) == ("[X]xxx", 1) assert pattern.substitute("yyyy", "[X]", 1) == ("yyyy", 0) + assert pattern.substitute("xxxx", "[X]", 2) is NotImplemented + + +def test_bounded_reference_expansion_retries_with_linear_memory_bound(): + subject = "x" * 4096 + pattern = pcre.compile(r"(?=(.*))") + expected = re.sub(r"(?=(.*))", r"[\1]", subject, count=8) + assert pattern.sub(r"[\1]", subject, count=8) == expected + + +def test_bounded_callout_is_cleared_before_context_reuse(): + pattern = pcre.compile(r"(x)") + assert pattern.sub(r"[\1]", "x" * 12, count=2) == "[x][x]" + "x" * 10 + assert pattern.sub(r"[\1]", "x" * 12) == "[x]" * 12 + assert pattern.sub(r"[\1]", "x" * 12, count=4) == "[x]" * 4 + "x" * 8 + + +def test_small_bounded_references_do_not_grow_template_cache(): + from pcre import pcre as pcre_module + + pcre.clear_cache() + pattern = pcre.compile(r"(x)") + before = pcre_module._replacement_cache_size() + for count in range(2, 9): + assert pattern.sub(r"[\1]", "x" * 12, count=count) == re.sub( + r"(x)", r"[\1]", "x" * 12, count=count + ) + assert pcre_module._replacement_cache_size() == before def test_count_one_shared_pattern_is_thread_safe(): @@ -75,3 +138,17 @@ def exercise(_: int): results = list(executor.map(exercise, range(256))) assert results == [("[x]xxx", 1)] * 256 + + +def test_small_bounded_callout_is_thread_safe_on_shared_pattern(): + pattern = pcre.compile(r"(?Px)") + + def exercise(index: int): + count = 2 + index % 7 + return count, pattern.subn(r"[\g]", "x" * 12, count=count) + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(exercise, range(512))) + + for count, result in results: + assert result == ("[x]" * count + "x" * (12 - count), count) From 2e06c53e2ca07742a2bade39dc6d2a2a832e299c Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 19:53:15 +0800 Subject: [PATCH 12/21] Accelerate stdlib flag compilation --- README.md | 3 + benchmarks/api_hotpaths.py | 3 + pcre/pcre.py | 38 ++++++++-- tests/test_stdlib_flag_fastpath.py | 111 +++++++++++++++++++++++++++++ 4 files changed, 150 insertions(+), 5 deletions(-) create mode 100644 tests/test_stdlib_flag_fastpath.py diff --git a/README.md b/README.md index add1a6a..7dfda32 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 +* 08/10/2026 **Stateless stdlib-flag dispatch**: exact `compile(pattern, re.RegexFlag)` calls now translate the finite stdlib bitset with plain integer probes and go directly to the existing bounded thread-local pattern cache. Pinned A/B measurements improve cached `re.I`, `re.I|re.M`, and `re.I|re.M|re.S|re.X` compilation by **6.2–6.7x** on Python 3.10 and **6.4–6.9x** on free-threaded Python 3.14t/GIL=0. All supported combinations are exhaustively checked for text and bytes, unsupported bits still raise, and the path adds no cache, retained flag object, or cross-thread state. ⚡🛡️ * 08/10/2026 **Call-local bounded substitution**: exact `Pattern.sub`/`subn` calls with counts from 2 through 8 now stay in PCRE2 for literal, numeric, explicit numeric, and named replacements. A stack-local substitute callout stops after the requested accepted replacement, is cleared before its match context can be reused, and uses a compact output buffer that grows geometrically only toward a strict linear ceiling. Pinned A/B measurements improve these bound forms by **4.5–8.6x** on Python 3.10 and **4.1–6.5x** on free-threaded Python 3.14t/GIL=0. The path retains no callback/template state and does not grow the replacement cache; count 9+, multiple/ambiguous references, subclasses, buffers, and callables remain on the compatibility loop. ⚡🛡️ * 08/10/2026 **Native count-one substitution**: exact `Pattern.sub`/`subn` calls with `count=1` now stay in PCRE2 for literal, numeric, explicit numeric, and named replacements instead of rebuilding the bounded result through a Python match loop. Pinned A/B measurements improve the four bound forms by **4.2–4.7x** on Python 3.10 and **4.1–4.3x** on free-threaded Python 3.14t/GIL=0; module-level forms improve by **2.4–3.2x**. Translation is call-local and never grows the replacement-template cache; count 9+, ambiguous templates, subclasses, mutable buffers, and callables retain the compatibility path. ⚡🛡️ * 08/10/2026 **Stateless `escape` fast path**: exact immutable text and bytes now use a native `re.escape`-compatible scanner with no cache, retained parsing state, or cross-thread ownership. Pinned A/B measurements against the previous Python wrapper improve short no-op text by **5.0x/3.6x** on Python 3.10/3.14t and no-op bytes by **6.9x/6.5x**; short escaped punctuation improves by **3.6x/3.1x**. Mutable buffers and subclasses continue through stdlib coercion/dynamic dispatch, and exhaustive byte plus randomized Unicode parity checks cover the native path. ⚡🛡️ @@ -93,6 +94,8 @@ hard CPU affinity. | Bound numeric-reference `sub(..., count=2)` | **4.6x** | **4.2x** | | Bound numeric-reference `sub(..., count=4)` | **6.0x** | **5.1x** | | Bound numeric-reference `sub(..., count=8)` | **8.6x** | **6.1x** | +| Cached compile with `re.I` | **6.7x** | **6.9x** | +| Cached compile with `re.I|re.M|re.S|re.X` | **6.2x** | **6.4x** | | Bound one-character literal `Pattern.split` | **2.1x** | **1.7x** | | Bound backreference `sub` hot path | **1.38 μs** | **1.14 μs** | | One-match numeric-reference `Pattern.sub` | **0.45 μs** | **0.34 μs** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index a292266..cabd973 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -17,6 +17,7 @@ import concurrent.futures import os +import re import statistics import sys import time @@ -46,6 +47,7 @@ def main() -> int: captured = pattern.match(short_subject) named_captured = pcre.compile("(?Px)").match(short_subject) multi_captured = pcre.compile("(?Px)(?Px)").match(short_subject) + stdlib_flags = re.I | re.M | re.S | re.X if captured is None: raise AssertionError("benchmark pattern failed to produce a match") if named_captured is None: @@ -74,6 +76,7 @@ def main() -> int: ("module.escape.text", lambda: pcre.escape("identifier_123")), ("module.escape.bytes", lambda: pcre.escape(b"identifier123")), ("module.escape.special", lambda: pcre.escape("a+b [c]")), + ("module.compile.reflags", lambda: pcre.compile("(x)", stdlib_flags)), ("match.expand.numeric", lambda: captured.expand(r"[\1]")), ("match.expand.explicit", lambda: captured.expand(r"[\g<1>]")), ("match.expand.named", lambda: named_captured.expand(r"[\g]")), diff --git a/pcre/pcre.py b/pcre/pcre.py index f3d2731..9271379 100644 --- a/pcre/pcre.py +++ b/pcre/pcre.py @@ -174,9 +174,15 @@ def _extract_jit_override(flags: int) -> bool | None: _std_re.RegexFlag.VERBOSE: _pcre2.PCRE2_EXTENDED, } +# Keep the hot coercion loop on plain integers. ``RegexFlag.__and__`` creates +# a new IntFlag object for every probe, which dominates cached compile calls. +_STD_RE_FLAG_PAIRS: tuple[tuple[int, int], ...] = tuple( + (int(flag), native_value) for flag, native_value in _STD_RE_FLAG_MAP.items() +) + _STD_RE_FLAG_MASK = 0 -for _flag in _STD_RE_FLAG_MAP: - _STD_RE_FLAG_MASK |= int(_flag) +for _flag_value, _native_value in _STD_RE_FLAG_PAIRS: + _STD_RE_FLAG_MASK |= _flag_value def _convert_regex_compat(pattern: str) -> str: @@ -205,7 +211,8 @@ def _apply_default_unicode_flags(pattern: Any, flags: int) -> int: def _coerce_stdlib_regexflag(flag: _std_re.RegexFlag) -> int: - unsupported_bits = int(flag) & ~( + flag_value = int(flag) + unsupported_bits = flag_value & ~( _STD_RE_FLAG_MASK | RE_TEMPLATE_FLAG | RE_UNICODE_FLAG ) if unsupported_bits: @@ -215,8 +222,8 @@ def _coerce_stdlib_regexflag(flag: _std_re.RegexFlag) -> int: ) resolved = 0 - for std_flag, native_value in _STD_RE_FLAG_MAP.items(): - if flag & std_flag: + for std_flag_value, native_value in _STD_RE_FLAG_PAIRS: + if flag_value & std_flag_value: resolved |= native_value return resolved @@ -1162,6 +1169,27 @@ def compile(pattern: Any, flags: FlagInput = 0) -> Pattern: ) return _policy_wrapper(compiled, thread_mode) + # Exact built-in patterns with stdlib RegexFlag values have no PyPcre-only + # thread/JIT markers. Translate their small finite bitset once and go + # directly to the existing bounded, thread-local flagged cache. Other + # inputs keep the fully dynamic normalization path below. + if ( + isinstance(flags, _std_re.RegexFlag) + and type(pattern) in (str, bytes) + and cached_compile is _ORIGINAL_CACHED_COMPILE + ): + resolved_stdlib_flags = _coerce_stdlib_regexflag(flags) + thread_mode = ( + _THREAD_MODE_AUTO if get_thread_default() else _THREAD_MODE_DISABLED + ) + return _compile_flagged_builtin( + pattern, + resolved_stdlib_flags, + bool(_DEFAULT_JIT), + bool(_DEFAULT_COMPAT_REGEX), + thread_mode, + ) + resolved_flags = _normalise_flags(flags) threads_requested = bool(resolved_flags & THREADS) no_threads_requested = bool(resolved_flags & NO_THREADS) diff --git a/tests/test_stdlib_flag_fastpath.py b/tests/test_stdlib_flag_fastpath.py new file mode 100644 index 0000000..e16cdc7 --- /dev/null +++ b/tests/test_stdlib_flag_fastpath.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import concurrent.futures +import itertools +import re + +import pcre_ext_c +import pytest + +import pcre +from pcre import pcre as pcre_module + +_SUPPORTED_STDLIB_FLAGS = tuple( + flag + for flag in ( + getattr(re.RegexFlag, "TEMPLATE", None), + re.RegexFlag.IGNORECASE, + re.RegexFlag.MULTILINE, + re.RegexFlag.DOTALL, + re.RegexFlag.UNICODE, + re.RegexFlag.VERBOSE, + ) + if flag is not None +) + + +@pytest.mark.parametrize("is_bytes", [False, True]) +def test_every_supported_stdlib_regexflag_combination_is_exact(is_bytes: bool): + source = b"a.b" if is_bytes else "a.b" + default_flags = ( + 0 + if is_bytes + else ( + pcre_ext_c.PCRE2_UTF + | pcre_ext_c.PCRE2_UCP + | int(pcre.Flag.NEVER_BACKSLASH_C) + ) + ) + native_by_stdlib = { + re.RegexFlag.IGNORECASE: pcre_ext_c.PCRE2_CASELESS, + re.RegexFlag.MULTILINE: pcre_ext_c.PCRE2_MULTILINE, + re.RegexFlag.DOTALL: pcre_ext_c.PCRE2_DOTALL, + re.RegexFlag.VERBOSE: pcre_ext_c.PCRE2_EXTENDED, + } + + for enabled in itertools.product( + (False, True), repeat=len(_SUPPORTED_STDLIB_FLAGS) + ): + flags = re.RegexFlag(0) + expected = default_flags + for include, stdlib_flag in zip(enabled, _SUPPORTED_STDLIB_FLAGS): + if include: + flags |= stdlib_flag + expected |= native_by_stdlib.get(stdlib_flag, 0) + + assert pcre.compile(source, flags).flags == expected + + +@pytest.mark.parametrize( + "unsupported", + [ + re.RegexFlag.ASCII, + re.RegexFlag.DEBUG, + re.RegexFlag.LOCALE, + re.RegexFlag.IGNORECASE | re.RegexFlag.ASCII, + re.RegexFlag.MULTILINE | re.RegexFlag.DEBUG, + ], +) +def test_direct_stdlib_regexflag_path_rejects_unsupported_bits(unsupported): + with pytest.raises(ValueError, match="Unsupported stdlib re flag"): + pcre.compile("a", unsupported) + + +def test_direct_stdlib_regexflag_path_uses_bounded_thread_local_cache(): + original_limit = pcre.get_cache_limit() + try: + pcre.set_cache_limit(2) + for index in range(12): + pcre.compile(f"pattern-{index}", re.RegexFlag.IGNORECASE) + assert len(pcre_module._DEFAULT_COMPILE_LOCAL.flagged_cache) <= 2 + finally: + pcre.set_cache_limit(original_limit) + pcre.clear_cache() + + +def test_stdlib_regexflag_fast_path_is_safe_across_worker_threads(): + flags = re.RegexFlag.IGNORECASE | re.RegexFlag.MULTILINE | re.RegexFlag.DOTALL + + def exercise(index: int): + pattern = pcre.compile(r"^a.(?Pb)$", flags) + match = pattern.search("ignored\nA\nb") + return index, pattern.flags, None if match is None else match.group("tail") + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(exercise, range(256))) + + expected_native = ( + pcre_ext_c.PCRE2_UTF + | pcre_ext_c.PCRE2_UCP + | int(pcre.Flag.NEVER_BACKSLASH_C) + | pcre_ext_c.PCRE2_CASELESS + | pcre_ext_c.PCRE2_MULTILINE + | pcre_ext_c.PCRE2_DOTALL + ) + assert results == [(index, expected_native, "b") for index in range(256)] + + +def test_template_retains_deprecation_and_template_flag_semantics(): + with pytest.warns(DeprecationWarning, match="deprecated"): + compiled = pcre.template(r"(?P\w+)") + assert compiled.fullmatch("hello") is not None From 9e7230debdf7a47cfd6b5c79be242aca34746731 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 20:05:56 +0800 Subject: [PATCH 13/21] Accelerate exact lastindex resolution --- README.md | 2 + benchmarks/api_hotpaths.py | 1 + pcre_ext/pcre2.c | 41 +++++++++++++ tests/test_lastindex_ovector_fastpath.py | 78 ++++++++++++++++++++++++ 4 files changed, 122 insertions(+) create mode 100644 tests/test_lastindex_ovector_fastpath.py diff --git a/README.md b/README.md index 7dfda32..9ceb98c 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 +* 08/10/2026 **Exact ovector `lastindex` shortcut**: a Match with zero or one participating capture now derives `lastindex`/`lastgroup` directly from its immutable ovector; only matches with multiple participating captures pay for the exact AUTO_CALLOUT ordering replay. Pinned measurements reduce the first-read `lastindex` portion by **11–15x** on Python 3.10 and **8–16x** on free-threaded Python 3.14t/GIL=0, while preserving nested/lookaround/duplicate-name semantics. The shortcut adds no object field, replay code, cache entry, or retained capture value; concurrent first publication still uses the Match critical section. ⚡🛡️ * 08/10/2026 **Stateless stdlib-flag dispatch**: exact `compile(pattern, re.RegexFlag)` calls now translate the finite stdlib bitset with plain integer probes and go directly to the existing bounded thread-local pattern cache. Pinned A/B measurements improve cached `re.I`, `re.I|re.M`, and `re.I|re.M|re.S|re.X` compilation by **6.2–6.7x** on Python 3.10 and **6.4–6.9x** on free-threaded Python 3.14t/GIL=0. All supported combinations are exhaustively checked for text and bytes, unsupported bits still raise, and the path adds no cache, retained flag object, or cross-thread state. ⚡🛡️ * 08/10/2026 **Call-local bounded substitution**: exact `Pattern.sub`/`subn` calls with counts from 2 through 8 now stay in PCRE2 for literal, numeric, explicit numeric, and named replacements. A stack-local substitute callout stops after the requested accepted replacement, is cleared before its match context can be reused, and uses a compact output buffer that grows geometrically only toward a strict linear ceiling. Pinned A/B measurements improve these bound forms by **4.5–8.6x** on Python 3.10 and **4.1–6.5x** on free-threaded Python 3.14t/GIL=0. The path retains no callback/template state and does not grow the replacement cache; count 9+, multiple/ambiguous references, subclasses, buffers, and callables remain on the compatibility loop. ⚡🛡️ * 08/10/2026 **Native count-one substitution**: exact `Pattern.sub`/`subn` calls with `count=1` now stay in PCRE2 for literal, numeric, explicit numeric, and named replacements instead of rebuilding the bounded result through a Python match loop. Pinned A/B measurements improve the four bound forms by **4.2–4.7x** on Python 3.10 and **4.1–4.3x** on free-threaded Python 3.14t/GIL=0; module-level forms improve by **2.4–3.2x**. Translation is call-local and never grows the replacement-template cache; count 9+, ambiguous templates, subclasses, mutable buffers, and callables retain the compatibility path. ⚡🛡️ @@ -96,6 +97,7 @@ hard CPU affinity. | Bound numeric-reference `sub(..., count=8)` | **8.6x** | **6.1x** | | Cached compile with `re.I` | **6.7x** | **6.9x** | | Cached compile with `re.I|re.M|re.S|re.X` | **6.2x** | **6.4x** | +| First-read `lastindex` cost, sole capture | **11.8x** | **8.2x** | | Bound one-character literal `Pattern.split` | **2.1x** | **1.7x** | | Bound backreference `sub` hot path | **1.38 μs** | **1.14 μs** | | One-match numeric-reference `Pattern.sub` | **0.45 μs** | **0.34 μs** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index cabd973..7498613 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -73,6 +73,7 @@ def main() -> int: lambda: named_pattern.sub(r"[\g]", short_subject), ), ("match.groups", captured.groups), + ("match.first_lastindex", lambda: pattern.match("x").lastindex), ("module.escape.text", lambda: pcre.escape("identifier_123")), ("module.escape.bytes", lambda: pcre.escape(b"identifier123")), ("module.escape.special", lambda: pcre.escape("a+b [c]")), diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index a122080..8e946cc 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -943,6 +943,47 @@ Match_get_lastindex(MatchObject *self, void *closure) Py_RETURN_NONE; } + int cached_lastindex = -2; + Py_BEGIN_CRITICAL_SECTION(self); + cached_lastindex = self->lastindex_cache; + Py_END_CRITICAL_SECTION(); + if (cached_lastindex != -2) { + if (cached_lastindex < 0) { + Py_RETURN_NONE; + } + return PyLong_FromLong(cached_lastindex); + } + + /* The expensive AUTO_CALLOUT replay is only needed to order two or more + * participating captures. With zero or one set ovector pair, the exact + * Python lastindex is already known from the immutable match snapshot. */ + int sole_participant = -1; + for (uint32_t index = 1; index < self->ovec_count; ++index) { + Py_ssize_t start = self->ovector[(size_t)index * 2]; + Py_ssize_t end = self->ovector[(size_t)index * 2 + 1]; + if (start < 0 || end < 0) { + continue; + } + if (sole_participant >= 0) { + sole_participant = -2; + break; + } + sole_participant = (int)index; + } + if (sole_participant >= -1) { + int lastindex = sole_participant; + Py_BEGIN_CRITICAL_SECTION(self); + if (self->lastindex_cache == -2) { + self->lastindex_cache = lastindex; + } + lastindex = self->lastindex_cache; + Py_END_CRITICAL_SECTION(); + if (lastindex < 0) { + Py_RETURN_NONE; + } + return PyLong_FromLong(lastindex); + } + int lastindex = -1; int replay_ok = 0; Py_BEGIN_CRITICAL_SECTION(self); diff --git a/tests/test_lastindex_ovector_fastpath.py b/tests/test_lastindex_ovector_fastpath.py new file mode 100644 index 0000000..be80329 --- /dev/null +++ b/tests/test_lastindex_ovector_fastpath.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import concurrent.futures + +import pytest + +import pcre + + +@pytest.mark.parametrize( + ("pattern", "subject", "expected_index", "expected_group"), + [ + (r"(x)", "x", 1, None), + (r"(x)?y", "y", None, None), + (r"(a)|(b)|(c)", "b", 2, None), + (r"(?:(a)|(?:(b)|(c)))", "c", 3, None), + (r"(?Px)", "x", 1, "only"), + ], +) +def test_zero_or_one_participating_capture_has_exact_lastindex( + pattern, subject, expected_index, expected_group +): + match = pcre.fullmatch(pattern, subject) + assert match is not None + assert match.lastindex == expected_index + assert match.lastgroup == expected_group + + +@pytest.mark.parametrize( + ("pattern", "subject", "expected_index", "expected_group"), + [ + (r"((a)|(b))", "a", 1, None), + (r"(?Pa(?Pb))", "ab", 1, "outer"), + (r"(?=(?Pa))(?Pa)", "a", 2, "body"), + ], +) +def test_multiple_participants_keep_exact_replay_semantics( + pattern, subject, expected_index, expected_group +): + match = pcre.fullmatch(pattern, subject) + assert match is not None + assert match.lastindex == expected_index + assert match.lastgroup == expected_group + + +def test_sole_duplicate_name_capture_resolves_lastgroup(): + pattern = pcre.compile(r"(?J)(?a)|(?b)", pcre.Flag.NO_JIT) + first = pattern.fullmatch("a") + second = pattern.fullmatch("b") + assert first is not None + assert second is not None + assert (first.lastindex, first.lastgroup) == (1, "word") + assert (second.lastindex, second.lastgroup) == (2, "word") + + +def test_shared_match_lastindex_fast_path_is_thread_safe(): + match = pcre.compile(r"(a)|(b)|(c)").fullmatch("b") + assert match is not None + + def read(_: int): + return match.lastindex, match.lastgroup + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + assert list(executor.map(read, range(1024))) == [(2, None)] * 1024 + + +def test_distinct_match_lastindex_fast_path_is_thread_safe(): + pattern = pcre.compile(r"(?Pa)|(?Pb)|(?Pc)") + subjects = ("a", "b", "c") + + def match_and_read(index: int): + match = pattern.fullmatch(subjects[index % len(subjects)]) + assert match is not None + return match.lastindex, match.lastgroup + + expected = [((index % 3) + 1, subjects[index % 3]) for index in range(768)] + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + assert list(executor.map(match_and_read, range(768))) == expected From 2e7834b240f346935348147c3179657976831b4a Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 20:20:57 +0800 Subject: [PATCH 14/21] Validate UTF bytes patterns before PCRE compile --- pcre_ext/pcre2.c | 18 ++++- tests/test_compile_utf_no_check_safety.py | 92 +++++++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 tests/test_compile_utf_no_check_safety.py diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index 8e946cc..e74f7e2 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -5164,11 +5164,24 @@ Pattern_create(PyObject *pattern_obj, uint32_t options, int jit, int jit_explici } #endif + /* Python text is guaranteed to encode as valid UTF-8, but arbitrary + * bytes are not. PCRE2_NO_UTF_CHECK makes validity a hard caller + * precondition; forwarding malformed bytes invokes undefined behavior in + * the compiler. Validate bytes patterns in PCRE2 while retaining the + * requested option in Pattern.flags after a successful compile. */ + uint32_t engine_compile_options = compile_options; + int validate_bytes_utf = is_bytes && + (compile_options & PCRE2_UTF) != 0 && + (compile_options & PCRE2_NO_UTF_CHECK) != 0; + if (validate_bytes_utf) { + engine_compile_options &= ~PCRE2_NO_UTF_CHECK; + } + int error_code; PCRE2_SIZE error_offset; pcre2_code *code = pcre2_compile((PCRE2_SPTR)PyBytes_AS_STRING(pattern_bytes), (PCRE2_SIZE)pattern_length, - compile_options, + engine_compile_options, &error_code, &error_offset, NULL); @@ -5225,6 +5238,9 @@ Pattern_create(PyObject *pattern_obj, uint32_t options, int jit, int jit_explici if (pcre2_pattern_info(code, PCRE2_INFO_ALLOPTIONS, &effective_options) == 0) { pattern->compile_options = effective_options; } + if (validate_bytes_utf) { + pattern->compile_options |= PCRE2_NO_UTF_CHECK; + } pattern->compile_options = apply_leading_inline_options( PyBytes_AS_STRING(pattern_bytes), pattern_length, diff --git a/tests/test_compile_utf_no_check_safety.py b/tests/test_compile_utf_no_check_safety.py new file mode 100644 index 0000000..3e3f97b --- /dev/null +++ b/tests/test_compile_utf_no_check_safety.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import concurrent.futures +import subprocess +import sys + +import pcre_ext_c +import pytest + +import pcre + +_NO_UTF_CHECK = int(pcre.Flag.NO_UTF_CHECK) +_UTF_NO_CHECK = int(pcre.Flag.UTF | pcre.Flag.NO_UTF_CHECK) +_INVALID_UTF8_PATTERNS = ( + b"\xff", + b"\xfe", + b"\x80", + b"\xc0\xaf", + b"\xe2\x82", + b"\xf0\x80\x80\x80", + b"a\xed\xa0\x80z", +) + + +@pytest.mark.parametrize("pattern", _INVALID_UTF8_PATTERNS) +def test_invalid_utf_bytes_pattern_is_checked_despite_no_utf_check(pattern): + with pytest.raises(pcre.PcreError): + pcre_ext_c.compile(pattern, _UTF_NO_CHECK, jit=False) + with pytest.raises(pcre.PcreError): + pcre.compile(pattern, pcre.Flag.UTF | pcre.Flag.NO_UTF_CHECK) + + +@pytest.mark.parametrize("pattern", _INVALID_UTF8_PATTERNS) +def test_forced_validation_preserves_precise_pcre_error(pattern): + with pytest.raises(pcre.PcreError) as checked: + pcre_ext_c.compile(pattern, int(pcre.Flag.UTF), jit=False) + with pytest.raises(pcre.PcreError) as guarded: + pcre_ext_c.compile(pattern, _UTF_NO_CHECK, jit=False) + assert guarded.value.code == checked.value.code + assert guarded.value.offset == checked.value.offset + + +def test_valid_utf_bytes_pattern_preserves_requested_flag_and_behavior(): + source = "(?Pé+)".encode() + pattern = pcre_ext_c.compile(source, _UTF_NO_CHECK, jit=False) + match = pattern.fullmatch("éé".encode()) + assert match is not None + assert match.group("word") == "éé".encode() + assert pattern.flags & _NO_UTF_CHECK + + +def test_invalid_utf_no_check_compile_is_safe_in_subprocess(): + script = """ +import pcre +import pcre_ext_c + +flags = int(pcre.Flag.UTF | pcre.Flag.NO_UTF_CHECK) +for _ in range(2000): + for pattern in (b"\\xff", b"\\x80", b"\\xe2\\x82", b"a\\xed\\xa0\\x80z"): + try: + pcre_ext_c.compile(pattern, flags, jit=bool(_ & 1)) + except pcre.PcreError: + pass + else: + raise AssertionError(pattern) +""" + completed = subprocess.run( + [sys.executable, "-X", "faulthandler", "-c", script], + check=False, + capture_output=True, + text=True, + ) + assert completed.returncode == 0, completed.stderr + + +def test_invalid_utf_no_check_compile_is_thread_safe(): + flags = pcre.Flag.UTF | pcre.Flag.NO_UTF_CHECK + + def exercise(worker: int): + rejected = 0 + for index in range(1000): + pattern = _INVALID_UTF8_PATTERNS[ + (worker + index) % len(_INVALID_UTF8_PATTERNS) + ] + try: + pcre.compile(pattern, flags) + except pcre.PcreError: + rejected += 1 + return rejected + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + assert list(executor.map(exercise, range(8))) == [1000] * 8 From b3141a5769679797d53319be5ca3a3a0e5182e60 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 20:21:02 +0800 Subject: [PATCH 15/21] Accelerate template compatibility dispatch --- README.md | 3 + benchmarks/api_hotpaths.py | 1 + pcre/pcre.py | 10 ++-- tests/test_template_dispatch_fastpath.py | 74 ++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 tests/test_template_dispatch_fastpath.py diff --git a/README.md b/README.md index 9ceb98c..cbb4a69 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 +* 08/10/2026 **UTF bytes compile safety**: bytes patterns can no longer bypass PCRE2's UTF validation by combining `UTF` with `NO_UTF_CHECK`. Malformed inputs previously violated a PCRE2 compiler precondition and could corrupt memory or crash concurrent free-threaded compilation; they now raise the precise PCRE error, while valid bytes preserve the requested flag and behavior. Subprocess fault tests and 8-thread invalid-pattern stress cover the boundary. The fix adds no cache, copy, or retained input and only validates calls that explicitly requested the unsafe bytes combination. 🛡️ +* 08/10/2026 **Stateless 3.10 `template()` dispatch**: the deprecated compatibility helper now reuses its imported warning module and passes the precomputed default template flag directly, while dynamic flag objects and integer subclasses retain their original `__or__` dispatch. Pinned Python 3.10 calls improve from 4.81 μs at the merged base to 0.78 μs (**6.2x**); Python 3.14, where `re.TEMPLATE` no longer exists, remains effectively flat. Every call still warns, delegates to `compile`, and adds no cache or retained input. ⚡🛡️ * 08/10/2026 **Exact ovector `lastindex` shortcut**: a Match with zero or one participating capture now derives `lastindex`/`lastgroup` directly from its immutable ovector; only matches with multiple participating captures pay for the exact AUTO_CALLOUT ordering replay. Pinned measurements reduce the first-read `lastindex` portion by **11–15x** on Python 3.10 and **8–16x** on free-threaded Python 3.14t/GIL=0, while preserving nested/lookaround/duplicate-name semantics. The shortcut adds no object field, replay code, cache entry, or retained capture value; concurrent first publication still uses the Match critical section. ⚡🛡️ * 08/10/2026 **Stateless stdlib-flag dispatch**: exact `compile(pattern, re.RegexFlag)` calls now translate the finite stdlib bitset with plain integer probes and go directly to the existing bounded thread-local pattern cache. Pinned A/B measurements improve cached `re.I`, `re.I|re.M`, and `re.I|re.M|re.S|re.X` compilation by **6.2–6.7x** on Python 3.10 and **6.4–6.9x** on free-threaded Python 3.14t/GIL=0. All supported combinations are exhaustively checked for text and bytes, unsupported bits still raise, and the path adds no cache, retained flag object, or cross-thread state. ⚡🛡️ * 08/10/2026 **Call-local bounded substitution**: exact `Pattern.sub`/`subn` calls with counts from 2 through 8 now stay in PCRE2 for literal, numeric, explicit numeric, and named replacements. A stack-local substitute callout stops after the requested accepted replacement, is cleared before its match context can be reused, and uses a compact output buffer that grows geometrically only toward a strict linear ceiling. Pinned A/B measurements improve these bound forms by **4.5–8.6x** on Python 3.10 and **4.1–6.5x** on free-threaded Python 3.14t/GIL=0. The path retains no callback/template state and does not grow the replacement cache; count 9+, multiple/ambiguous references, subclasses, buffers, and callables remain on the compatibility loop. ⚡🛡️ @@ -98,6 +100,7 @@ hard CPU affinity. | Cached compile with `re.I` | **6.7x** | **6.9x** | | Cached compile with `re.I|re.M|re.S|re.X` | **6.2x** | **6.4x** | | First-read `lastindex` cost, sole capture | **11.8x** | **8.2x** | +| Deprecated `template()` compatibility call | **6.2x** | **1.1x** | | Bound one-character literal `Pattern.split` | **2.1x** | **1.7x** | | Bound backreference `sub` hot path | **1.38 μs** | **1.14 μs** | | One-match numeric-reference `Pattern.sub` | **0.45 μs** | **0.34 μs** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index 7498613..8f01593 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -78,6 +78,7 @@ def main() -> int: ("module.escape.bytes", lambda: pcre.escape(b"identifier123")), ("module.escape.special", lambda: pcre.escape("a+b [c]")), ("module.compile.reflags", lambda: pcre.compile("(x)", stdlib_flags)), + ("module.template", lambda: pcre.template("(x)")), ("match.expand.numeric", lambda: captured.expand(r"[\1]")), ("match.expand.explicit", lambda: captured.expand(r"[\g<1>]")), ("match.expand.named", lambda: named_captured.expand(r"[\g]")), diff --git a/pcre/pcre.py b/pcre/pcre.py index 9271379..5afb7de 100644 --- a/pcre/pcre.py +++ b/pcre/pcre.py @@ -8,6 +8,7 @@ from __future__ import annotations import re as _std_re +import warnings as _warnings from collections.abc import Iterable, Iterator, Mapping from functools import lru_cache from threading import local @@ -1476,16 +1477,17 @@ def subn( # add this function to bypass signatures unit test # re.template() is deprecated and removed since python 3.12 def template(pattern, flags=0): - import warnings - - warnings.warn( + _warnings.warn( "The re.template() function is deprecated " "as it is an undocumented function " "without an obvious purpose. " "Use re.compile() instead.", DeprecationWarning, ) - return compile(pattern, flags | RE_TEMPLATE) + template_flags = ( + RE_TEMPLATE if type(flags) is int and flags == 0 else flags | RE_TEMPLATE + ) + return compile(pattern, template_flags) _PARALLEL_EXEC_METHODS = frozenset({"match", "search", "fullmatch", "findall"}) diff --git a/tests/test_template_dispatch_fastpath.py b/tests/test_template_dispatch_fastpath.py new file mode 100644 index 0000000..7ef0efa --- /dev/null +++ b/tests/test_template_dispatch_fastpath.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import re +import warnings + +import pytest + +from pcre import pcre as pcre_module + + +def test_default_template_dispatch_passes_precomputed_flag(monkeypatch): + sentinel = object() + calls = [] + monkeypatch.setattr(pcre_module, "RE_TEMPLATE", sentinel) + monkeypatch.setattr( + pcre_module, + "compile", + lambda pattern, flags: calls.append((pattern, flags)) or "compiled", + ) + + with pytest.warns(DeprecationWarning, match="deprecated"): + assert pcre_module.template("pattern") == "compiled" + assert calls == [("pattern", sentinel)] + + +def test_dynamic_template_flags_keep_original_or_dispatch(monkeypatch): + sentinel = object() + calls = [] + + class DynamicFlags: + def __or__(self, other): + calls.append((self, other)) + return sentinel + + flags = DynamicFlags() + monkeypatch.setattr(pcre_module, "RE_TEMPLATE", re.RegexFlag.UNICODE) + monkeypatch.setattr( + pcre_module, + "compile", + lambda pattern, combined: (pattern, combined), + ) + + with pytest.warns(DeprecationWarning, match="deprecated"): + assert pcre_module.template("pattern", flags) == ("pattern", sentinel) + assert calls == [(flags, re.RegexFlag.UNICODE)] + + +def test_int_subclass_template_flags_keep_dynamic_or_dispatch(monkeypatch): + calls = [] + + class DynamicZero(int): + def __or__(self, other): + calls.append(other) + return 123 + + monkeypatch.setattr(pcre_module, "RE_TEMPLATE", re.RegexFlag.UNICODE) + monkeypatch.setattr( + pcre_module, + "compile", + lambda pattern, combined: (pattern, combined), + ) + + with pytest.warns(DeprecationWarning, match="deprecated"): + assert pcre_module.template("pattern", DynamicZero()) == ("pattern", 123) + assert calls == [re.RegexFlag.UNICODE] + + +def test_template_warns_on_every_call(): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", DeprecationWarning) + pcre_module.template("a") + pcre_module.template("a") + assert len(caught) == 2 + assert all(item.category is DeprecationWarning for item in caught) From 2bd7e88f6dbb044558825cc4a7553ca948f513ab Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 20:30:03 +0800 Subject: [PATCH 16/21] Accelerate literal capture findall --- README.md | 3 + benchmarks/api_hotpaths.py | 5 + pcre/pcre.py | 33 ++++- .../test_literal_capture_findall_fastpath.py | 123 ++++++++++++++++++ 4 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 tests/test_literal_capture_findall_fastpath.py diff --git a/README.md b/README.md index cbb4a69..663fe3f 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 +* 08/10/2026 **Bounded literal-capture `findall`**: exact default-option patterns containing one plain capture such as `(token)` now use immutable `str`/`bytes.count` plus list construction. Pinned delimiter-heavy measurements improve 100/500 captures by **5.0x/7.0x** on Python 3.10 and **6.2x/8.4x** on free-threaded Python 3.14t/GIL=0. The per-Pattern literal snapshot is capped at 64 code units, lives only as long as that Pattern (including its already bounded thread-local cache entry), and never retains subjects or results; metacharacters, options, subclasses, and non-default ranges keep the native PCRE2 path. 20,000 randomized parity cases and shared-pattern stress cover text/bytes behavior. ⚡🛡️ * 08/10/2026 **UTF bytes compile safety**: bytes patterns can no longer bypass PCRE2's UTF validation by combining `UTF` with `NO_UTF_CHECK`. Malformed inputs previously violated a PCRE2 compiler precondition and could corrupt memory or crash concurrent free-threaded compilation; they now raise the precise PCRE error, while valid bytes preserve the requested flag and behavior. Subprocess fault tests and 8-thread invalid-pattern stress cover the boundary. The fix adds no cache, copy, or retained input and only validates calls that explicitly requested the unsafe bytes combination. 🛡️ * 08/10/2026 **Stateless 3.10 `template()` dispatch**: the deprecated compatibility helper now reuses its imported warning module and passes the precomputed default template flag directly, while dynamic flag objects and integer subclasses retain their original `__or__` dispatch. Pinned Python 3.10 calls improve from 4.81 μs at the merged base to 0.78 μs (**6.2x**); Python 3.14, where `re.TEMPLATE` no longer exists, remains effectively flat. Every call still warns, delegates to `compile`, and adds no cache or retained input. ⚡🛡️ * 08/10/2026 **Exact ovector `lastindex` shortcut**: a Match with zero or one participating capture now derives `lastindex`/`lastgroup` directly from its immutable ovector; only matches with multiple participating captures pay for the exact AUTO_CALLOUT ordering replay. Pinned measurements reduce the first-read `lastindex` portion by **11–15x** on Python 3.10 and **8–16x** on free-threaded Python 3.14t/GIL=0, while preserving nested/lookaround/duplicate-name semantics. The shortcut adds no object field, replay code, cache entry, or retained capture value; concurrent first publication still uses the Match critical section. ⚡🛡️ @@ -101,6 +102,8 @@ hard CPU affinity. | Cached compile with `re.I|re.M|re.S|re.X` | **6.2x** | **6.4x** | | First-read `lastindex` cost, sole capture | **11.8x** | **8.2x** | | Deprecated `template()` compatibility call | **6.2x** | **1.1x** | +| Literal-capture `findall`, 100 matches | **5.0x** | **6.2x** | +| Literal-capture `findall`, 500 matches | **7.0x** | **8.4x** | | Bound one-character literal `Pattern.split` | **2.1x** | **1.7x** | | Bound backreference `sub` hot path | **1.38 μs** | **1.14 μs** | | One-match numeric-reference `Pattern.sub` | **0.45 μs** | **0.34 μs** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index 8f01593..3736fd6 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -43,6 +43,7 @@ def main() -> int: subject = "x" * 1000 short_subject = "x" * 10 pattern = pcre.compile("(x)") + literal_capture_pattern = pcre.compile("(token)") named_pattern = pcre.compile("(?Px)") captured = pattern.match(short_subject) named_captured = pcre.compile("(?Px)").match(short_subject) @@ -59,6 +60,10 @@ def main() -> int: ("bound.search", lambda: pattern.search(subject)), ("bound.fullmatch", lambda: pattern.fullmatch(subject)), ("bound.findall", lambda: pattern.findall(short_subject)), + ( + "bound.findall.capture", + lambda: literal_capture_pattern.findall("token," * 100), + ), ("bound.finditer", lambda: list(pattern.finditer(short_subject))), ("bound.split", lambda: pattern.split("x " * 8)), ("bound.sub.literal", lambda: pattern.sub("[X]", short_subject)), diff --git a/pcre/pcre.py b/pcre/pcre.py index 5afb7de..140a287 100644 --- a/pcre/pcre.py +++ b/pcre/pcre.py @@ -306,6 +306,7 @@ class Pattern: "_groups_hint", "_thread_mode", "_is_c_pattern", + "_literal_findall", "_literal_split", ) @@ -319,6 +320,7 @@ def __init__(self, pattern: _CPattern) -> None: self._groups_hint = maybe_infer_group_count(pattern.pattern) literal_split: str | bytes | None = None + literal_findall: str | bytes | None = None if self._is_c_pattern: source = pattern.pattern if type(source) in (str, bytes) and source: @@ -332,12 +334,20 @@ def __init__(self, pattern: _CPattern) -> None: if type(source) is str else 0 ) - if ( - not any(char in metacharacters for char in source) - and pattern.flags == expected_flags - ): - literal_split = source + if pattern.flags == expected_flags: + if not any(char in metacharacters for char in source): + literal_split = source + if 3 <= len(source) <= 66: + opening = "(" if type(source) is str else b"(" + closing = ")" if type(source) is str else b")" + if source[:1] == opening and source[-1:] == closing: + inner = source[1:-1] + if inner and not any( + char in metacharacters for char in inner + ): + literal_findall = inner self._literal_split = literal_split + self._literal_findall = literal_findall def __repr__(self) -> str: # pragma: no cover - delegated to C repr return repr(self._pattern) @@ -574,6 +584,19 @@ def findall( ) -> List[Any]: if type(subject) is memoryview: subject = subject.tobytes() + literal_capture = getattr(self, "_literal_findall", None) + if ( + getattr(self, "_is_c_pattern", False) + and type(self) is Pattern + and literal_capture is not None + and type(subject) is type(literal_capture) + and type(pos) is int + and pos == 0 + and endpos is None + and type(options) is int + and options == 0 + ): + return [literal_capture] * subject.count(literal_capture) literal_source = getattr(self, "_literal_split", None) if ( getattr(self, "_is_c_pattern", False) diff --git a/tests/test_literal_capture_findall_fastpath.py b/tests/test_literal_capture_findall_fastpath.py new file mode 100644 index 0000000..afc6b70 --- /dev/null +++ b/tests/test_literal_capture_findall_fastpath.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import concurrent.futures +import re + +import pytest + +import pcre +from pcre import pcre as pcre_module + + +@pytest.mark.parametrize( + ("literal", "subject"), + [ + ("x", "xx xy x"), + ("token", "token,token;not-token"), + ("é", "é é éé"), + ("雪", "雪雨雪"), + ("a-b", "a-ba-b--a-b"), + (b"x", b"xx xy x"), + (b"token", b"token,token;not-token"), + ("é".encode(), "é e éé".encode()), + ], +) +def test_plain_capture_findall_matches_stdlib(literal, subject): + opening = "(" if isinstance(literal, str) else b"(" + closing = ")" if isinstance(literal, str) else b")" + pattern = opening + literal + closing + assert pcre.compile(pattern).findall(subject) == re.compile(pattern).findall( + subject + ) + + +@pytest.mark.parametrize( + "pattern", + [ + r"()", + r"(x+)", + r"(x|y)", + r"(\.)", + r"((x))", + r"(?Px)", + rb"()", + rb"(x+)", + rb"(x|y)", + rb"(\.)", + ], +) +def test_nonliteral_capture_shapes_keep_native_findall(pattern): + compiled = pcre.compile(pattern) + assert compiled._literal_findall is None + subject = b"x.xxy" if isinstance(pattern, bytes) else "x.xxy" + assert compiled.findall(subject) == re.compile(pattern).findall(subject) + + +def test_literal_capture_metadata_has_strict_size_bound(): + accepted = "x" * 64 + rejected = "x" * 65 + assert pcre.compile(f"({accepted})")._literal_findall == accepted + assert pcre.compile(f"({rejected})")._literal_findall is None + + +def test_literal_capture_fast_path_respects_nondefault_arguments(): + pattern = pcre.compile("(token)") + subject = "token-token-token" + assert pattern.findall(subject, pos=1) == re.compile("(token)").findall(subject, 1) + assert pattern.findall(subject, endpos=9) == re.compile("(token)").findall( + subject, 0, 9 + ) + assert pattern.findall(subject, options=pcre.Flag.NOTEMPTY) == [ + "token", + "token", + "token", + ] + assert pattern.findall(subject, pos=False) == ["token", "token", "token"] + assert pattern.findall(subject, options=False) == ["token", "token", "token"] + + +def test_literal_capture_fast_path_excludes_options_and_subclasses(): + flagged = pcre.compile("(x)", pcre.Flag.CASELESS) + assert flagged._literal_findall is None + assert flagged.findall("xX") == ["x", "X"] + + class Text(str): + pass + + plain = pcre.compile("(x)") + assert plain.findall(Text("xxx")) == ["x", "x", "x"] + + class PatternSubclass(pcre.Pattern): + pass + + wrapped = PatternSubclass(plain._pattern) + assert wrapped.findall("xxx") == ["x", "x", "x"] + + +def test_literal_capture_metadata_stays_within_bounded_pattern_cache(): + original_limit = pcre.get_cache_limit() + try: + pcre.set_cache_limit(3) + for index in range(40): + compiled = pcre.compile(f"(literal{index})") + assert compiled._literal_findall == f"literal{index}" + cache = pcre_module._DEFAULT_COMPILE_LOCAL.cache + assert len(cache) <= 3 + assert ( + sum(len(item._literal_findall or "") for item in cache.values()) <= 3 * 64 + ) + finally: + pcre.set_cache_limit(original_limit) + pcre.clear_cache() + + +def test_literal_capture_findall_is_thread_safe_on_shared_pattern(): + pattern = pcre.compile("(token)") + subjects = ["token," * (index % 32) for index in range(512)] + + def exercise(subject: str): + return pattern.findall(subject) + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(exercise, subjects)) + assert results == [["token"] * (index % 32) for index in range(512)] From 9486b12f998b40f5524c634f8be96bbb60e547e1 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 20:45:42 +0800 Subject: [PATCH 17/21] Accelerate literal capture split --- README.md | 3 + benchmarks/api_hotpaths.py | 4 + pcre/pcre.py | 12 +++ pcre_ext/pcre2.c | 76 ++++++++++++++++ tests/test_literal_capture_split_fastpath.py | 94 ++++++++++++++++++++ 5 files changed, 189 insertions(+) create mode 100644 tests/test_literal_capture_split_fastpath.py diff --git a/README.md b/README.md index 663fe3f..479a4f6 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 +* 08/10/2026 **Literal-capture `split` ownership fast path**: exact default-option patterns containing one plain capture such as `(token)` now use CPython's immutable splitter and assemble the captured-delimiter result in C. Temporary piece references transfer directly into the final list, so the path performs no duplicate substring allocations and retains no subject, result, or call-local state. Pinned 100/500/2,000-capture measurements improve by **2.6x/2.8x/3.1x** on Python 3.10 and **3.5x/3.8x/3.8x** on free-threaded Python 3.14t/GIL=0; allocation of the API-required 2N+1 output objects is now the dominant floor. All negative and bounded limits, text/bytes, mixed Unicode kinds, subclasses, and shared-Pattern concurrency are covered against `stdlib.re`; the existing literal snapshot remains capped at 64 code units within the bounded Pattern lifetime. ⚡🛡️ * 08/10/2026 **Bounded literal-capture `findall`**: exact default-option patterns containing one plain capture such as `(token)` now use immutable `str`/`bytes.count` plus list construction. Pinned delimiter-heavy measurements improve 100/500 captures by **5.0x/7.0x** on Python 3.10 and **6.2x/8.4x** on free-threaded Python 3.14t/GIL=0. The per-Pattern literal snapshot is capped at 64 code units, lives only as long as that Pattern (including its already bounded thread-local cache entry), and never retains subjects or results; metacharacters, options, subclasses, and non-default ranges keep the native PCRE2 path. 20,000 randomized parity cases and shared-pattern stress cover text/bytes behavior. ⚡🛡️ * 08/10/2026 **UTF bytes compile safety**: bytes patterns can no longer bypass PCRE2's UTF validation by combining `UTF` with `NO_UTF_CHECK`. Malformed inputs previously violated a PCRE2 compiler precondition and could corrupt memory or crash concurrent free-threaded compilation; they now raise the precise PCRE error, while valid bytes preserve the requested flag and behavior. Subprocess fault tests and 8-thread invalid-pattern stress cover the boundary. The fix adds no cache, copy, or retained input and only validates calls that explicitly requested the unsafe bytes combination. 🛡️ * 08/10/2026 **Stateless 3.10 `template()` dispatch**: the deprecated compatibility helper now reuses its imported warning module and passes the precomputed default template flag directly, while dynamic flag objects and integer subclasses retain their original `__or__` dispatch. Pinned Python 3.10 calls improve from 4.81 μs at the merged base to 0.78 μs (**6.2x**); Python 3.14, where `re.TEMPLATE` no longer exists, remains effectively flat. Every call still warns, delegates to `compile`, and adds no cache or retained input. ⚡🛡️ @@ -104,6 +105,8 @@ hard CPU affinity. | Deprecated `template()` compatibility call | **6.2x** | **1.1x** | | Literal-capture `findall`, 100 matches | **5.0x** | **6.2x** | | Literal-capture `findall`, 500 matches | **7.0x** | **8.4x** | +| Literal-capture `split`, 100 captures | **2.6x** | **3.5x** | +| Literal-capture `split`, 2,000 captures | **3.1x** | **3.8x** | | Bound one-character literal `Pattern.split` | **2.1x** | **1.7x** | | Bound backreference `sub` hot path | **1.38 μs** | **1.14 μs** | | One-match numeric-reference `Pattern.sub` | **0.45 μs** | **0.34 μs** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index 3736fd6..e087bf1 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -66,6 +66,10 @@ def main() -> int: ), ("bound.finditer", lambda: list(pattern.finditer(short_subject))), ("bound.split", lambda: pattern.split("x " * 8)), + ( + "bound.split.capture", + lambda: literal_capture_pattern.split("token," * 100), + ), ("bound.sub.literal", lambda: pattern.sub("[X]", short_subject)), ("bound.sub.literal1", lambda: pattern.sub("[X]", short_subject, count=1)), ("bound.sub.literal4", lambda: pattern.sub("[X]", short_subject, count=4)), diff --git a/pcre/pcre.py b/pcre/pcre.py index 140a287..904399d 100644 --- a/pcre/pcre.py +++ b/pcre/pcre.py @@ -679,6 +679,18 @@ def split(self, subject: Any, maxsplit: Any = 0) -> List[Any]: split_limit = -1 if maxsplit == 0 else (0 if maxsplit < 0 else maxsplit) return subject.split(self._literal_split, split_limit) + literal_capture = self._literal_findall + if ( + self._is_c_pattern + and type(self) is Pattern + and literal_capture is not None + and type(subject) is type(literal_capture) + and type(maxsplit) is int + ): + return self._pattern._split_literal_capture_fast( + subject, literal_capture, maxsplit + ) + # The common immutable/default shape can go straight to the C splitter. # Keep subclasses, buffer exporters, and non-default limits on the # compatibility path so their coercion and override semantics remain diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index e74f7e2..6c3fb2c 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -4688,6 +4688,81 @@ Pattern_findall_fast(PatternObject *self, PyObject *const *args, Py_ssize_t narg return Pattern_findall(self, args[0], 0, -1, 0); } +static PyObject * +Pattern_split_literal_capture_fast(PatternObject *self, + PyObject *const *args, + Py_ssize_t nargs) +{ + (void)self; + if (nargs != 3) { + PyErr_Format( + PyExc_TypeError, + "_split_literal_capture_fast() takes exactly 3 positional arguments (%zd given)", + nargs + ); + return NULL; + } + + int subject_is_bytes = PyBytes_CheckExact(args[0]); + if ((!subject_is_bytes && !PyUnicode_CheckExact(args[0])) || + (subject_is_bytes + ? !PyBytes_CheckExact(args[1]) + : !PyUnicode_CheckExact(args[1]))) { + Py_RETURN_NOTIMPLEMENTED; + } + + Py_ssize_t maxsplit = PyLong_AsSsize_t(args[2]); + if (maxsplit == -1 && PyErr_Occurred()) { + return NULL; + } + Py_ssize_t split_limit = maxsplit == 0 + ? -1 + : (maxsplit < 0 ? 0 : maxsplit); + + PyObject *pieces = subject_is_bytes + ? PyObject_CallMethod(args[0], "split", "On", args[1], split_limit) + : PyUnicode_Split(args[0], args[1], split_limit); + if (pieces == NULL) { + return NULL; + } + if (!PyList_CheckExact(pieces)) { + Py_DECREF(pieces); + PyErr_SetString(PyExc_RuntimeError, "built-in split returned a non-list"); + return NULL; + } + + Py_ssize_t piece_count = PyList_GET_SIZE(pieces); + if (piece_count <= 1) { + return pieces; + } + if (piece_count > PY_SSIZE_T_MAX / 2) { + Py_DECREF(pieces); + PyErr_NoMemory(); + return NULL; + } + + PyObject *result = PyList_New(piece_count * 2 - 1); + if (result == NULL) { + Py_DECREF(pieces); + return NULL; + } + for (Py_ssize_t index = 0; index < piece_count; ++index) { + PyObject *piece = PyList_GET_ITEM(pieces, index); + /* Transfer the temporary list's owned piece reference directly into + * the final list. Leave a valid singleton behind so list teardown + * never observes a NULL slot. */ + Py_INCREF(Py_None); + PyList_SET_ITEM(pieces, index, Py_None); + PyList_SET_ITEM(result, index * 2, piece); + if (index + 1 < piece_count) { + Py_INCREF(args[1]); + PyList_SET_ITEM(result, index * 2 + 1, args[1]); + } + } + Py_DECREF(pieces); + return result; +} + static PyObject * Pattern_substitute_fast(PatternObject *self, PyObject *const *args, Py_ssize_t nargs) { @@ -5067,6 +5142,7 @@ static PyMethodDef Pattern_methods[] = { {"search", (PyCFunction)Pattern_search_method, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Search the subject for the pattern." )}, {"fullmatch", (PyCFunction)Pattern_fullmatch_method, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Require the pattern to match the entire subject." )}, {"_findall_fast", (PyCFunction)(void(*)(void))Pattern_findall_fast, METH_FASTCALL, NULL}, + {"_split_literal_capture_fast", (PyCFunction)(void(*)(void))Pattern_split_literal_capture_fast, METH_FASTCALL, NULL}, {"_substitute_fast", (PyCFunction)(void(*)(void))Pattern_substitute_fast, METH_FASTCALL, NULL}, {"_substitute_python_fast", (PyCFunction)(void(*)(void))Pattern_substitute_python_fast, METH_FASTCALL, NULL}, {"_match_fast", (PyCFunction)(void(*)(void))Pattern_match_fast, METH_FASTCALL, NULL}, diff --git a/tests/test_literal_capture_split_fastpath.py b/tests/test_literal_capture_split_fastpath.py new file mode 100644 index 0000000..90ab35c --- /dev/null +++ b/tests/test_literal_capture_split_fastpath.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import concurrent.futures +import random +import re + +import pytest + +import pcre + + +@pytest.mark.parametrize( + ("literal", "subject"), + [ + ("x", "xxx"), + ("token", "tokentokentailtoken"), + ("é", "éaéé"), + ("雪", "雪雨雪雪"), + ("a-b", "a-ba-b--a-b"), + (b"x", b"xxx"), + (b"token", b"tokentokentailtoken"), + ("é".encode(), "éaéé".encode()), + ], +) +@pytest.mark.parametrize("maxsplit", [-8, -1, 0, 1, 2, 8]) +def test_literal_capture_split_matches_stdlib(literal, subject, maxsplit): + opening = "(" if isinstance(literal, str) else b"(" + closing = ")" if isinstance(literal, str) else b")" + source = opening + literal + closing + assert pcre.compile(source).split(subject, maxsplit) == re.compile(source).split( + subject, maxsplit + ) + + +@pytest.mark.parametrize("pattern", [r"()", r"(x+)", r"(x|y)", rb"()", rb"(x+)"]) +def test_nonliteral_capture_shapes_keep_native_split(pattern): + compiled = pcre.compile(pattern) + assert compiled._literal_findall is None + subject = b"xxy" if isinstance(pattern, bytes) else "xxy" + assert compiled.split(subject) == re.compile(pattern).split(subject) + + +def test_literal_capture_split_excludes_flags_and_subclasses(): + flagged = pcre.compile("(x)", pcre.Flag.CASELESS) + assert flagged._literal_findall is None + assert flagged.split("xX") == ["", "x", "", "X", ""] + + class Text(str): + pass + + plain = pcre.compile("(x)") + subject = Text("x-x") + assert plain.split(subject) == re.compile("(x)").split(subject) + + class PatternSubclass(pcre.Pattern): + pass + + wrapped = PatternSubclass(plain._pattern) + assert wrapped.split("x-x") == re.compile("(x)").split("x-x") + + +def test_literal_capture_split_private_entry_rejects_invalid_shapes(): + backend = pcre.compile("(x)")._pattern + with pytest.raises(TypeError, match="exactly 3 positional"): + backend._split_literal_capture_fast("x", "x") + assert backend._split_literal_capture_fast("x", b"x", 0) is NotImplemented + assert ( + backend._split_literal_capture_fast(bytearray(b"x"), b"x", 0) is NotImplemented + ) + with pytest.raises(OverflowError): + backend._split_literal_capture_fast("x", "x", 10**100) + + +def test_literal_capture_split_randomized_parity(): + generator = random.Random(0x5A117) + alphabet = "abcé雪" + for _ in range(2_000): + literal = "".join(generator.choices(alphabet, k=generator.randrange(1, 9))) + subject = "".join(generator.choices(alphabet, k=generator.randrange(65))) + maxsplit = generator.randrange(-3, 9) + source = f"({literal})" + assert pcre.compile(source).split(subject, maxsplit) == re.compile( + source + ).split(subject, maxsplit) + + +def test_literal_capture_split_is_thread_safe_on_shared_pattern(): + pattern = pcre.compile("(token)") + subjects = ["token," * (index % 32) for index in range(512)] + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(pattern.split, subjects)) + + assert results == [re.compile("(token)").split(subject) for subject in subjects] From ce1da24a1ac56740e1c1dfe3317fdf19bccdcfce Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 21:01:19 +0800 Subject: [PATCH 18/21] Accelerate adjacent literal capture findall --- README.md | 4 + benchmarks/api_hotpaths.py | 5 + pcre/pcre.py | 44 +++++- ..._multi_literal_capture_findall_fastpath.py | 144 ++++++++++++++++++ 4 files changed, 192 insertions(+), 5 deletions(-) create mode 100644 tests/test_multi_literal_capture_findall_fastpath.py diff --git a/README.md b/README.md index 479a4f6..3a68733 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 +* 08/10/2026 **Adjacent literal-capture `findall`**: exact default-option patterns made entirely from two through eight adjacent plain captures, such as `(token)(-)(id)`, now use one immutable literal count and the prevalidated capture tuple. Pinned two-capture measurements improve 100/500 matches by **7.5x/13.5x** on Python 3.10 and **10.8x/18.3x** on free-threaded Python 3.14t/GIL=0; eight captures reach **14.9x/24.5x** and **19.5x/29.4x**, respectively. The Pattern-local descriptor is capped at eight groups and 64 total literal units, lives only within the already bounded Pattern/cache lifetime, and never retains subjects or results. Metacharacters, intervening text, options, subclasses, ranges, and ninth-or-later groups keep the native PCRE2 path; 3,000 randomized parity cases and shared-Pattern stress cover Unicode, bytes, limits, and thread safety. ⚡🛡️ * 08/10/2026 **Literal-capture `split` ownership fast path**: exact default-option patterns containing one plain capture such as `(token)` now use CPython's immutable splitter and assemble the captured-delimiter result in C. Temporary piece references transfer directly into the final list, so the path performs no duplicate substring allocations and retains no subject, result, or call-local state. Pinned 100/500/2,000-capture measurements improve by **2.6x/2.8x/3.1x** on Python 3.10 and **3.5x/3.8x/3.8x** on free-threaded Python 3.14t/GIL=0; allocation of the API-required 2N+1 output objects is now the dominant floor. All negative and bounded limits, text/bytes, mixed Unicode kinds, subclasses, and shared-Pattern concurrency are covered against `stdlib.re`; the existing literal snapshot remains capped at 64 code units within the bounded Pattern lifetime. ⚡🛡️ * 08/10/2026 **Bounded literal-capture `findall`**: exact default-option patterns containing one plain capture such as `(token)` now use immutable `str`/`bytes.count` plus list construction. Pinned delimiter-heavy measurements improve 100/500 captures by **5.0x/7.0x** on Python 3.10 and **6.2x/8.4x** on free-threaded Python 3.14t/GIL=0. The per-Pattern literal snapshot is capped at 64 code units, lives only as long as that Pattern (including its already bounded thread-local cache entry), and never retains subjects or results; metacharacters, options, subclasses, and non-default ranges keep the native PCRE2 path. 20,000 randomized parity cases and shared-pattern stress cover text/bytes behavior. ⚡🛡️ * 08/10/2026 **UTF bytes compile safety**: bytes patterns can no longer bypass PCRE2's UTF validation by combining `UTF` with `NO_UTF_CHECK`. Malformed inputs previously violated a PCRE2 compiler precondition and could corrupt memory or crash concurrent free-threaded compilation; they now raise the precise PCRE error, while valid bytes preserve the requested flag and behavior. Subprocess fault tests and 8-thread invalid-pattern stress cover the boundary. The fix adds no cache, copy, or retained input and only validates calls that explicitly requested the unsafe bytes combination. 🛡️ @@ -105,6 +106,9 @@ hard CPU affinity. | Deprecated `template()` compatibility call | **6.2x** | **1.1x** | | Literal-capture `findall`, 100 matches | **5.0x** | **6.2x** | | Literal-capture `findall`, 500 matches | **7.0x** | **8.4x** | +| Two literal captures `findall`, 100 matches | **7.5x** | **10.8x** | +| Two literal captures `findall`, 500 matches | **13.5x** | **18.3x** | +| Eight literal captures `findall`, 500 matches | **24.5x** | **29.4x** | | Literal-capture `split`, 100 captures | **2.6x** | **3.5x** | | Literal-capture `split`, 2,000 captures | **3.1x** | **3.8x** | | Bound one-character literal `Pattern.split` | **2.1x** | **1.7x** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index e087bf1..81d1785 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -44,6 +44,7 @@ def main() -> int: short_subject = "x" * 10 pattern = pcre.compile("(x)") literal_capture_pattern = pcre.compile("(token)") + literal_multi_capture_pattern = pcre.compile("(token)(-)(id)") named_pattern = pcre.compile("(?Px)") captured = pattern.match(short_subject) named_captured = pcre.compile("(?Px)").match(short_subject) @@ -64,6 +65,10 @@ def main() -> int: "bound.findall.capture", lambda: literal_capture_pattern.findall("token," * 100), ), + ( + "bound.findall.multi", + lambda: literal_multi_capture_pattern.findall("token-id," * 100), + ), ("bound.finditer", lambda: list(pattern.finditer(short_subject))), ("bound.split", lambda: pattern.split("x " * 8)), ( diff --git a/pcre/pcre.py b/pcre/pcre.py index 904399d..dd42afd 100644 --- a/pcre/pcre.py +++ b/pcre/pcre.py @@ -307,6 +307,7 @@ class Pattern: "_thread_mode", "_is_c_pattern", "_literal_findall", + "_literal_findall_multi", "_literal_split", ) @@ -321,6 +322,7 @@ def __init__(self, pattern: _CPattern) -> None: literal_split: str | bytes | None = None literal_findall: str | bytes | None = None + literal_findall_multi: tuple[str | bytes, tuple[str | bytes, ...]] | None = None if self._is_c_pattern: source = pattern.pattern if type(source) in (str, bytes) and source: @@ -337,17 +339,35 @@ def __init__(self, pattern: _CPattern) -> None: if pattern.flags == expected_flags: if not any(char in metacharacters for char in source): literal_split = source - if 3 <= len(source) <= 66: + if 3 <= len(source) <= 80: opening = "(" if type(source) is str else b"(" closing = ")" if type(source) is str else b")" - if source[:1] == opening and source[-1:] == closing: - inner = source[1:-1] - if inner and not any( + literal_groups: list[str | bytes] = [] + cursor = 0 + literal_units = 0 + while cursor < len(source) and len(literal_groups) < 8: + if source[cursor : cursor + 1] != opening: + break + closing_index = source.find(closing, cursor + 1) + inner = source[cursor + 1 : closing_index] + if not inner or any( char in metacharacters for char in inner ): - literal_findall = inner + break + literal_units += len(inner) + if literal_units > 64: + break + literal_groups.append(inner) + cursor = closing_index + 1 + if cursor == len(source) and len(literal_groups) == 1: + literal_findall = literal_groups[0] + elif cursor == len(source) and len(literal_groups) >= 2: + empty = "" if type(source) is str else b"" + groups = tuple(literal_groups) + literal_findall_multi = (empty.join(groups), groups) self._literal_split = literal_split self._literal_findall = literal_findall + self._literal_findall_multi = literal_findall_multi def __repr__(self) -> str: # pragma: no cover - delegated to C repr return repr(self._pattern) @@ -597,6 +617,20 @@ def findall( and options == 0 ): return [literal_capture] * subject.count(literal_capture) + literal_multi = getattr(self, "_literal_findall_multi", None) + if ( + getattr(self, "_is_c_pattern", False) + and type(self) is Pattern + and literal_multi is not None + and type(subject) is type(literal_multi[0]) + and type(pos) is int + and pos == 0 + and endpos is None + and type(options) is int + and options == 0 + ): + needle, groups = literal_multi + return [groups] * subject.count(needle) literal_source = getattr(self, "_literal_split", None) if ( getattr(self, "_is_c_pattern", False) diff --git a/tests/test_multi_literal_capture_findall_fastpath.py b/tests/test_multi_literal_capture_findall_fastpath.py new file mode 100644 index 0000000..1b412f4 --- /dev/null +++ b/tests/test_multi_literal_capture_findall_fastpath.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import concurrent.futures +import random +import re + +import pytest + +import pcre +from pcre import pcre as pcre_module + + +@pytest.mark.parametrize( + ("groups", "subject"), + [ + (("a", "b"), "zababx"), + (("token", "-", "id"), "token-id token-id"), + (("é", "雪"), "é雪xé雪"), + (("a-b", "é", "雪"), "a-bé雪a-bé雪"), + ((b"a", b"b"), b"zababx"), + ((b"token", b"-", b"id"), b"token-id token-id"), + (("é".encode(), "雪".encode()), "é雪xé雪".encode()), + ], +) +def test_adjacent_literal_capture_findall_matches_stdlib(groups, subject): + opening = "(" if isinstance(groups[0], str) else b"(" + closing = ")" if isinstance(groups[0], str) else b")" + source = ("" if isinstance(opening, str) else b"").join( + opening + group + closing for group in groups + ) + compiled = pcre.compile(source) + assert compiled._literal_findall_multi == ( + ("" if isinstance(opening, str) else b"").join(groups), + groups, + ) + assert compiled.findall(subject) == re.compile(source).findall(subject) + + +def test_adjacent_literal_capture_metadata_has_strict_bounds(): + eight_groups = "(a)" * 8 + assert pcre.compile(eight_groups)._literal_findall_multi is not None + assert pcre.compile("(a)" * 9)._literal_findall_multi is None + + accepted = f"({'x' * 32})({'y' * 32})" + rejected = f"({'x' * 32})({'y' * 33})" + assert pcre.compile(accepted)._literal_findall_multi is not None + assert pcre.compile(rejected)._literal_findall_multi is None + + +@pytest.mark.parametrize( + "source", + [ + r"(a+)(b)", + r"(a)(b|c)", + r"((a))(b)", + r"(?Pa)(b)", + r"(a)x(b)", + r"(a)(b)$", + rb"(a+)(b)", + rb"(a)(b|c)", + ], +) +def test_nonliteral_multi_capture_shapes_keep_native_findall(source): + compiled = pcre.compile(source) + assert compiled._literal_findall_multi is None + subject = b"abacabc" if isinstance(source, bytes) else "abacabc" + assert compiled.findall(subject) == re.compile(source).findall(subject) + + +def test_multi_capture_fast_path_respects_nondefault_arguments(): + compiled = pcre.compile("(a)(b)") + stdlib = re.compile("(a)(b)") + subject = "ababab" + assert compiled.findall(subject, pos=1) == stdlib.findall(subject, 1) + assert compiled.findall(subject, endpos=5) == stdlib.findall(subject, 0, 5) + assert compiled.findall(subject, options=pcre.Flag.NOTEMPTY) == stdlib.findall( + subject + ) + assert compiled.findall(subject, pos=False) == stdlib.findall(subject) + assert compiled.findall(subject, options=False) == stdlib.findall(subject) + + +def test_multi_capture_fast_path_excludes_flags_and_subclasses(): + flagged = pcre.compile("(a)(b)", pcre.Flag.CASELESS) + assert flagged._literal_findall_multi is None + assert flagged.findall("abAB") == [("a", "b"), ("A", "B")] + + class Text(str): + pass + + plain = pcre.compile("(a)(b)") + assert plain.findall(Text("abab")) == [("a", "b"), ("a", "b")] + + class PatternSubclass(pcre.Pattern): + pass + + wrapped = PatternSubclass(plain._pattern) + assert wrapped.findall("abab") == [("a", "b"), ("a", "b")] + + +def test_multi_capture_metadata_stays_within_bounded_pattern_cache(): + original_limit = pcre.get_cache_limit() + try: + pcre.set_cache_limit(3) + for index in range(40): + compiled = pcre.compile(f"(literal{index})(suffix)") + assert compiled._literal_findall_multi is not None + cache = pcre_module._DEFAULT_COMPILE_LOCAL.cache + assert len(cache) <= 3 + retained_units = 0 + for item in cache.values(): + descriptor = item._literal_findall_multi + assert descriptor is not None + retained_units += len(descriptor[0]) + sum(map(len, descriptor[1])) + assert retained_units <= 3 * 128 + finally: + pcre.set_cache_limit(original_limit) + pcre.clear_cache() + + +def test_multi_capture_findall_randomized_parity(): + generator = random.Random(0xF1ADA11) + alphabet = "abcé雪" + for _ in range(3_000): + groups = tuple( + "".join(generator.choices(alphabet, k=generator.randrange(1, 6))) + for _ in range(generator.randrange(2, 9)) + ) + source = "".join(f"({group})" for group in groups) + subject = "".join(generator.choices(alphabet, k=generator.randrange(96))) + assert pcre.compile(source).findall(subject) == re.compile(source).findall( + subject + ) + + +def test_multi_capture_findall_is_thread_safe_on_shared_pattern(): + pattern = pcre.compile("(token)(-)(id)") + subjects = ["token-id," * (index % 32) for index in range(512)] + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(pattern.findall, subjects)) + + expected_group = ("token", "-", "id") + assert results == [[expected_group] * (index % 32) for index in range(512)] From aa4f63470f44237407b8ce061ec90f8a3d5c69e5 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 21:36:51 +0800 Subject: [PATCH 19/21] Harden UTF byte compilation and accelerate capture split --- README.md | 3 + benchmarks/api_hotpaths.py | 4 + pcre/pcre.py | 13 ++ pcre_ext/pcre2.c | 105 +++++++++++++++- tests/test_compile_utf_no_check_safety.py | 34 ++++++ ...st_multi_literal_capture_split_fastpath.py | 114 ++++++++++++++++++ 6 files changed, 270 insertions(+), 3 deletions(-) create mode 100644 tests/test_multi_literal_capture_split_fastpath.py diff --git a/README.md b/README.md index 3a68733..322148d 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Fast, free-threaded Python bindings for `PCRE2` with a stable `stdlib.re`-compat ## Latest News 🚀 +* 08/10/2026 **Adjacent literal-capture `split`**: the same bounded two-through-eight-capture descriptor now drives a call-local C splitter/assembler for exact patterns such as `(token)(-)(id)`. It allocates one final list, transfers temporary piece ownership without duplicate substring references, and inserts only immutable prevalidated captures. Pinned three-capture measurements improve 100/500 matches by **3.9x/4.5x** on Python 3.10 and **5.4x/6.5x** on free-threaded Python 3.14t/GIL=0; eight captures reach **5.4x/6.6x** and **8.2x/9.0x**, respectively. No new descriptor, cache, retained subject, or cross-call state is added; maxsplit translation, overflow arithmetic, text/bytes, subclasses, malformed private calls, 3,000 randomized cases, and shared-Pattern concurrency are covered. ⚡🛡️ * 08/10/2026 **Adjacent literal-capture `findall`**: exact default-option patterns made entirely from two through eight adjacent plain captures, such as `(token)(-)(id)`, now use one immutable literal count and the prevalidated capture tuple. Pinned two-capture measurements improve 100/500 matches by **7.5x/13.5x** on Python 3.10 and **10.8x/18.3x** on free-threaded Python 3.14t/GIL=0; eight captures reach **14.9x/24.5x** and **19.5x/29.4x**, respectively. The Pattern-local descriptor is capped at eight groups and 64 total literal units, lives only within the already bounded Pattern/cache lifetime, and never retains subjects or results. Metacharacters, intervening text, options, subclasses, ranges, and ninth-or-later groups keep the native PCRE2 path; 3,000 randomized parity cases and shared-Pattern stress cover Unicode, bytes, limits, and thread safety. ⚡🛡️ * 08/10/2026 **Literal-capture `split` ownership fast path**: exact default-option patterns containing one plain capture such as `(token)` now use CPython's immutable splitter and assemble the captured-delimiter result in C. Temporary piece references transfer directly into the final list, so the path performs no duplicate substring allocations and retains no subject, result, or call-local state. Pinned 100/500/2,000-capture measurements improve by **2.6x/2.8x/3.1x** on Python 3.10 and **3.5x/3.8x/3.8x** on free-threaded Python 3.14t/GIL=0; allocation of the API-required 2N+1 output objects is now the dominant floor. All negative and bounded limits, text/bytes, mixed Unicode kinds, subclasses, and shared-Pattern concurrency are covered against `stdlib.re`; the existing literal snapshot remains capped at 64 code units within the bounded Pattern lifetime. ⚡🛡️ * 08/10/2026 **Bounded literal-capture `findall`**: exact default-option patterns containing one plain capture such as `(token)` now use immutable `str`/`bytes.count` plus list construction. Pinned delimiter-heavy measurements improve 100/500 captures by **5.0x/7.0x** on Python 3.10 and **6.2x/8.4x** on free-threaded Python 3.14t/GIL=0. The per-Pattern literal snapshot is capped at 64 code units, lives only as long as that Pattern (including its already bounded thread-local cache entry), and never retains subjects or results; metacharacters, options, subclasses, and non-default ranges keep the native PCRE2 path. 20,000 randomized parity cases and shared-pattern stress cover text/bytes behavior. ⚡🛡️ @@ -109,6 +110,8 @@ hard CPU affinity. | Two literal captures `findall`, 100 matches | **7.5x** | **10.8x** | | Two literal captures `findall`, 500 matches | **13.5x** | **18.3x** | | Eight literal captures `findall`, 500 matches | **24.5x** | **29.4x** | +| Three literal captures `split`, 500 captures | **4.5x** | **6.5x** | +| Eight literal captures `split`, 500 captures | **6.6x** | **9.0x** | | Literal-capture `split`, 100 captures | **2.6x** | **3.5x** | | Literal-capture `split`, 2,000 captures | **3.1x** | **3.8x** | | Bound one-character literal `Pattern.split` | **2.1x** | **1.7x** | diff --git a/benchmarks/api_hotpaths.py b/benchmarks/api_hotpaths.py index 81d1785..8ff5b4d 100644 --- a/benchmarks/api_hotpaths.py +++ b/benchmarks/api_hotpaths.py @@ -75,6 +75,10 @@ def main() -> int: "bound.split.capture", lambda: literal_capture_pattern.split("token," * 100), ), + ( + "bound.split.multi", + lambda: literal_multi_capture_pattern.split("token-id," * 100), + ), ("bound.sub.literal", lambda: pattern.sub("[X]", short_subject)), ("bound.sub.literal1", lambda: pattern.sub("[X]", short_subject, count=1)), ("bound.sub.literal4", lambda: pattern.sub("[X]", short_subject, count=4)), diff --git a/pcre/pcre.py b/pcre/pcre.py index dd42afd..4109caf 100644 --- a/pcre/pcre.py +++ b/pcre/pcre.py @@ -725,6 +725,19 @@ def split(self, subject: Any, maxsplit: Any = 0) -> List[Any]: subject, literal_capture, maxsplit ) + literal_multi = self._literal_findall_multi + if ( + self._is_c_pattern + and type(self) is Pattern + and literal_multi is not None + and type(subject) is type(literal_multi[0]) + and type(maxsplit) is int + ): + needle, groups = literal_multi + return self._pattern._split_literal_captures_fast( + subject, needle, groups, maxsplit + ) + # The common immutable/default shape can go straight to the C splitter. # Keep subclasses, buffer exporters, and non-default limits on the # compatibility path so their coercion and override semantics remain diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index 6c3fb2c..a3f3c09 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -4763,6 +4763,102 @@ Pattern_split_literal_capture_fast(PatternObject *self, return result; } +static PyObject * +Pattern_split_literal_captures_fast(PatternObject *self, + PyObject *const *args, + Py_ssize_t nargs) +{ + (void)self; + if (nargs != 4) { + PyErr_Format( + PyExc_TypeError, + "_split_literal_captures_fast() takes exactly 4 positional arguments (%zd given)", + nargs + ); + return NULL; + } + + int subject_is_bytes = PyBytes_CheckExact(args[0]); + if ((!subject_is_bytes && !PyUnicode_CheckExact(args[0])) || + (subject_is_bytes + ? !PyBytes_CheckExact(args[1]) + : !PyUnicode_CheckExact(args[1])) || + !PyTuple_CheckExact(args[2])) { + Py_RETURN_NOTIMPLEMENTED; + } + + Py_ssize_t group_count = PyTuple_GET_SIZE(args[2]); + if (group_count < 2 || group_count > 8) { + Py_RETURN_NOTIMPLEMENTED; + } + for (Py_ssize_t index = 0; index < group_count; ++index) { + PyObject *group = PyTuple_GET_ITEM(args[2], index); + if (subject_is_bytes + ? !PyBytes_CheckExact(group) + : !PyUnicode_CheckExact(group)) { + Py_RETURN_NOTIMPLEMENTED; + } + } + + Py_ssize_t maxsplit = PyLong_AsSsize_t(args[3]); + if (maxsplit == -1 && PyErr_Occurred()) { + return NULL; + } + Py_ssize_t split_limit = maxsplit == 0 + ? -1 + : (maxsplit < 0 ? 0 : maxsplit); + + PyObject *pieces = subject_is_bytes + ? PyObject_CallMethod(args[0], "split", "On", args[1], split_limit) + : PyUnicode_Split(args[0], args[1], split_limit); + if (pieces == NULL) { + return NULL; + } + if (!PyList_CheckExact(pieces)) { + Py_DECREF(pieces); + PyErr_SetString(PyExc_RuntimeError, "built-in split returned a non-list"); + return NULL; + } + + Py_ssize_t piece_count = PyList_GET_SIZE(pieces); + if (piece_count <= 1) { + return pieces; + } + Py_ssize_t match_count = piece_count - 1; + if (match_count > (PY_SSIZE_T_MAX - piece_count) / group_count) { + Py_DECREF(pieces); + PyErr_NoMemory(); + return NULL; + } + + Py_ssize_t result_count = piece_count + match_count * group_count; + PyObject *result = PyList_New(result_count); + if (result == NULL) { + Py_DECREF(pieces); + return NULL; + } + Py_ssize_t output_index = 0; + for (Py_ssize_t piece_index = 0; + piece_index < piece_count; + ++piece_index) { + PyObject *piece = PyList_GET_ITEM(pieces, piece_index); + Py_INCREF(Py_None); + PyList_SET_ITEM(pieces, piece_index, Py_None); + PyList_SET_ITEM(result, output_index++, piece); + if (piece_index + 1 < piece_count) { + for (Py_ssize_t group_index = 0; + group_index < group_count; + ++group_index) { + PyObject *group = PyTuple_GET_ITEM(args[2], group_index); + Py_INCREF(group); + PyList_SET_ITEM(result, output_index++, group); + } + } + } + Py_DECREF(pieces); + return result; +} + static PyObject * Pattern_substitute_fast(PatternObject *self, PyObject *const *args, Py_ssize_t nargs) { @@ -5143,6 +5239,7 @@ static PyMethodDef Pattern_methods[] = { {"fullmatch", (PyCFunction)Pattern_fullmatch_method, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("Require the pattern to match the entire subject." )}, {"_findall_fast", (PyCFunction)(void(*)(void))Pattern_findall_fast, METH_FASTCALL, NULL}, {"_split_literal_capture_fast", (PyCFunction)(void(*)(void))Pattern_split_literal_capture_fast, METH_FASTCALL, NULL}, + {"_split_literal_captures_fast", (PyCFunction)(void(*)(void))Pattern_split_literal_captures_fast, METH_FASTCALL, NULL}, {"_substitute_fast", (PyCFunction)(void(*)(void))Pattern_substitute_fast, METH_FASTCALL, NULL}, {"_substitute_python_fast", (PyCFunction)(void(*)(void))Pattern_substitute_python_fast, METH_FASTCALL, NULL}, {"_match_fast", (PyCFunction)(void(*)(void))Pattern_match_fast, METH_FASTCALL, NULL}, @@ -5243,11 +5340,13 @@ Pattern_create(PyObject *pattern_obj, uint32_t options, int jit, int jit_explici /* Python text is guaranteed to encode as valid UTF-8, but arbitrary * bytes are not. PCRE2_NO_UTF_CHECK makes validity a hard caller * precondition; forwarding malformed bytes invokes undefined behavior in - * the compiler. Validate bytes patterns in PCRE2 while retaining the - * requested option in Pattern.flags after a successful compile. */ + * the compiler. An inline option such as (?u) can enable UTF after the + * outer options have been parsed, so checking only compile_options & UTF + * is not sufficient. Validate every bytes pattern that asks us to skip + * checks, while retaining the requested option in Pattern.flags after a + * successful compile. */ uint32_t engine_compile_options = compile_options; int validate_bytes_utf = is_bytes && - (compile_options & PCRE2_UTF) != 0 && (compile_options & PCRE2_NO_UTF_CHECK) != 0; if (validate_bytes_utf) { engine_compile_options &= ~PCRE2_NO_UTF_CHECK; diff --git a/tests/test_compile_utf_no_check_safety.py b/tests/test_compile_utf_no_check_safety.py index 3e3f97b..1b0a53d 100644 --- a/tests/test_compile_utf_no_check_safety.py +++ b/tests/test_compile_utf_no_check_safety.py @@ -22,6 +22,15 @@ ) +@pytest.mark.parametrize("suffix", _INVALID_UTF8_PATTERNS) +def test_inline_utf_bytes_pattern_is_checked_despite_no_utf_check(suffix): + pattern = b"(?u)" + suffix + with pytest.raises(pcre.PcreError): + pcre_ext_c.compile(pattern, _NO_UTF_CHECK, jit=False) + with pytest.raises(pcre.PcreError): + pcre.compile(pattern, pcre.Flag.NO_UTF_CHECK) + + @pytest.mark.parametrize("pattern", _INVALID_UTF8_PATTERNS) def test_invalid_utf_bytes_pattern_is_checked_despite_no_utf_check(pattern): with pytest.raises(pcre.PcreError): @@ -73,6 +82,31 @@ def test_invalid_utf_no_check_compile_is_safe_in_subprocess(): assert completed.returncode == 0, completed.stderr +def test_inline_utf_no_check_compile_is_safe_in_subprocess(): + script = """ +import pcre +import pcre_ext_c + +flags = int(pcre.Flag.NO_UTF_CHECK) +for _ in range(2000): + for suffix in (b"\\xff", b"\\x80", b"\\xe2\\x82", b"a\\xed\\xa0\\x80z"): + pattern = b"(?u)" + suffix + try: + pcre_ext_c.compile(pattern, flags, jit=bool(_ & 1)) + except pcre.PcreError: + pass + else: + raise AssertionError(pattern) +""" + completed = subprocess.run( + [sys.executable, "-X", "faulthandler", "-c", script], + check=False, + capture_output=True, + text=True, + ) + assert completed.returncode == 0, completed.stderr + + def test_invalid_utf_no_check_compile_is_thread_safe(): flags = pcre.Flag.UTF | pcre.Flag.NO_UTF_CHECK diff --git a/tests/test_multi_literal_capture_split_fastpath.py b/tests/test_multi_literal_capture_split_fastpath.py new file mode 100644 index 0000000..23b6d27 --- /dev/null +++ b/tests/test_multi_literal_capture_split_fastpath.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import concurrent.futures +import random +import re +import sys + +import pytest + +import pcre + + +@pytest.mark.parametrize( + ("groups", "subject"), + [ + (("a", "b"), "ababtailab"), + (("token", "-", "id"), "token-idtoken-idtail"), + (("é", "雪"), "é雪xé雪é雪"), + (("a-b", "é", "雪"), "a-bé雪a-bé雪"), + ((b"a", b"b"), b"ababtailab"), + ((b"token", b"-", b"id"), b"token-idtoken-idtail"), + (("é".encode(), "雪".encode()), "é雪xé雪é雪".encode()), + ], +) +@pytest.mark.parametrize("maxsplit", [-8, -1, 0, 1, 2, 8]) +def test_adjacent_literal_capture_split_matches_stdlib(groups, subject, maxsplit): + opening = "(" if isinstance(groups[0], str) else b"(" + closing = ")" if isinstance(groups[0], str) else b")" + empty = "" if isinstance(opening, str) else b"" + source = empty.join(opening + group + closing for group in groups) + assert pcre.compile(source).split(subject, maxsplit) == re.compile(source).split( + subject, maxsplit + ) + + +def test_multi_capture_split_excludes_flags_and_subclasses(): + flagged = pcre.compile("(a)(b)", pcre.Flag.CASELESS) + assert flagged._literal_findall_multi is None + assert flagged.split("abAB") == ["", "a", "b", "", "A", "B", ""] + + class Text(str): + pass + + plain = pcre.compile("(a)(b)") + subject = Text("abab") + assert plain.split(subject) == re.compile("(a)(b)").split(subject) + + class PatternSubclass(pcre.Pattern): + pass + + wrapped = PatternSubclass(plain._pattern) + assert wrapped.split("abab") == re.compile("(a)(b)").split("abab") + + +def test_multi_capture_split_private_entry_rejects_invalid_shapes(): + backend = pcre.compile("(a)(b)")._pattern + with pytest.raises(TypeError, match="exactly 4 positional"): + backend._split_literal_captures_fast("ab", "ab", ("a", "b")) + assert ( + backend._split_literal_captures_fast("ab", b"ab", ("a", "b"), 0) + is NotImplemented + ) + assert ( + backend._split_literal_captures_fast("ab", "ab", ["a", "b"], 0) + is NotImplemented + ) + assert backend._split_literal_captures_fast("ab", "ab", ("a",), 0) is NotImplemented + assert ( + backend._split_literal_captures_fast("ab", "ab", ("a", b"b"), 0) + is NotImplemented + ) + with pytest.raises(OverflowError): + backend._split_literal_captures_fast("ab", "ab", ("a", "b"), 10**100) + + +def test_multi_capture_split_randomized_parity(): + generator = random.Random(0x5A117A11) + alphabet = "abcé雪" + for _ in range(3_000): + groups = tuple( + "".join(generator.choices(alphabet, k=generator.randrange(1, 6))) + for _ in range(generator.randrange(2, 9)) + ) + source = "".join(f"({group})" for group in groups) + subject = "".join(generator.choices(alphabet, k=generator.randrange(96))) + maxsplit = generator.randrange(-3, 9) + assert pcre.compile(source).split(subject, maxsplit) == re.compile( + source + ).split(subject, maxsplit) + + +def test_multi_capture_split_is_thread_safe_on_shared_pattern(): + pattern = pcre.compile("(token)(-)(id)") + stdlib = re.compile("(token)(-)(id)") + subjects = ["token-id," * (index % 32) for index in range(512)] + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(pattern.split, subjects)) + + assert results == [stdlib.split(subject) for subject in subjects] + + +def test_multi_capture_split_releases_all_result_references(): + pattern = pcre.compile("(literal-one)(literal-two)(literal-three)") + descriptor = pattern._literal_findall_multi + assert descriptor is not None + groups = descriptor[1] + baseline = tuple(sys.getrefcount(group) for group in groups) + + for _ in range(2_000): + result = pattern.split("literal-oneliteral-twoliteral-three," * 16) + del result + + assert tuple(sys.getrefcount(group) for group in groups) == baseline From 2d4a21fd2c47c2cea1536c901e7abc2a038932a4 Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 21:39:21 +0800 Subject: [PATCH 20/21] Match escape argument errors to stdlib --- pcre_ext/string_helpers.c | 45 +++++++++++++++++++++++++---------- tests/test_escape_fastpath.py | 17 +++++++++++++ 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/pcre_ext/string_helpers.c b/pcre_ext/string_helpers.c index b88076e..9d2836c 100644 --- a/pcre_ext/string_helpers.c +++ b/pcre_ext/string_helpers.c @@ -147,23 +147,44 @@ module_escape(PyObject *Py_UNUSED(module), Py_ssize_t keyword_count = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames); PyObject *pattern = NULL; - if (nargs == 1 && keyword_count == 0) { - pattern = args[0]; - } else if (nargs == 0 && keyword_count == 1) { - PyObject *keyword = PyTuple_GET_ITEM(kwnames, 0); - if (PyUnicode_Check(keyword) && - PyUnicode_CompareWithASCIIString(keyword, "pattern") == 0) { - pattern = args[0]; - } else { + if (nargs > 1) { + PyErr_Format(PyExc_TypeError, + "escape() takes 1 positional argument but %zd were given", + nargs); + return NULL; + } + + for (Py_ssize_t index = 0; index < keyword_count; ++index) { + PyObject *keyword = PyTuple_GET_ITEM(kwnames, index); + if (!PyUnicode_Check(keyword) || + PyUnicode_CompareWithASCIIString(keyword, "pattern") != 0) { PyErr_Format(PyExc_TypeError, "escape() got an unexpected keyword argument '%U'", keyword); return NULL; } - } else { - PyErr_Format(PyExc_TypeError, - "escape() takes 1 argument (%zd given)", - nargs + keyword_count); + if (nargs != 0) { + PyErr_SetString(PyExc_TypeError, + "escape() got multiple values for argument 'pattern'"); + return NULL; + } + if (pattern != NULL) { + PyErr_SetString(PyExc_TypeError, + "escape() got multiple values for argument 'pattern'"); + return NULL; + } + pattern = args[nargs + index]; + } + + if (pattern == NULL) { + if (nargs == 0) { + PyErr_SetString(PyExc_TypeError, + "escape() missing 1 required positional argument: 'pattern'"); + } else { + pattern = args[0]; + } + } + if (pattern == NULL) { return NULL; } diff --git a/tests/test_escape_fastpath.py b/tests/test_escape_fastpath.py index 9daba51..54f680c 100644 --- a/tests/test_escape_fastpath.py +++ b/tests/test_escape_fastpath.py @@ -58,6 +58,23 @@ def test_escape_keyword_and_signature_match_stdlib(): assert inspect.signature(pcre.escape) == inspect.signature(re.escape) +@pytest.mark.parametrize( + ("args", "kwargs"), + [ + (("a", "b"), {}), + (("a",), {"pattern": "b"}), + ((), {}), + ((), {"unexpected": "a"}), + ], +) +def test_escape_argument_errors_match_stdlib(args, kwargs): + with pytest.raises(TypeError) as expected: + re.escape(*args, **kwargs) + with pytest.raises(TypeError) as actual: + pcre.escape(*args, **kwargs) + assert str(actual.value) == str(expected.value) + + @pytest.mark.parametrize("value", [None, 1, object()]) def test_escape_invalid_input_matches_stdlib_exception(value): with pytest.raises(TypeError): From b15a95e029fa14aef65e6318171408f0d613092c Mon Sep 17 00:00:00 2001 From: Qubitium Date: Mon, 10 Aug 2026 21:56:53 +0800 Subject: [PATCH 21/21] Cover real inline UTF compile safety --- pcre_ext/pcre2.c | 2 +- tests/test_compile_utf_no_check_safety.py | 34 +++++++++++++++++++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index a3f3c09..31bf971 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -5340,7 +5340,7 @@ Pattern_create(PyObject *pattern_obj, uint32_t options, int jit, int jit_explici /* Python text is guaranteed to encode as valid UTF-8, but arbitrary * bytes are not. PCRE2_NO_UTF_CHECK makes validity a hard caller * precondition; forwarding malformed bytes invokes undefined behavior in - * the compiler. An inline option such as (?u) can enable UTF after the + * the compiler. An inline directive such as (*UTF) can enable UTF after the * outer options have been parsed, so checking only compile_options & UTF * is not sufficient. Validate every bytes pattern that asks us to skip * checks, while retaining the requested option in Pattern.flags after a diff --git a/tests/test_compile_utf_no_check_safety.py b/tests/test_compile_utf_no_check_safety.py index 1b0a53d..c8a38f7 100644 --- a/tests/test_compile_utf_no_check_safety.py +++ b/tests/test_compile_utf_no_check_safety.py @@ -24,7 +24,7 @@ @pytest.mark.parametrize("suffix", _INVALID_UTF8_PATTERNS) def test_inline_utf_bytes_pattern_is_checked_despite_no_utf_check(suffix): - pattern = b"(?u)" + suffix + pattern = b"(*UTF)" + suffix with pytest.raises(pcre.PcreError): pcre_ext_c.compile(pattern, _NO_UTF_CHECK, jit=False) with pytest.raises(pcre.PcreError): @@ -58,6 +58,24 @@ def test_valid_utf_bytes_pattern_preserves_requested_flag_and_behavior(): assert pattern.flags & _NO_UTF_CHECK +def test_valid_inline_utf_bytes_pattern_preserves_flag_and_behavior(): + source = b"(*UTF)(?P\xc3\xa9+)" + pattern = pcre_ext_c.compile(source, _NO_UTF_CHECK, jit=False) + match = pattern.fullmatch("éé".encode()) + assert match is not None + assert match.group("word") == "éé".encode() + assert pattern.flags & _NO_UTF_CHECK + assert pattern.flags & int(pcre.Flag.UTF) + + +def test_no_utf_check_does_not_reject_arbitrary_non_utf_pattern(): + pattern = pcre_ext_c.compile(b"\xff", _NO_UTF_CHECK, jit=False) + match = pattern.fullmatch(b"\xff") + assert match is not None + assert match.group() == b"\xff" + assert pattern.flags & _NO_UTF_CHECK + + def test_invalid_utf_no_check_compile_is_safe_in_subprocess(): script = """ import pcre @@ -90,7 +108,7 @@ def test_inline_utf_no_check_compile_is_safe_in_subprocess(): flags = int(pcre.Flag.NO_UTF_CHECK) for _ in range(2000): for suffix in (b"\\xff", b"\\x80", b"\\xe2\\x82", b"a\\xed\\xa0\\x80z"): - pattern = b"(?u)" + suffix + pattern = b"(*UTF)" + suffix try: pcre_ext_c.compile(pattern, flags, jit=bool(_ & 1)) except pcre.PcreError: @@ -107,13 +125,19 @@ def test_inline_utf_no_check_compile_is_safe_in_subprocess(): assert completed.returncode == 0, completed.stderr -def test_invalid_utf_no_check_compile_is_thread_safe(): - flags = pcre.Flag.UTF | pcre.Flag.NO_UTF_CHECK +@pytest.mark.parametrize( + ("prefix", "flags"), + [ + (b"", pcre.Flag.UTF | pcre.Flag.NO_UTF_CHECK), + (b"(*UTF)", pcre.Flag.NO_UTF_CHECK), + ], +) +def test_invalid_utf_no_check_compile_is_thread_safe(prefix, flags): def exercise(worker: int): rejected = 0 for index in range(1000): - pattern = _INVALID_UTF8_PATTERNS[ + pattern = prefix + _INVALID_UTF8_PATTERNS[ (worker + index) % len(_INVALID_UTF8_PATTERNS) ] try: