From c31a6b785c036fbb3996cdace0b7a2d2bbf8deb0 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Mon, 7 Sep 2026 11:40:36 +1000 Subject: [PATCH 01/10] Implement ASTLinesAdapter class ASTLinesAdapter uses ast-produced col_offset/end_col_offset which are produced by the python's own parser to calculate convert offsets to str offset, which should be more reliable than the ad-hoc calculations that generally would only have worked with ASCII-only source code. --- rope/base/codeanalyze.py | 131 +++++++++++++++ ropetest/codeanalyzetest.py | 320 +++++++++++++++++++++++++++++++++++- 2 files changed, 450 insertions(+), 1 deletion(-) diff --git a/rope/base/codeanalyze.py b/rope/base/codeanalyze.py index 25203cce..24d20f66 100644 --- a/rope/base/codeanalyze.py +++ b/rope/base/codeanalyze.py @@ -90,6 +90,137 @@ def _clamp(self, min_value, max_value, value): return max(min_value, min(max_value, value)) +class ASTLinesAdapter: + """ + Convert between stdlib's ast `col_offset` and rope's `region` offsets. + + ast `col_offset` are given in byte offset of the utf-8 source code from + the CPython interpreter, this is not necessarily identical to the str + offset (i.e. unicode codepoint offset) that rope uses. + + Semantics are intended to match ``ast.get_source_segment`` exactly, + including its quirks: + + - Line numbers are 1-indexed (``node.lineno`` / ``node.end_lineno``), + matching the ``ast`` module. + - Column offsets are UTF-8 *byte* offsets into the line, not character + offsets -- this matters for any line containing non-ASCII characters. + - Line splitting uses the same boundaries the CPython parser uses + (``\\n``, ``\\r\\n``, ``\\r``) rather than the broader set recognized by + ``str.splitlines()`` (form feed, vertical tab, etc.), since those extra + separators don't correspond to a new line as far as ``lineno`` is + concerned. + """ + + class LineNumberOutOfRange(IndexError): + pass + + # Same pattern rule as `ast._line_pattern` used by `ast._splitlines_no_ff` + # to split lines in a way that mimics how the CPython parser splits source + # code. + LINE_PATTERN_STR = re.compile("(.*?(?:\r\n|\n|\r|$))") + LINE_PATTERN_BYTES = re.compile(b"(.*?(?:\r\n|\n|\r|$))") + + def __init__(self, source: str): + self._code_str: str = source + self._code_bytes: bytes = source.encode("utf-8") + self._starts_str: list[int] = self._initialize_line_starts( + self.LINE_PATTERN_STR, self._code_str + ) + self._starts_bytes: list[int] = self._initialize_line_starts( + self.LINE_PATTERN_BYTES, self._code_bytes + ) + + @classmethod + def _initialize_line_starts(cls, pattern, source) -> list[int]: + matches = list(pattern.finditer(source)) + if len(matches) > 1 and matches[-1].start() == matches[-1].end(): + matches.pop() + starts = [m.start() for m in matches] + if source[-1:] in [b"\n", b"\r", "\n", "\r"]: + starts.append(len(source)) + else: + starts.append(len(source) + 1) + return starts + + def __len__(self): + return len(self._starts_bytes) - 1 + + def _validate_lineno(self, lineno: int) -> int: + if not (0 < lineno <= len(self)): + raise ASTLinesAdapter.LineNumberOutOfRange( + f"Line number out of range: {lineno}" + ) + return lineno - 1 + + def _get_line_text(self, line_idx: int, col_offset: int | None = None) -> str: + return self._get_line_bytes(line_idx, col_offset).decode("utf-8") + + def _get_line_bytes(self, line_idx: int, col_offset: int | None = None) -> bytes: + start = self._starts_bytes[line_idx] + end = self._starts_bytes[line_idx + 1] + if col_offset is not None: + end = min(end, start + col_offset) + return self._code_bytes[start:end].rstrip(b"\r\n") + + def _line_region_offset(self, line_idx: int, col_offset: int) -> int: + """str offset relative to the start of line""" + if col_offset == 0: + return 0 + prefix = self._get_line_text(line_idx, col_offset) + return len(prefix) + + def _absolute_region_offset(self, line_idx: int, col_offset: int) -> int: + """str offset relative to the start of file""" + line_offset = self._line_region_offset(line_idx, col_offset) + return self._starts_str[line_idx] + line_offset + + @classmethod + def _pad_whitespace(cls, source: str) -> str: + """Equivalent to `ast._pad_whitespace(source)`""" + return "".join(c if c in "\f\t" else " " for c in source) + + def get_line(self, lineno: int) -> str: + line_idx = self._validate_lineno(lineno) + return self._get_line_text(line_idx) + + def get_source_segment(self, node, *, padded: bool = False) -> str | None: + """Equivalent to ``ast.get_source_segment(source, node, padded=padded)``. + + ``ast.get_source_segment(source, node)`` is convenient, but every call + re-splits the *entire* source string into lines (via the private + ``ast._splitlines_no_ff``) and re-encodes whatever lines it touches to + UTF-8. If you're pulling source segments for many nodes out of the same + file (e.g. walking a whole AST), that's a lot of repeated O(n) work. + """ + + region_start, region_end = self[node] + if region_start is None and region_end is None: + return None + source_segment = self._code_str[region_start:region_end] + + if padded and node.lineno != node.end_lineno: + line_idx = node.lineno - 1 + prefix = self._get_line_text(line_idx, node.col_offset) + padding = self._pad_whitespace(prefix) + source_segment = padding + source_segment + + return source_segment + + def __getitem__(self, node) -> tuple[int, int] | tuple[None, None]: + try: + line_idx = node.lineno - 1 + end_line_idx = node.end_lineno - 1 + col_offset = node.col_offset + end_col_offset = node.end_col_offset + except AttributeError: + return (None, None) + + region_start = self._absolute_region_offset(line_idx, col_offset) + region_end = self._absolute_region_offset(end_line_idx, end_col_offset) + return (region_start, region_end) + + class ArrayLinesAdapter: def __init__(self, lines): self.lines = lines diff --git a/ropetest/codeanalyzetest.py b/ropetest/codeanalyzetest.py index a73f607c..2b5675f9 100644 --- a/ropetest/codeanalyzetest.py +++ b/ropetest/codeanalyzetest.py @@ -1,9 +1,18 @@ +import ast +import textwrap import unittest from textwrap import dedent +import pytest + import rope.base.evaluate from rope.base import codeanalyze, exceptions, libutils, worder -from rope.base.codeanalyze import LogicalLineFinder, SourceLinesAdapter, get_block_start +from rope.base.codeanalyze import ( + ASTLinesAdapter, + LogicalLineFinder, + SourceLinesAdapter, + get_block_start, +) from ropetest import testutils @@ -1020,3 +1029,312 @@ class CustomLogicalLineFinderTest(LogicalLineFinderTest): def _logical_finder(self, code): lines = SourceLinesAdapter(code) return codeanalyze.CachingLogicalLineFinder(lines, codeanalyze.custom_generator) + + +class TestASTLinesAdapter: + @pytest.fixture(scope="class") + @classmethod + def source(cls, request) -> str: + return request.cls.SOURCE + + @pytest.fixture(scope="class") + @classmethod + def tree(self, source) -> ast.Module: + return ast.parse(source) + + @pytest.fixture(scope="class") + @classmethod + def ast_adapter(self, source) -> ASTLinesAdapter: + return ASTLinesAdapter(source) + + class TestAgreesWithAst: + SMALL_SOURCE = textwrap.dedent( + """ + def greet(name: str) -> str: + message = f"Hello, {name}! 你好" + return message + + class Foo: + def bar(self, x, y): + return x + y - len("héllo") + + def multiline_start_on_unicode(データ): + 処理 = データ + "更多文字" + config = { + "き": 1, + "り": 2, + } + return 処理, config + """ + ) + + # Deliberately full of unicode corner cases + UNICODE_EDGE_SOURCE = ( + "def f1():\n" + " party = \"\U0001F389\U0001F389\U0001F389\" + \"test\"\n" + " combining = \"e\u0301\u0301 vs \u00e9\"\n" + " return party, combining\n" + "\n" + "def f2(\u0627\u0633\u0645):\n" # Arabic identifier (RTL script) + " greeting = \"\u0645\u0631\u062d\u0628\u0627 \" + \u0627\u0633\u0645\n" + " return greeting\n" + "\r\n" # CRLF blank line + "def f3():\r\n" # rest of this function uses CRLF endings + " zwj = \"\U0001F9D1\u200d\U0001F680\"\r\n" # person + ZWJ + rocket + " multi = {\r\n" + " \"\u05d0\": 1,\r\n" # Hebrew (RTL script) as dict key + " \"\u05d1\": 2,\r\n" + " }\r\n" + " return zwj, multi\r\n" + "\r" # bare-CR blank line + "def f4():\r" # rest of this function uses bare-CR endings + " variation = \"\u2764\ufe0f\"\r" # heart + variation selector-16 + " return variation\r" + "\n" + "def f5():\n" + " x = 1\t+ \x0c{\n" # form feed + tab before a multi-line dict + " 1: 2,\n" + " }\n" + " return x\n" + ) + + SOURCES: dict[str, str] = { + "small": SMALL_SOURCE, + "unicode_edges": UNICODE_EDGE_SOURCE, + } + + @pytest.fixture(scope="class", params=sorted(SOURCES)) + @classmethod + def source(cls, request) -> str: + return cls.SOURCES[request.param] + + @pytest.fixture(scope="class") + @classmethod + def nodes(cls, source: str) -> list[ast.AST]: + tree = ast.parse(source) + return [n for n in ast.walk(tree)] + + def test_unpadded_segment_matches(self, source, ast_adapter, nodes): + for node in nodes: + assert ast_adapter.get_source_segment(node) == ast.get_source_segment(source, node) + + def test_padded_segment_matches(self, source, ast_adapter, nodes): + for node in nodes: + expected = ast.get_source_segment(source, node, padded=True) + actual = ast_adapter.get_source_segment(node, padded=True) + assert actual == expected + + def test_offsets_slice_source_to_unpadded_segment(self, source, ast_adapter, nodes): + for node in nodes: + start, end = ast_adapter[node] + expected = ast.get_source_segment(source, node) + if (start, end) == (None, None): + assert expected is None + else: + assert source[start:end] == expected + + class TestLineEndings: + def test_empty_source_has_one_line(self): + ast_adapter = ASTLinesAdapter("") + assert len(ast_adapter) == 1 + assert ast_adapter._starts_str == [0, 1] + assert ast_adapter.get_line(1) == "" + with pytest.raises(ASTLinesAdapter.LineNumberOutOfRange): + ast_adapter.get_line(2) + + def test_source_without_trailing_newline(self): + ast_adapter = ASTLinesAdapter("abc") + assert len(ast_adapter) == 1 + assert ast_adapter._starts_str == [0, 4] + assert ast_adapter.get_line(1) == "abc" + with pytest.raises(ASTLinesAdapter.LineNumberOutOfRange): + ast_adapter.get_line(2) + + def test_source_with_trailing_lf_has_no_phantom_line(self): + ast_adapter = ASTLinesAdapter("abc\n") + assert len(ast_adapter) == 1 + assert ast_adapter._starts_str == [0, 4] + assert ast_adapter.get_line(1) == "abc" + with pytest.raises(ASTLinesAdapter.LineNumberOutOfRange): + ast_adapter.get_line(2) + + def test_source_with_trailing_crlf_has_no_phantom_line(self): + ast_adapter = ASTLinesAdapter("abc\r\n") + assert len(ast_adapter) == 1 + assert ast_adapter._starts_str == [0, 5] + assert ast_adapter.get_line(1) == "abc" + with pytest.raises(ASTLinesAdapter.LineNumberOutOfRange): + ast_adapter.get_line(2) + + def test_source_with_trailing_cr_has_no_phantom_line(self): + ast_adapter = ASTLinesAdapter("abc\r") + assert len(ast_adapter) == 1 + assert ast_adapter._starts_str == [0, 4] + assert ast_adapter.get_line(1) == "abc" + with pytest.raises(ASTLinesAdapter.LineNumberOutOfRange): + ast_adapter.get_line(2) + + def test_source_ends_with_multiple_newlines(self): + ast_adapter = ASTLinesAdapter("abc\n\n") + assert len(ast_adapter) == 2 + assert ast_adapter._starts_str == [0, 4, 5] + assert ast_adapter.get_line(1) == "abc" + assert ast_adapter.get_line(2) == "" + with pytest.raises(ASTLinesAdapter.LineNumberOutOfRange): + ast_adapter.get_line(3) + + def test_multiple_lines_lf(self): + ast_adapter = ASTLinesAdapter("a\nbb\nccc\n") + assert ast_adapter._starts_str == [0, 2, 5, 9] + assert ast_adapter.get_line(1) == "a" + assert ast_adapter.get_line(2) == "bb" + assert ast_adapter.get_line(3) == "ccc" + with pytest.raises(ASTLinesAdapter.LineNumberOutOfRange): + ast_adapter.get_line(4) + + class TestMixedLineEndings: + SOURCE = ( + "s = 1\n" + "café = {\n" + " '你': 100,\r\n" + " '好': 200,\n" + " 30: 300,\r" + " 40: 400,\r\n" + "}\n" + "print(café)" + ) + + @pytest.fixture(scope="class") + @classmethod + def dict_node(self, tree) -> ast.Dict: + node = next( + n + for n in ast.walk(tree) + if isinstance(n, ast.Dict) + ) + assert node.lineno != node.end_lineno, "expected a multi-line node for this test to mean anything" + return node + + def test_unpadded_get_source_segment_of_node_with_mixed_line_endings(self, dict_node, ast_adapter): + assert ast_adapter.get_source_segment(dict_node) == ast.get_source_segment( + self.SOURCE, dict_node + ) + + def test_padded_get_source_segment_of_node_with_mixed_line_endings(self, dict_node, ast_adapter): + assert ast_adapter.get_source_segment( + dict_node, padded=True + ) == ast.get_source_segment( + self.SOURCE, dict_node, padded=True + ) + + def test_line_starts_found_for_every_ending_style(self, ast_adapter): + assert len(ast_adapter) == 8 + + def test_get_line(self, ast_adapter): + lines = [ast_adapter.get_line(line_idx + 1) for line_idx in range(len(ast_adapter))] + expected = [ + 's = 1', + 'café = {', + " '你': 100,", + " '好': 200,", + ' 30: 300,', + ' 40: 400,', + '}', + 'print(café)', + ] + assert lines == expected + + class TestPaddedMultiByteFirstLine: + SOURCE = 'x = "你好" + {\n 1: 2,\n}\n' + + @pytest.fixture + def dict_node(self, tree) -> ast.Dict: + dict_node = next(n for n in ast.walk(tree) if isinstance(n, ast.Dict)) + assert dict_node.lineno != dict_node.end_lineno + assert dict_node.col_offset > 0, "col_offset must be non-zero for this test to mean anything" + return dict_node + + def test_padded_segment_matches_ast(self, dict_node, ast_adapter): + expected = ast.get_source_segment(self.SOURCE, dict_node, padded=True) + actual = ast_adapter.get_source_segment(dict_node, padded=True) + assert actual == expected + + def test_padding_length_is_char_count_not_byte_count(self, dict_node, ast_adapter): + segment = ast_adapter.get_source_segment(dict_node, padded=True) + first_line_of_segment = segment.splitlines()[0] + leading_spaces = len(first_line_of_segment) - len(first_line_of_segment.lstrip(" ")) + # 'x = "你好" + ' is 11 characters (5 ASCII + 2 CJK chars + 4 ASCII), + # despite being 15 UTF-8 bytes. + assert leading_spaces == 11 + + class TestMissingLocationInfoTest: + SOURCE = 'abc = def' + + def test_get_source_segment_returns_none(self, ast_adapter): + node = ast.Module(body=[], type_ignores=[]) + assert not hasattr(node, 'lineno') + assert not hasattr(node, 'col_offset') + + assert ast_adapter.get_source_segment(node) is None + + def test_get_source_region_returns_none(self, ast_adapter): + node = ast.Module(body=[], type_ignores=[]) + assert not hasattr(node, 'lineno') + assert not hasattr(node, 'col_offset') + + assert ast_adapter[node] == (None, None) + + class TestRegionOffset: + SOURCE = textwrap.dedent( + """ + def multiline_start_on_unicode(データ): + 処理 = データ + "更多文字" + config = { + "き": 1, + "り": 2, + } + return 処理, config + """ + ) + + def test_zero_offset_is_always_zero(self, ast_adapter): + for line_idx in range(len(ast_adapter)): + assert ast_adapter._line_region_offset(line_idx, 0) == 0 + + def test_ascii_only_line_matches_col_offset(self, tree, source, ast_adapter): + node = next( + n + for n in ast.walk(tree) + if isinstance(n, ast.Name) and n.id == "config" + ) + line_idx = 3 + assert node.lineno == node.end_lineno == 4 + line = ast_adapter.get_line(4) + assert line == " config = {" + + start = ast_adapter._line_region_offset(line_idx, node.col_offset) + assert start == node.col_offset == 4 + end = ast_adapter._line_region_offset(line_idx, node.end_col_offset) + assert end == node.end_col_offset == 10 + assert line[start:end] == "config" + + def test_full_line_byte_length_matches_full_line_char_length(self, ast_adapter): + for line_idx in range(len(ast_adapter)): + line_bytes = ast_adapter._get_line_bytes(line_idx) + assert ast_adapter._line_region_offset(line_idx, len(line_bytes)) == len( + ast_adapter._get_line_text(line_idx) + ) + + @pytest.mark.parametrize( + "line_text, byte_col, expected_char_col", + [ + ("hello world\n", 5, 5), # pure ASCII: byte offset == char offset + ("你好世界\n", 3, 1), # first char is 3 bytes -> 1 char consumed + ("你好世界\n", 6, 2), # first two chars are 6 bytes -> 2 chars + ('"héllo"\n', 2, 2), # ASCII prefix before the accented char + ('"héllo"\n', 4, 3), # 'é' (2 bytes) fully consumed -> 3 chars ('"h\u00e9') + ], + ) + def test_known_byte_to_char_conversions(self, line_text, byte_col, expected_char_col): + ast_adapter = ASTLinesAdapter(line_text) + assert ast_adapter._line_region_offset(0, byte_col) == expected_char_col From c058ff33c5070e42645bc70ddfca9333231ccd98 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Mon, 7 Sep 2026 11:51:40 +1000 Subject: [PATCH 02/10] Replace ad-hoc offset computations in patchedast to use ASTLinesAdapter col_offset/end_col_offset uses utf-8 offset which generally only accidentally coincides with str-offset that rope uses as node.region. ASTLinesAdapter ensures correct computation of offset. --- rope/refactor/patchedast.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index a10350ae..07b1e9d1 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -67,6 +67,7 @@ def __init__(self, source, children=False): self.source = _Source(source) self.children = children self.lines = codeanalyze.SourceLinesAdapter(source) + self.ast_adapter = codeanalyze.ASTLinesAdapter(source) self.children_stack = [] Number = object() @@ -217,7 +218,8 @@ def _find_next_statement_start(self): for children in reversed(self.children_stack): for child in children: if isinstance(child, ast.stmt): - return child.col_offset + self.lines.get_line_start(child.lineno) + start, _ = self.ast_adapter[child] + return start return len(self.source.source) def _join(self, iterable, separator): @@ -576,11 +578,8 @@ def _If(self, node): def _is_elif(self, node): if not isinstance(node, ast.If): return False - offset = self.lines.get_line_start(node.lineno) + node.col_offset - word = self.source[offset : offset + 4] - # XXX: This is a bug; the offset does not point to the first - alt_word = self.source[offset - 5 : offset - 1] - return "elif" in (word, alt_word) + start, end = self.ast_adapter[node] + return "elif" in self.source[start:start+4] def _IfExp(self, node): return self._handle(node, [node.body, "if", node.test, "else", node.orelse]) @@ -813,27 +812,23 @@ def _MatchSequence(self, node): *closing_paren, ] else: - node_start = (node.lineno, node.col_offset) - node_end = (node.end_lineno, node.end_col_offset) - children = [self.lines[node_start:node_end]] + empty_tuple = self.ast_adapter.get_source_segment(node) + children = [empty_tuple] self._handle(node, children) def _get_surrounding_parens(self, node: ast.MatchSequence): - node_start = (node.lineno, node.col_offset) - first_pattern_start = (node.patterns[0].lineno, node.patterns[0].col_offset) - opening_paren = self.lines[node_start:first_pattern_start].strip() + node_start, node_end = self.ast_adapter[node] + first_pattern_start, _ = self.ast_adapter[node.patterns[0]] + _, last_pattern_end = self.ast_adapter[node.patterns[-1]] + opening_paren = self.source[node_start:first_pattern_start].strip() + closing_paren = self.source[last_pattern_end:node_end].strip() + if opening_paren not in ["[", "(", ""]: warnings.warn( f"Unexpected character in MatchSequence's opening_paren <{opening_paren}>; please report!", RuntimeWarning, ) - last_pattern_end = ( - node.patterns[-1].end_lineno, - node.patterns[-1].end_col_offset, - ) - node_end = (node.end_lineno, node.end_col_offset) - closing_paren = self.lines[last_pattern_end:node_end].strip() if closing_paren not in ["]", ")", ""]: warnings.warn( f"Unexpected character in MatchSequence's closing_paren <{closing_paren}>; please report!", From 484a9308e2be4ad27782dc644e0cb6626f66508d Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Mon, 7 Sep 2026 11:59:43 +1000 Subject: [PATCH 03/10] Remove unused _Source.__getslice__() method All slices are handled by __getitem__() since Python 3. --- rope/refactor/patchedast.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index 07b1e9d1..904485c3 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -1010,9 +1010,6 @@ def find_backwards(self, pattern, offset): def __getitem__(self, index): return self.source[index] - def __getslice__(self, i, j): - return self.source[i:j] - def _get_number_pattern(self): # HACK: It is merely an approaximation and does the job integer = r"\-?(0[xo][\da-fA-F]+|\d+)" From a9091eb5ed6e3ecf1a82ad55c540dbdffd2a22f2 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Mon, 7 Sep 2026 12:01:53 +1000 Subject: [PATCH 04/10] Remove SourceLinesAdapter.__getitem__() implementation Now we have a better ASTLinesAdapter that actually handles unicode correctly. --- rope/base/codeanalyze.py | 19 ------------------- ropetest/codeanalyzetest.py | 20 -------------------- 2 files changed, 39 deletions(-) diff --git a/rope/base/codeanalyze.py b/rope/base/codeanalyze.py index 24d20f66..e64103d9 100644 --- a/rope/base/codeanalyze.py +++ b/rope/base/codeanalyze.py @@ -70,25 +70,6 @@ def get_line_start(self, lineno): def get_line_end(self, lineno): return self.starts[lineno] - 1 - def __getitem__(self, subscript): - start_offset = self._calculate_offset(subscript.start) - stop_offset = self._calculate_offset(subscript.stop) - return self.code[start_offset:stop_offset] - - def _calculate_offset(self, coord: tuple[int, int]) -> int: - lineno, col_offset = coord - lineno = self._clamp(0, self.length(), lineno) - col_offset = self._clamp( - 0, - self.get_line_end(lineno) - self.get_line_start(lineno), - col_offset, - ) - - return self.get_line_start(lineno) + col_offset - - def _clamp(self, min_value, max_value, value): - return max(min_value, min(max_value, value)) - class ASTLinesAdapter: """ diff --git a/ropetest/codeanalyzetest.py b/ropetest/codeanalyzetest.py index 2b5675f9..c27694de 100644 --- a/ropetest/codeanalyzetest.py +++ b/ropetest/codeanalyzetest.py @@ -47,26 +47,6 @@ def test_source_lines_last_line_with_no_new_line(self): to_lines = SourceLinesAdapter("line1") self.assertEqual(1, to_lines.get_line_number(5)) - def test_source_lines_getitem_range(self): - to_lines = SourceLinesAdapter("line1\nline2\nline3\nline4\n") - self.assertEqual('ne2\nli', to_lines[(2, 2):(3, 2)]) - - def test_source_lines_getitem_start_lineno_out_of_range(self): - to_lines = SourceLinesAdapter("line1\nline2\nline3\nline4\n") - self.assertEqual("", to_lines[(100, 2):(3, 2)]) - - def test_source_lines_getitem_start_col_offset_out_of_range(self): - to_lines = SourceLinesAdapter("line1\nline2\nline3\nline4\n") - self.assertEqual('\nli', to_lines[(2, 100):(3, 2)]) - - def test_source_lines_getitem_end_lineno_out_of_range(self): - to_lines = SourceLinesAdapter("line1\nline2\nline3\nline4\n") - self.assertEqual("ne2\nline3\nline4\n", to_lines[(2, 2):(100, 2)]) - - def test_source_lines_getitem_end_col_offset_out_of_range(self): - to_lines = SourceLinesAdapter("line1\nline2\nline3\nline4\n") - self.assertEqual('ne2\nline3', to_lines[(2, 2):(3, 100)]) - class WordRangeFinderTest(unittest.TestCase): def _find_primary(self, code, offset): From da2079f97c3d4b8239816499b20157b8fa07d25e Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Mon, 7 Sep 2026 12:23:49 +1000 Subject: [PATCH 05/10] Added test for unicode handling around MatchSequence --- ropetest/refactor/patchedasttest.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ropetest/refactor/patchedasttest.py b/ropetest/refactor/patchedasttest.py index 67a521b7..c014f6ed 100644 --- a/ropetest/refactor/patchedasttest.py +++ b/ropetest/refactor/patchedasttest.py @@ -1595,6 +1595,26 @@ def test_match_node_with_match_sequence_with_star_and_value(self): "MatchSequence", ["[", "", "MatchStar", "", ",", " ", "MatchValue", "", "]"] ) + @testutils.only_for_versions_higher("3.10") + def test_match_node_with_match_sequence_with_multibyte_unicode(self): + source = dedent("""\ + match x: + case ["😃", *rest] as myval: + print(myval) + """) + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + self.assert_single_case_match_block(checker, "MatchAs") + checker.check_children("MatchAs", [ + "MatchSequence", " ", "as", " ", "myval", + ]) + checker.check_children("MatchSequence", [ + "[", "", "MatchValue", "", ",", " ", "MatchStar", "", "]", + ]) + checker.check_children("MatchStar", [ + "*", "", "rest" + ]) + @testutils.only_for_versions_higher("3.10") def test_match_node_with_match_as_capture_pattern(self): source = dedent("""\ From d85b3d68001e66e710899bfbcf9ba589281f2f12 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Mon, 7 Sep 2026 13:03:33 +1000 Subject: [PATCH 06/10] Black --- rope/refactor/patchedast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index 904485c3..2a52fc92 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -579,7 +579,7 @@ def _is_elif(self, node): if not isinstance(node, ast.If): return False start, end = self.ast_adapter[node] - return "elif" in self.source[start:start+4] + return "elif" in self.source[start : start + 4] def _IfExp(self, node): return self._handle(node, [node.body, "if", node.test, "else", node.orelse]) From c66a85c0e51e0836f89865f70e2d17cef5b09644 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Mon, 7 Sep 2026 13:05:53 +1000 Subject: [PATCH 07/10] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 010425d5..1650aa77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,12 @@ - #850 Update and pin black version in pre-commit and Github Actions - #851 Bump supported python version to up to Python 3.14 -- #852 Implement patchedast handlers for TypeAlias +- #852 Implement patchedast handlers for TypeAlias - #853 Implement patchedast handlers TypeVar - #847 Avoid printing autoimport syntax errors (@yangfan-yf-yf) - #623, #819, #863 Support MatchOr, MatchSequence, MatchStar (@jheld, @lieryan) - #870 Add default implementation for is_dir() (@lieryan) +- #872 Fix unicode handling in patchedast (@lieryan) # Release 1.14.0 From 58b815f06d935b02144a71c591f0073fee6fa7f0 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Tue, 15 Sep 2026 22:27:30 +1000 Subject: [PATCH 08/10] Add ascii-only line optimisation The vast majority of Python source code is going to be mostly ascii-only text, we can completely skip the entire unicode encode/decode overhead when there's only ascii characters. --- rope/base/codeanalyze.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/rope/base/codeanalyze.py b/rope/base/codeanalyze.py index e64103d9..234626e8 100644 --- a/rope/base/codeanalyze.py +++ b/rope/base/codeanalyze.py @@ -144,10 +144,19 @@ def _get_line_bytes(self, line_idx: int, col_offset: int | None = None) -> bytes end = min(end, start + col_offset) return self._code_bytes[start:end].rstrip(b"\r\n") + def _is_line_ascii_only(self, line_idx: int): + start_str = self._starts_str[line_idx] + end_str = self._starts_str[line_idx + 1] + start_bytes = self._starts_bytes[line_idx] + end_bytes = self._starts_bytes[line_idx + 1] + # str length == bytes length iff they're ascii-only because any + # non-ascii characters would be at least 2 bytes + return (end_str - start_str) == (end_bytes - start_bytes) + def _line_region_offset(self, line_idx: int, col_offset: int) -> int: """str offset relative to the start of line""" - if col_offset == 0: - return 0 + if col_offset == 0 or self._is_line_ascii_only(line_idx): + return col_offset prefix = self._get_line_text(line_idx, col_offset) return len(prefix) From 8de78a70ff347f86d6d88f28919066be576ea7c3 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Tue, 15 Sep 2026 22:41:27 +1000 Subject: [PATCH 09/10] Add check for missing location info on an AST node This matches the behavior of ast.get_source_segment(). --- rope/base/codeanalyze.py | 2 ++ ropetest/codeanalyzetest.py | 24 ++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/rope/base/codeanalyze.py b/rope/base/codeanalyze.py index 234626e8..d8b8da08 100644 --- a/rope/base/codeanalyze.py +++ b/rope/base/codeanalyze.py @@ -199,6 +199,8 @@ def get_source_segment(self, node, *, padded: bool = False) -> str | None: def __getitem__(self, node) -> tuple[int, int] | tuple[None, None]: try: + if node.end_lineno is None or node.end_col_offset is None: + return (None, None) line_idx = node.lineno - 1 end_line_idx = node.end_lineno - 1 col_offset = node.col_offset diff --git a/ropetest/codeanalyzetest.py b/ropetest/codeanalyzetest.py index c27694de..23b4ae35 100644 --- a/ropetest/codeanalyzetest.py +++ b/ropetest/codeanalyzetest.py @@ -1250,20 +1250,40 @@ def test_padding_length_is_char_count_not_byte_count(self, dict_node, ast_adapte class TestMissingLocationInfoTest: SOURCE = 'abc = def' - def test_get_source_segment_returns_none(self, ast_adapter): + def test_get_source_segment_returns_none_without_location(self, ast_adapter): node = ast.Module(body=[], type_ignores=[]) assert not hasattr(node, 'lineno') assert not hasattr(node, 'col_offset') assert ast_adapter.get_source_segment(node) is None - def test_get_source_region_returns_none(self, ast_adapter): + def test_get_source_region_returns_none_without_location(self, ast_adapter): node = ast.Module(body=[], type_ignores=[]) assert not hasattr(node, 'lineno') assert not hasattr(node, 'col_offset') assert ast_adapter[node] == (None, None) + def test_get_source_segment_returns_none_without_end_location(self, ast_adapter): + node = ast.Add( + lineno=1, + col_offset=1, + end_lineno=None, + end_col_offset=None, + ) + + assert ast_adapter.get_source_segment(node) is None + + def test_get_source_region_returns_none_without_end_location(self, ast_adapter): + node = ast.Add( + lineno=1, + col_offset=1, + end_lineno=None, + end_col_offset=None, + ) + + assert ast_adapter[node] == (None, None) + class TestRegionOffset: SOURCE = textwrap.dedent( """ From acf6100bddf6945237306c0388f0f780c67a0dab Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Wed, 16 Sep 2026 00:37:22 +1000 Subject: [PATCH 10/10] Fix AST creation for Python 3.15 --- ropetest/codeanalyzetest.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/ropetest/codeanalyzetest.py b/ropetest/codeanalyzetest.py index 23b4ae35..6dd16ab3 100644 --- a/ropetest/codeanalyzetest.py +++ b/ropetest/codeanalyzetest.py @@ -1265,22 +1265,20 @@ def test_get_source_region_returns_none_without_location(self, ast_adapter): assert ast_adapter[node] == (None, None) def test_get_source_segment_returns_none_without_end_location(self, ast_adapter): - node = ast.Add( - lineno=1, - col_offset=1, - end_lineno=None, - end_col_offset=None, - ) + node = ast.Add() + node.lineno = 1 + node.col_offset = 1 + node.end_lineno = None + node.end_col_offset = None assert ast_adapter.get_source_segment(node) is None def test_get_source_region_returns_none_without_end_location(self, ast_adapter): - node = ast.Add( - lineno=1, - col_offset=1, - end_lineno=None, - end_col_offset=None, - ) + node = ast.Add() + node.lineno = 1 + node.col_offset = 1 + node.end_lineno = None + node.end_col_offset = None assert ast_adapter[node] == (None, None)