From ea07af53113d64cca9fb37562f78495d91f49d15 Mon Sep 17 00:00:00 2001 From: Yang Fan <2022112761@stu.hit.edu.cn> Date: Thu, 10 Sep 2026 12:54:58 +0800 Subject: [PATCH] Fix UTF-8 offsets in sequence patterns --- CHANGELOG.md | 3 ++- rope/refactor/patchedast.py | 26 +++++++++++++++++--------- ropetest/refactor/patchedasttest.py | 23 +++++++++++++++++++++++ 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 010425d5..26a48d91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ # **Upcoming release** +- Fix patched-AST sequence patterns containing non-ASCII text - #850 Update and pin black version in pre-commit and Github Actions - #851 Bump supported python version to up to Python 3.14 -- #852 Implement patchedast handlers for TypeAlias +- #852 Implement patchedast handlers for TypeAlias - #853 Implement patchedast handlers TypeVar - #847 Avoid printing autoimport syntax errors (@yangfan-yf-yf) - #623, #819, #863 Support MatchOr, MatchSequence, MatchStar (@jheld, @lieryan) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index a10350ae..3784a5ff 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -813,27 +813,35 @@ def _MatchSequence(self, node): *closing_paren, ] else: - node_start = (node.lineno, node.col_offset) - node_end = (node.end_lineno, node.end_col_offset) - children = [self.lines[node_start:node_end]] + node_start = self._get_ast_offset(node.lineno, node.col_offset) + node_end = self._get_ast_offset(node.end_lineno, node.end_col_offset) + children = [self.source.source[node_start:node_end]] self._handle(node, children) + def _get_ast_offset(self, lineno, col_offset): + # AST columns count UTF-8 bytes; source slices count Unicode characters. + line = self.lines.get_line(lineno) + prefix = line.encode("utf-8")[:col_offset].decode("utf-8") + return self.lines.get_line_start(lineno) + len(prefix) + def _get_surrounding_parens(self, node: ast.MatchSequence): - node_start = (node.lineno, node.col_offset) - first_pattern_start = (node.patterns[0].lineno, node.patterns[0].col_offset) - opening_paren = self.lines[node_start:first_pattern_start].strip() + node_start = self._get_ast_offset(node.lineno, node.col_offset) + first_pattern_start = self._get_ast_offset( + node.patterns[0].lineno, node.patterns[0].col_offset + ) + opening_paren = self.source.source[node_start:first_pattern_start].strip() if opening_paren not in ["[", "(", ""]: warnings.warn( f"Unexpected character in MatchSequence's opening_paren <{opening_paren}>; please report!", RuntimeWarning, ) - last_pattern_end = ( + last_pattern_end = self._get_ast_offset( node.patterns[-1].end_lineno, node.patterns[-1].end_col_offset, ) - node_end = (node.end_lineno, node.end_col_offset) - closing_paren = self.lines[last_pattern_end:node_end].strip() + node_end = self._get_ast_offset(node.end_lineno, node.end_col_offset) + closing_paren = self.source.source[last_pattern_end:node_end].strip() if closing_paren not in ["]", ")", ""]: warnings.warn( f"Unexpected character in MatchSequence's closing_paren <{closing_paren}>; please report!", diff --git a/ropetest/refactor/patchedasttest.py b/ropetest/refactor/patchedasttest.py index 67a521b7..13087353 100644 --- a/ropetest/refactor/patchedasttest.py +++ b/ropetest/refactor/patchedasttest.py @@ -1557,6 +1557,29 @@ def test_match_node_with_match_sequence_with_internal_parens(self): "MatchSequence", "", ",", " ", "MatchSequence" ]) + @testutils.only_for_versions_higher("3.10") + def test_match_sequence_with_non_ascii_text(self): + patterns = [ + "[é, y]", + "('你好', y)", + "['é', [x, y]]", + "['é', []]", + "['é', ()]", + "'é', [x, y]", + "[\n 'é', [x, y]\n ]", + ] + for pattern in patterns: + with self.subTest(pattern=pattern): + source = f"match value:\n case {pattern}:\n pass\n" + ast_frag = patchedast.get_patched_ast(source, True) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + for node in ast.walk(ast_frag): + if isinstance(node, ast.MatchSequence): + self.assertEqual( + ast.get_source_segment(source, node), + source[node.region[0]:node.region[1]], + ) + @testutils.only_for_versions_higher("3.10") def test_match_node_with_match_sequence_empty_round_parens(self): source = dedent("""\