Fix UTF-8 offsets in sequence patterns

CHANGELOG.md

@@ -1,8 +1,9 @@
 # **Upcoming release**
 
+- Fix patched-AST sequence patterns containing non-ASCII text
 - #850 Update and pin black version in pre-commit and Github Actions
 - #851 Bump supported python version to up to Python 3.14
-- #852 Implement patchedast handlers for TypeAlias 
+- #852 Implement patchedast handlers for TypeAlias
 - #853 Implement patchedast handlers TypeVar
 - #847 Avoid printing autoimport syntax errors (@yangfan-yf-yf)
 - #623, #819, #863 Support MatchOr, MatchSequence, MatchStar (@jheld, @lieryan)

rope/refactor/patchedast.py

@@ -813,27 +813,35 @@ def _MatchSequence(self, node):
                 *closing_paren,
             ]
         else:
-            node_start = (node.lineno, node.col_offset)
-            node_end = (node.end_lineno, node.end_col_offset)
-            children = [self.lines[node_start:node_end]]
+            node_start = self._get_ast_offset(node.lineno, node.col_offset)
+            node_end = self._get_ast_offset(node.end_lineno, node.end_col_offset)
+            children = [self.source.source[node_start:node_end]]
         self._handle(node, children)
 
+    def _get_ast_offset(self, lineno, col_offset):
+        # AST columns count UTF-8 bytes; source slices count Unicode characters.
+        line = self.lines.get_line(lineno)
+        prefix = line.encode("utf-8")[:col_offset].decode("utf-8")
+        return self.lines.get_line_start(lineno) + len(prefix)
+
     def _get_surrounding_parens(self, node: ast.MatchSequence):
-        node_start = (node.lineno, node.col_offset)
-        first_pattern_start = (node.patterns[0].lineno, node.patterns[0].col_offset)
-        opening_paren = self.lines[node_start:first_pattern_start].strip()
+        node_start = self._get_ast_offset(node.lineno, node.col_offset)
+        first_pattern_start = self._get_ast_offset(
+            node.patterns[0].lineno, node.patterns[0].col_offset
+        )
+        opening_paren = self.source.source[node_start:first_pattern_start].strip()
         if opening_paren not in ["[", "(", ""]:
             warnings.warn(
                 f"Unexpected character in MatchSequence's opening_paren <{opening_paren}>; please report!",
                 RuntimeWarning,
             )
 
-        last_pattern_end = (
+        last_pattern_end = self._get_ast_offset(
             node.patterns[-1].end_lineno,
             node.patterns[-1].end_col_offset,
         )
-        node_end = (node.end_lineno, node.end_col_offset)
-        closing_paren = self.lines[last_pattern_end:node_end].strip()
+        node_end = self._get_ast_offset(node.end_lineno, node.end_col_offset)
+        closing_paren = self.source.source[last_pattern_end:node_end].strip()
         if closing_paren not in ["]", ")", ""]:
             warnings.warn(
                 f"Unexpected character in MatchSequence's closing_paren <{closing_paren}>; please report!",

ropetest/refactor/patchedasttest.py

@@ -1557,6 +1557,29 @@ def test_match_node_with_match_sequence_with_internal_parens(self):
             "MatchSequence", "", ",", " ", "MatchSequence"
         ])
 
+    @testutils.only_for_versions_higher("3.10")
+    def test_match_sequence_with_non_ascii_text(self):
+        patterns = [
+            "[é, y]",
+            "('你好', y)",
+            "['é', [x, y]]",
+            "['é', []]",
+            "['é', ()]",
+            "'é', [x, y]",
+            "[\n        'é', [x, y]\n    ]",
+        ]
+        for pattern in patterns:
+            with self.subTest(pattern=pattern):
+                source = f"match value:\n    case {pattern}:\n        pass\n"
+                ast_frag = patchedast.get_patched_ast(source, True)
+                self.assertEqual(source, patchedast.write_ast(ast_frag))
+                for node in ast.walk(ast_frag):
+                    if isinstance(node, ast.MatchSequence):
+                        self.assertEqual(
+                            ast.get_source_segment(source, node),
+                            source[node.region[0]:node.region[1]],
+                        )
+
     @testutils.only_for_versions_higher("3.10")
     def test_match_node_with_match_sequence_empty_round_parens(self):
         source = dedent("""\