patchedast: locate multi-line f-strings, replacement fields and match-sequence delimiters by parser coordinates

CHANGELOG.md

@@ -8,6 +8,7 @@
 - #623, #819, #863 Support MatchOr, MatchSequence, MatchStar (@jheld, @lieryan)
 - #870 Add default implementation for is_dir() (@lieryan)
 - #872 Fix unicode handling in patchedast (@lieryan)
+- Locate multi-line f-strings, replacement fields and match-sequence delimiters in patchedast by parser coordinates
 
 # Release 1.14.0
 

rope/refactor/patchedast.py

@@ -1,6 +1,8 @@
 import collections
+import io
 import numbers
 import re
+import tokenize
 import warnings
 from itertools import chain
 
@@ -395,11 +397,10 @@ def end_quote_char():
             return self.source[end - len(quote_char) : end]
 
         QUOTE_CHARS = ['"""', "'''", '"', "'"]
-        offset = self.source.offset
-        start, end = self.source.consume_string(
-            end=self._find_next_statement_start(),
-        )
-        self.source.offset = offset
+        # The parser's coordinates bound the whole literal, implicit
+        # concatenation included.  A textual string pattern cannot: since
+        # PEP 701 a replacement field may span lines (``f"{\n x\n}"``).
+        start, end = self.ast_adapter[node]
 
         children = []
         children.append(start_quote_char())
@@ -410,6 +411,14 @@ def end_quote_char():
         self._handle(node, children)
 
     def _FormattedValue(self, node):
+        # Since Python 3.12 the parser locates each replacement field at its
+        # own "{".  Starting there keeps a "{" inside a plain string of the
+        # same implicit concatenation -- ``"({})," f"{a}"`` -- from being
+        # taken as the field's opening brace.  Older parsers give the
+        # enclosing literal's position, which does not point at a "{".
+        field_start, _ = self.ast_adapter[node]
+        if field_start >= self.source.offset and self.source[field_start] == "{":
+            self.source.offset = field_start
         children = []
         children.append("{")
         children.append(node.value)
@@ -817,24 +826,32 @@ def _MatchSequence(self, node):
         self._handle(node, children)
 
     def _get_surrounding_parens(self, node: ast.MatchSequence):
+        """The sequence's own ``[...]``/``(...)`` delimiters, if it has them.
+
+        The text between the node's start and its first pattern can also
+        hold grouping parentheses of that pattern -- ``case (("a" | b), c)``
+        -- which the AST does not represent.  The sequence is delimited only
+        when the bracket at its first character is the one closed at its
+        last character; ``case (1), (2)`` starts and ends with parentheses
+        that belong to its elements.
+        """
         node_start, node_end = self.ast_adapter[node]
-        first_pattern_start, _ = self.ast_adapter[node.patterns[0]]
-        _, last_pattern_end = self.ast_adapter[node.patterns[-1]]
-        opening_paren = self.source[node_start:first_pattern_start].strip()
-        closing_paren = self.source[last_pattern_end:node_end].strip()
-
-        if opening_paren not in ["[", "(", ""]:
-            warnings.warn(
-                f"Unexpected character in MatchSequence's opening_paren <{opening_paren}>; please report!",
-                RuntimeWarning,
-            )
-
-        if closing_paren not in ["]", ")", ""]:
-            warnings.warn(
-                f"Unexpected character in MatchSequence's closing_paren <{closing_paren}>; please report!",
-                RuntimeWarning,
-            )
-        return opening_paren, closing_paren
+        segment = self.source[node_start:node_end]
+        pairs = {"[": "]", "(": ")"}
+        if segment[:1] not in pairs or segment[-1:] != pairs[segment[0]]:
+            return "", ""
+        depth = 0
+        for token in tokenize.generate_tokens(io.StringIO(segment).readline):
+            if token.type != tokenize.OP or token.string not in "()[]{}":
+                continue
+            depth += 1 if token.string in "([{" else -1
+            if depth == 0:
+                end_row = segment.count("\n") + 1
+                end_col = len(segment) - (segment.rfind("\n") + 1)
+                if token.end != (end_row, end_col):
+                    return "", ""
+                return segment[0], segment[-1]
+        return "", ""
 
     def _MatchStar(self, node):
         self._handle(node, ["*", node.name or "_"])

ropetest/refactor/extracttest.py

@@ -1284,7 +1284,6 @@ def new_func(a_var):
         """)
         self.assertEqual(expected, refactored)
 
-    @unittest.expectedFailure
     @testutils.only_for_versions_higher("3.6")
     def test_extract_method_f_string_false_format_value_in_regular_string(self):
         code = dedent("""\

ropetest/refactor/patchedasttest.py

@@ -326,6 +326,15 @@ def test_handling_format_strings_basic(self):
         checker.check_children("JoinedStr", ['f"', "abc", "FormattedValue", "", '"'])
         checker.check_children("FormattedValue", ["{", "", "Name", "", "}"])
 
+    @testutils.only_for_versions_higher("3.12")
+    def test_handling_format_strings_with_multiline_replacement_field(self):
+        source = 'x = (f"{a}{\n    b\n}{c}").upper()\n'
+        ast_frag = patchedast.get_patched_ast(source, True)
+        checker = _ResultChecker(self, ast_frag)
+        start = source.index('f"')
+        checker.check_region("JoinedStr", start, source.index(")."))
+        self.assertEqual(source, patchedast.write_ast(ast_frag))
+
     @testutils.only_for_versions_higher("3.6")
     def test_handling_format_strings_with_implicit_join(self):
         source = '''"1" + rf'abc{a}' f"""xxx{b} """\n'''
@@ -1557,6 +1566,35 @@ def test_match_node_with_match_sequence_with_internal_parens(self):
             "MatchSequence", "", ",", " ", "MatchSequence"
         ])
 
+    @testutils.only_for_versions_higher("3.10")
+    def test_match_node_with_match_sequence_with_grouped_first_pattern(self):
+        source = dedent("""\
+            match x:
+                case (("a" | None), None):
+                    pass
+        """)
+        ast_frag = patchedast.get_patched_ast(source, True)
+        checker = _ResultChecker(self, ast_frag)
+        checker.check_children("MatchSequence", [
+            "(", "(", "MatchOr", ")", ",", " ", "MatchSingleton", "", ")",
+        ])
+        start = source.index("((")
+        checker.check_region("MatchSequence", start, source.index(":\n", start))
+        self.assertEqual(source, patchedast.write_ast(ast_frag))
+
+    @testutils.only_for_versions_higher("3.10")
+    def test_match_node_with_match_sequence_of_parenthesized_patterns(self):
+        source = dedent("""\
+            match x:
+                case (1), (2):
+                    pass
+        """)
+        ast_frag = patchedast.get_patched_ast(source, True)
+        checker = _ResultChecker(self, ast_frag)
+        start = source.index("(1)")
+        checker.check_region("MatchSequence", start, start + len("(1), (2)"))
+        self.assertEqual(source, patchedast.write_ast(ast_frag))
+
     @testutils.only_for_versions_higher("3.10")
     def test_match_node_with_match_sequence_empty_round_parens(self):
         source = dedent("""\