From 770a416042566662c088de543c6b70329e46b328 Mon Sep 17 00:00:00 2001 From: Marlon Costa Date: Fri, 25 Sep 2026 12:28:51 -0300 Subject: [PATCH 1/3] Bound f-strings and match-sequence delimiters by parser coordinates A PEP 701 replacement field may span lines, which the textual string pattern cannot match, so JoinedStr raised MismatchedTokenError; its region now comes from the node's own coordinates, implicit concatenation included. A MatchSequence took every character before its first pattern as its opening delimiter, so a grouped first pattern (`((a | b), c)`) produced `((`; the delimiters are now the bracket pair that encloses the whole node, found by tokenizing its segment. Co-Authored-By: Claude Opus 5.5 --- rope/refactor/patchedast.py | 53 +++++++++++++++++------------ ropetest/refactor/patchedasttest.py | 38 +++++++++++++++++++++ 2 files changed, 69 insertions(+), 22 deletions(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index 2a52fc924..c5d5a13d1 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -1,6 +1,8 @@ import collections +import io import numbers import re +import tokenize import warnings from itertools import chain @@ -395,11 +397,10 @@ def end_quote_char(): return self.source[end - len(quote_char) : end] QUOTE_CHARS = ['"""', "'''", '"', "'"] - offset = self.source.offset - start, end = self.source.consume_string( - end=self._find_next_statement_start(), - ) - self.source.offset = offset + # The parser's coordinates bound the whole literal, implicit + # concatenation included. A textual string pattern cannot: since + # PEP 701 a replacement field may span lines (``f"{\n x\n}"``). + start, end = self.ast_adapter[node] children = [] children.append(start_quote_char()) @@ -817,24 +818,32 @@ def _MatchSequence(self, node): self._handle(node, children) def _get_surrounding_parens(self, node: ast.MatchSequence): + """The sequence's own ``[...]``/``(...)`` delimiters, if it has them. + + The text between the node's start and its first pattern can also + hold grouping parentheses of that pattern -- ``case (("a" | b), c)`` + -- which the AST does not represent. The sequence is delimited only + when the bracket at its first character is the one closed at its + last character; ``case (1), (2)`` starts and ends with parentheses + that belong to its elements. + """ node_start, node_end = self.ast_adapter[node] - first_pattern_start, _ = self.ast_adapter[node.patterns[0]] - _, last_pattern_end = self.ast_adapter[node.patterns[-1]] - opening_paren = self.source[node_start:first_pattern_start].strip() - closing_paren = self.source[last_pattern_end:node_end].strip() - - if opening_paren not in ["[", "(", ""]: - warnings.warn( - f"Unexpected character in MatchSequence's opening_paren <{opening_paren}>; please report!", - RuntimeWarning, - ) - - if closing_paren not in ["]", ")", ""]: - warnings.warn( - f"Unexpected character in MatchSequence's closing_paren <{closing_paren}>; please report!", - RuntimeWarning, - ) - return opening_paren, closing_paren + segment = self.source[node_start:node_end] + pairs = {"[": "]", "(": ")"} + if segment[:1] not in pairs or segment[-1:] != pairs[segment[0]]: + return "", "" + depth = 0 + for token in tokenize.generate_tokens(io.StringIO(segment).readline): + if token.type != tokenize.OP or token.string not in "()[]{}": + continue + depth += 1 if token.string in "([{" else -1 + if depth == 0: + end_row = segment.count("\n") + 1 + end_col = len(segment) - (segment.rfind("\n") + 1) + if token.end != (end_row, end_col): + return "", "" + return segment[0], segment[-1] + return "", "" def _MatchStar(self, node): self._handle(node, ["*", node.name or "_"]) diff --git a/ropetest/refactor/patchedasttest.py b/ropetest/refactor/patchedasttest.py index c014f6ed1..bfcfb0641 100644 --- a/ropetest/refactor/patchedasttest.py +++ b/ropetest/refactor/patchedasttest.py @@ -326,6 +326,15 @@ def test_handling_format_strings_basic(self): checker.check_children("JoinedStr", ['f"', "abc", "FormattedValue", "", '"']) checker.check_children("FormattedValue", ["{", "", "Name", "", "}"]) + @testutils.only_for_versions_higher("3.12") + def test_handling_format_strings_with_multiline_replacement_field(self): + source = 'x = (f"{a}{\n b\n}{c}").upper()\n' + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + start = source.index('f"') + checker.check_region("JoinedStr", start, source.index(").")) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + @testutils.only_for_versions_higher("3.6") def test_handling_format_strings_with_implicit_join(self): source = '''"1" + rf'abc{a}' f"""xxx{b} """\n''' @@ -1557,6 +1566,35 @@ def test_match_node_with_match_sequence_with_internal_parens(self): "MatchSequence", "", ",", " ", "MatchSequence" ]) + @testutils.only_for_versions_higher("3.10") + def test_match_node_with_match_sequence_with_grouped_first_pattern(self): + source = dedent("""\ + match x: + case (("a" | None), None): + pass + """) + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + checker.check_children("MatchSequence", [ + "(", "(", "MatchOr", ")", ",", " ", "MatchSingleton", "", ")", + ]) + start = source.index("((") + checker.check_region("MatchSequence", start, source.index(":\n", start)) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + + @testutils.only_for_versions_higher("3.10") + def test_match_node_with_match_sequence_of_parenthesized_patterns(self): + source = dedent("""\ + match x: + case (1), (2): + pass + """) + ast_frag = patchedast.get_patched_ast(source, True) + checker = _ResultChecker(self, ast_frag) + start = source.index("(1)") + checker.check_region("MatchSequence", start, start + len("(1), (2)")) + self.assertEqual(source, patchedast.write_ast(ast_frag)) + @testutils.only_for_versions_higher("3.10") def test_match_node_with_match_sequence_empty_round_parens(self): source = dedent("""\ From d8b776f01cedf45d98068dbbbd0848e913c9e7f1 Mon Sep 17 00:00:00 2001 From: Marlon Costa Date: Fri, 25 Sep 2026 12:33:41 -0300 Subject: [PATCH 2/3] Start replacement fields at their parser coordinate In an implicit concatenation mixing f-strings and plain strings, the field's opening brace was searched for as text and matched a "{" inside a plain part (`"({})," f"{a}"`), shifting the field's region and duplicating source on write. Since Python 3.12 the parser locates each FormattedValue at its own "{"; the walker now starts there when the coordinate points at one. test_extract_method_f_string_false_format_value _in_regular_string, marked as an expected failure for this bug, passes. Co-Authored-By: Claude Opus 5.5 --- rope/refactor/patchedast.py | 8 ++++++++ ropetest/refactor/extracttest.py | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/rope/refactor/patchedast.py b/rope/refactor/patchedast.py index c5d5a13d1..e20c40609 100644 --- a/rope/refactor/patchedast.py +++ b/rope/refactor/patchedast.py @@ -411,6 +411,14 @@ def end_quote_char(): self._handle(node, children) def _FormattedValue(self, node): + # Since Python 3.12 the parser locates each replacement field at its + # own "{". Starting there keeps a "{" inside a plain string of the + # same implicit concatenation -- ``"({})," f"{a}"`` -- from being + # taken as the field's opening brace. Older parsers give the + # enclosing literal's position, which does not point at a "{". + field_start, _ = self.ast_adapter[node] + if field_start >= self.source.offset and self.source[field_start] == "{": + self.source.offset = field_start children = [] children.append("{") children.append(node.value) diff --git a/ropetest/refactor/extracttest.py b/ropetest/refactor/extracttest.py index 4b34d58d3..4c063adbc 100644 --- a/ropetest/refactor/extracttest.py +++ b/ropetest/refactor/extracttest.py @@ -1284,7 +1284,6 @@ def new_func(a_var): """) self.assertEqual(expected, refactored) - @unittest.expectedFailure @testutils.only_for_versions_higher("3.6") def test_extract_method_f_string_false_format_value_in_regular_string(self): code = dedent("""\ From 94603cbdd5167a0b9e1d63f436c953db7d613705 Mon Sep 17 00:00:00 2001 From: Marlon Costa Date: Fri, 25 Sep 2026 12:35:01 -0300 Subject: [PATCH 3/3] Add CHANGELOG entry for parser-coordinate patchedast fixes Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1650aa77a..c2a284e5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - #623, #819, #863 Support MatchOr, MatchSequence, MatchStar (@jheld, @lieryan) - #870 Add default implementation for is_dir() (@lieryan) - #872 Fix unicode handling in patchedast (@lieryan) +- Locate multi-line f-strings, replacement fields and match-sequence delimiters in patchedast by parser coordinates # Release 1.14.0