Fix patchedast region for implicitly-concatenated f-strings (fixes #879)

CHANGELOG.md

@@ -8,6 +8,7 @@
 - #623, #819, #863 Support MatchOr, MatchSequence, MatchStar (@jheld, @lieryan)
 - #870 Add default implementation for is_dir() (@lieryan)
 - #872 Fix unicode handling in patchedast (@lieryan)
+- #879 Fix patchedast region for implicitly-concatenated f-strings whose trailing parts hold no interpolation (@stevemadere)
 
 # Release 1.14.0
 

rope/refactor/patchedast.py

@@ -408,6 +408,27 @@ def end_quote_char():
                 children.append(part)
         children.append(end_quote_char())
         self._handle(node, children)
+        self._extend_joined_string_to(node, end)
+
+    def _extend_joined_string_to(self, node, string_end):
+        """Cover concatenation parts that follow the last interpolation.
+
+        `_handle` stops at the first closing quote it finds after the last
+        `FormattedValue`, so in an implicit concatenation whose trailing parts
+        are pure literal text -- ``f"{a} one " f"#52 two"`` -- it stops at the
+        end of the part holding the interpolation.  That both understates the
+        node's region and parks the scanner inside the literal, where an
+        unescaped ``#`` reads as the start of a comment and swallows the rest
+        of the line.  `consume_string` already measured the whole
+        concatenation, so close the gap from its end offset.
+        """
+        if self.source.offset >= string_end:
+            return
+        trailing_parts = self.source[self.source.offset : string_end]
+        if self.children:
+            node.sorted_children.append(trailing_parts)
+        self.source.offset = string_end
+        node.region = (node.region[0], string_end)
 
     def _FormattedValue(self, node):
         children = []

ropetest/refactor/patchedasttest.py

@@ -337,6 +337,48 @@ def test_handling_format_strings_with_implicit_join(self):
         )
         checker.check_children("FormattedValue", ["{", "", "Name", "", "}"])
 
+    @testutils.only_for_versions_higher("3.6")
+    def test_handling_format_strings_with_implicit_join_trailing_literal_part(self):
+        source = 'a = f"one {b}" f"two"\n'
+        ast_frag = patchedast.get_patched_ast(source, True)
+        checker = _ResultChecker(self, ast_frag)
+        checker.check_children(
+            "JoinedStr", ['f"', "one ", "FormattedValue", "", '"', ' f"two"']
+        )
+        checker.check_region("JoinedStr", 4, len('a = f"one {b}" f"two"'))
+
+    @testutils.only_for_versions_higher("3.6")
+    def test_handling_format_strings_with_hash_in_joined_trailing_part(self):
+        source = dedent("""\
+            E(
+                f"{a} one "
+                f"#52 two")
+        """)
+        ast_frag = patchedast.get_patched_ast(source, True)
+        checker = _ResultChecker(self, ast_frag)
+        checker.check_children(
+            "JoinedStr", ['f"', "", "FormattedValue", " one ", '"', '\n    f"#52 two"']
+        )
+        checker.check_children("Call", ["Name", "", "(", "\n    ", "JoinedStr", "", ")"])
+
+    @testutils.only_for_versions_higher("3.6")
+    def test_handling_format_strings_with_hash_in_joined_trailing_part_assignment(self):
+        source = 'a = 1\nb = (f"{a} one "\n     f"#52 two")\n'
+        ast_frag = patchedast.get_patched_ast(source, True)
+        checker = _ResultChecker(self, ast_frag)
+        checker.check_children(
+            "JoinedStr",
+            ['f"', "", "FormattedValue", " one ", '"', '\n     f"#52 two"'],
+        )
+
+    @testutils.only_for_versions_higher("3.6")
+    def test_handling_format_strings_followed_by_a_real_comment(self):
+        source = 'a = f"one {b}"  # trailing comment\nc = 1\n'
+        ast_frag = patchedast.get_patched_ast(source, True)
+        checker = _ResultChecker(self, ast_frag)
+        checker.check_children("JoinedStr", ['f"', "one ", "FormattedValue", "", '"'])
+        checker.check_region("JoinedStr", 4, len('a = f"one {b}"'))
+
     @testutils.only_for_versions_higher("3.6")
     def test_handling_format_strings_with_format_spec(self):
         source = 'f"abc{a:01}"\n'
@@ -2001,6 +2043,12 @@ def __call__(self, node):
         ast.call_for_nodes(self.ast, search)
         return search.result
 
+    def check_region(self, text, start, end):
+        node = self._find_node(text)
+        if node is None:
+            self.test_case.fail("Node <%s> cannot be found" % text)
+        self.test_case.assertEqual((start, end), node.region)
+
     def check_children(self, text, children):
         node = self._find_node(text)
         if node is None: