Improved error recovery for unclosed strings (including f- and t-strings) (#20848)

Author: MichaReiser

crates/ruff_linter/resources/test/fixtures/flake8_implicit_str_concat/ISC_syntax_error.py

@@ -1,10 +1,10 @@
-# The lexer doesn't emit a string token if it's unterminated
+# The lexer emits a string token if it's unterminated
 "a" "b
 "a" "b" "c
 "a" """b
 c""" "d
 
-# For f-strings, the `FStringRanges` won't contain the range for
+# This is also true for
 # unterminated f-strings.
 f"a" f"b
 f"a" f"b" f"c

crates/ruff_linter/src/rules/flake8_implicit_str_concat/snapshots/ruff_linter__rules__flake8_implicit_str_concat__tests__ISC001_ISC_syntax_error.py.snap

@@ -1,30 +1,31 @@
 ---
 source: crates/ruff_linter/src/rules/flake8_implicit_str_concat/mod.rs
 ---
-invalid-syntax: missing closing quote in string literal
- --> ISC_syntax_error.py:2:5
+ISC001 Implicitly concatenated string literals on one line
+ --> ISC_syntax_error.py:2:1
   |
-1 | # The lexer doesn't emit a string token if it's unterminated
+1 | # The lexer emits a string token if it's unterminated
 2 | "a" "b
-  |     ^^
+  | ^^^^^^
 3 | "a" "b" "c
 4 | "a" """b
   |
+help: Combine string literals
 
-invalid-syntax: Expected a statement
- --> ISC_syntax_error.py:2:7
+invalid-syntax: missing closing quote in string literal
+ --> ISC_syntax_error.py:2:5
   |
-1 | # The lexer doesn't emit a string token if it's unterminated
+1 | # The lexer emits a string token if it's unterminated
 2 | "a" "b
-  |       ^
+  |     ^^
 3 | "a" "b" "c
 4 | "a" """b
   |
 
 ISC001 Implicitly concatenated string literals on one line
  --> ISC_syntax_error.py:3:1
   |
-1 | # The lexer doesn't emit a string token if it's unterminated
+1 | # The lexer emits a string token if it's unterminated
 2 | "a" "b
 3 | "a" "b" "c
   | ^^^^^^^
@@ -33,24 +34,25 @@ ISC001 Implicitly concatenated string literals on one line
   |
 help: Combine string literals
 
-invalid-syntax: missing closing quote in string literal
- --> ISC_syntax_error.py:3:9
+ISC001 Implicitly concatenated string literals on one line
+ --> ISC_syntax_error.py:3:5
   |
-1 | # The lexer doesn't emit a string token if it's unterminated
+1 | # The lexer emits a string token if it's unterminated
 2 | "a" "b
 3 | "a" "b" "c
-  |         ^^
+  |     ^^^^^^
 4 | "a" """b
 5 | c""" "d
   |
+help: Combine string literals
 
-invalid-syntax: Expected a statement
- --> ISC_syntax_error.py:3:11
+invalid-syntax: missing closing quote in string literal
+ --> ISC_syntax_error.py:3:9
   |
-1 | # The lexer doesn't emit a string token if it's unterminated
+1 | # The lexer emits a string token if it's unterminated
 2 | "a" "b
 3 | "a" "b" "c
-  |           ^
+  |         ^^
 4 | "a" """b
 5 | c""" "d
   |
@@ -64,36 +66,39 @@ ISC001 Implicitly concatenated string literals on one line
 5 | | c""" "d
   | |____^
 6 |
-7 |   # For f-strings, the `FStringRanges` won't contain the range for
+7 |   # This is also true for
   |
 help: Combine string literals
 
-invalid-syntax: missing closing quote in string literal
- --> ISC_syntax_error.py:5:6
+ISC001 Implicitly concatenated string literals on one line
+ --> ISC_syntax_error.py:4:5
   |
-3 | "a" "b" "c
-4 | "a" """b
-5 | c""" "d
-  |      ^^
+2 |   "a" "b
+3 |   "a" "b" "c
+4 |   "a" """b
+  |  _____^
+5 | | c""" "d
+  | |_______^
 6 |
-7 | # For f-strings, the `FStringRanges` won't contain the range for
+7 |   # This is also true for
   |
+help: Combine string literals
 
-invalid-syntax: Expected a statement
- --> ISC_syntax_error.py:5:8
+invalid-syntax: missing closing quote in string literal
+ --> ISC_syntax_error.py:5:6
   |
 3 | "a" "b" "c
 4 | "a" """b
 5 | c""" "d
-  |        ^
+  |      ^^
 6 |
-7 | # For f-strings, the `FStringRanges` won't contain the range for
+7 | # This is also true for
   |
 
 invalid-syntax: f-string: unterminated string
   --> ISC_syntax_error.py:9:8
    |
- 7 | # For f-strings, the `FStringRanges` won't contain the range for
+ 7 | # This is also true for
  8 | # unterminated f-strings.
  9 | f"a" f"b
    |        ^
@@ -104,7 +109,7 @@ invalid-syntax: f-string: unterminated string
 invalid-syntax: Expected FStringEnd, found newline
   --> ISC_syntax_error.py:9:9
    |
- 7 | # For f-strings, the `FStringRanges` won't contain the range for
+ 7 | # This is also true for
  8 | # unterminated f-strings.
  9 | f"a" f"b
    |         ^
@@ -183,14 +188,6 @@ invalid-syntax: f-string: unterminated triple-quoted string
    | |__^
    |
 
-invalid-syntax: unexpected EOF while parsing
-  --> ISC_syntax_error.py:30:1
-   |
-28 |     "i" "j"
-29 | )
-   |  ^
-   |
-
 invalid-syntax: f-string: unterminated string
   --> ISC_syntax_error.py:30:1
    |

crates/ruff_linter/src/rules/flake8_implicit_str_concat/snapshots/ruff_linter__rules__flake8_implicit_str_concat__tests__ISC002_ISC_syntax_error.py.snap

@@ -4,45 +4,24 @@ source: crates/ruff_linter/src/rules/flake8_implicit_str_concat/mod.rs
 invalid-syntax: missing closing quote in string literal
  --> ISC_syntax_error.py:2:5
   |
-1 | # The lexer doesn't emit a string token if it's unterminated
+1 | # The lexer emits a string token if it's unterminated
 2 | "a" "b
   |     ^^
 3 | "a" "b" "c
 4 | "a" """b
   |
 
-invalid-syntax: Expected a statement
- --> ISC_syntax_error.py:2:7
-  |
-1 | # The lexer doesn't emit a string token if it's unterminated
-2 | "a" "b
-  |       ^
-3 | "a" "b" "c
-4 | "a" """b
-  |
-
 invalid-syntax: missing closing quote in string literal
  --> ISC_syntax_error.py:3:9
   |
-1 | # The lexer doesn't emit a string token if it's unterminated
+1 | # The lexer emits a string token if it's unterminated
 2 | "a" "b
 3 | "a" "b" "c
   |         ^^
 4 | "a" """b
 5 | c""" "d
   |
 
-invalid-syntax: Expected a statement
- --> ISC_syntax_error.py:3:11
-  |
-1 | # The lexer doesn't emit a string token if it's unterminated
-2 | "a" "b
-3 | "a" "b" "c
-  |           ^
-4 | "a" """b
-5 | c""" "d
-  |
-
 invalid-syntax: missing closing quote in string literal
  --> ISC_syntax_error.py:5:6
   |
@@ -51,24 +30,13 @@ invalid-syntax: missing closing quote in string literal
 5 | c""" "d
   |      ^^
 6 |
-7 | # For f-strings, the `FStringRanges` won't contain the range for
-  |
-
-invalid-syntax: Expected a statement
- --> ISC_syntax_error.py:5:8
-  |
-3 | "a" "b" "c
-4 | "a" """b
-5 | c""" "d
-  |        ^
-6 |
-7 | # For f-strings, the `FStringRanges` won't contain the range for
+7 | # This is also true for
   |
 
 invalid-syntax: f-string: unterminated string
   --> ISC_syntax_error.py:9:8
    |
- 7 | # For f-strings, the `FStringRanges` won't contain the range for
+ 7 | # This is also true for
  8 | # unterminated f-strings.
  9 | f"a" f"b
    |        ^
@@ -79,7 +47,7 @@ invalid-syntax: f-string: unterminated string
 invalid-syntax: Expected FStringEnd, found newline
   --> ISC_syntax_error.py:9:9
    |
- 7 | # For f-strings, the `FStringRanges` won't contain the range for
+ 7 | # This is also true for
  8 | # unterminated f-strings.
  9 | f"a" f"b
    |         ^
@@ -133,14 +101,6 @@ invalid-syntax: f-string: unterminated triple-quoted string
    | |__^
    |
 
-invalid-syntax: unexpected EOF while parsing
-  --> ISC_syntax_error.py:30:1
-   |
-28 |     "i" "j"
-29 | )
-   |  ^
-   |
-
 invalid-syntax: f-string: unterminated string
   --> ISC_syntax_error.py:30:1
    |

crates/ruff_linter/src/rules/pylint/snapshots/ruff_linter__rules__pylint__tests__PLE2510_invalid_characters_syntax_error.py.snap

@@ -23,16 +23,17 @@ invalid-syntax: missing closing quote in string literal
 9 | # Unterminated f-string
   |
 
-invalid-syntax: Expected a statement
- --> invalid_characters_syntax_error.py:7:7
+PLE2510 Invalid unescaped character backspace, use "\b" instead
+ --> invalid_characters_syntax_error.py:7:6
   |
 5 | b = '␈'
 6 | # Unterminated string
 7 | b = '␈
-  |       ^
+  |      ^
 8 | b = '␈'
 9 | # Unterminated f-string
   |
+help: Replace with escape sequence
 
 PLE2510 Invalid unescaped character backspace, use "\b" instead
   --> invalid_characters_syntax_error.py:8:6
@@ -46,6 +47,18 @@ PLE2510 Invalid unescaped character backspace, use "\b" instead
    |
 help: Replace with escape sequence
 
+PLE2510 Invalid unescaped character backspace, use "\b" instead
+  --> invalid_characters_syntax_error.py:10:7
+   |
+ 8 | b = '␈'
+ 9 | # Unterminated f-string
+10 | b = f'␈
+   |       ^
+11 | b = f'␈'
+12 | # Implicitly concatenated
+   |
+help: Replace with escape sequence
+
 invalid-syntax: f-string: unterminated string
   --> invalid_characters_syntax_error.py:10:7
    |
@@ -109,11 +122,12 @@ invalid-syntax: missing closing quote in string literal
    |              ^^
    |
 
-invalid-syntax: Expected a statement
-  --> invalid_characters_syntax_error.py:13:16
+PLE2510 Invalid unescaped character backspace, use "\b" instead
+  --> invalid_characters_syntax_error.py:13:15
    |
 11 | b = f'␈'
 12 | # Implicitly concatenated
 13 | b = '␈' f'␈' '␈
-   |                ^
+   |               ^
    |
+help: Replace with escape sequence

crates/ruff_linter/src/rules/pyupgrade/rules/f_strings.rs

@@ -6,7 +6,7 @@ use rustc_hash::{FxHashMap, FxHashSet};
 use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_python_ast::helpers::any_over_expr;
 use ruff_python_ast::str::{leading_quote, trailing_quote};
-use ruff_python_ast::{self as ast, Expr, Keyword};
+use ruff_python_ast::{self as ast, Expr, Keyword, StringFlags};
 use ruff_python_literal::format::{
     FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate,
 };
@@ -430,7 +430,7 @@ pub(crate) fn f_strings(checker: &Checker, call: &ast::ExprCall, summary: &Forma
                 // dot is the start of an attribute access.
                 break token.start();
             }
-            TokenKind::String => {
+            TokenKind::String if !token.unwrap_string_flags().is_unclosed() => {
                 match FStringConversion::try_convert(token.range(), &mut summary, checker.locator())
                 {
                     // If the format string contains side effects that would need to be repeated,