Skip to content

Commit a012e2b

Browse files
MichaReiserMichaReiser
committed
Use same heuristic for triple quoted strings
1 parent f8d5130 commit a012e2b

6 files changed

Lines changed: 53 additions & 90 deletions

File tree

‎crates/ruff_python_parser/src/lexer.rs‎

Lines changed: 22 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -995,7 +995,7 @@ impl<'src> Lexer<'src> {
995995
self.current_flags |= TokenFlags::UNCLOSED_STRING;
996996

997997
self.push_error(LexicalError::new(
998-
LexicalErrorType::StringError,
998+
LexicalErrorType::UnclosedStringError,
999999
self.token_range(),
10001000
));
10011001

@@ -1508,39 +1508,32 @@ impl<'src> Lexer<'src> {
15081508
return;
15091509
};
15101510

1511-
if interpolated_string.is_triple_quoted() {
1512-
return;
1513-
}
1511+
let current_string_flags = self.current_flags().as_any_string_flags();
15141512

1515-
// Only unclosed strings, that have the same quote character and aren't triple-quoted
1513+
// Only unclosed strings, that have the same quote character
15161514
if !matches!(self.current_kind, TokenKind::String)
15171515
|| !self.current_flags.is_unclosed()
1518-
|| self.current_flags.prefix() != AnyStringPrefix::Regular(StringLiteralPrefix::Empty)
1519-
|| self.current_flags.quote_style().as_char() != interpolated_string.quote_char()
1520-
|| self.current_flags.is_triple_quoted()
1516+
|| current_string_flags.prefix() != AnyStringPrefix::Regular(StringLiteralPrefix::Empty)
1517+
|| current_string_flags.quote_style().as_char() != interpolated_string.quote_char()
1518+
|| current_string_flags.is_triple_quoted() != interpolated_string.is_triple_quoted()
15211519
{
15221520
return;
15231521
}
15241522

1525-
let TokenValue::String(value) = &self.current_value else {
1526-
return;
1527-
};
1528-
1529-
for c in value.chars() {
1530-
if is_python_whitespace(c) {
1531-
continue;
1532-
}
1523+
// Only if the string's first line only contains whitespace,
1524+
// or ends in a comment (not `f"{"abc`)
1525+
let first_line = &self.source
1526+
[(self.current_range.start() + current_string_flags.quote_len()).to_usize()..];
15331527

1534-
if c == '#' {
1528+
for c in first_line.chars() {
1529+
if matches!(dbg!(c), '\n' | '\r' | '#') {
15351530
break;
15361531
}
15371532

15381533
// `f'{'ab`, we want to parse `ab` as a normal string and not the closing element of the f-string
1539-
return;
1540-
}
1541-
1542-
if !matches!(self.cursor.first(), '\n' | '\r' | EOF_CHAR) {
1543-
return;
1534+
if !is_python_whitespace(c) {
1535+
return;
1536+
}
15441537
}
15451538

15461539
if self.errors.last().is_some_and(|error| {
@@ -1550,13 +1543,17 @@ impl<'src> Lexer<'src> {
15501543
self.errors.pop();
15511544
}
15521545

1553-
self.current_range = TextRange::at(self.current_range.start(), TextSize::new(1));
1546+
self.current_range =
1547+
TextRange::at(self.current_range.start(), self.current_flags.quote_len());
15541548
self.current_kind = kind.end_token();
15551549
self.current_value = TokenValue::None;
15561550
self.current_flags = TokenFlags::empty();
15571551

15581552
self.nesting = interpolated_string.nesting();
15591553
self.interpolated_strings.pop();
1554+
1555+
self.cursor = Cursor::new(self.source);
1556+
self.cursor.skip_bytes(self.current_range.end().to_usize());
15601557
}
15611558

15621559
/// Re-lex `r"` in a format specifier position.
@@ -2958,7 +2955,7 @@ t"{(lambda x:{x})}"
29582955
```
29592956
[
29602957
LexicalError {
2961-
error: StringError,
2958+
error: UnclosedStringError,
29622959
location: 3..4,
29632960
},
29642961
LexicalError {
@@ -3020,7 +3017,7 @@ t"{(lambda x:{x})}"
30203017
```
30213018
[
30223019
LexicalError {
3023-
error: StringError,
3020+
error: UnclosedStringError,
30243021
location: 7..9,
30253022
},
30263023
LexicalError {

‎crates/ruff_python_parser/src/parser/expression.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1721,6 +1721,13 @@ impl<'src> Parser<'src> {
17211721
let start = self.node_start();
17221722
self.bump(TokenKind::Lbrace);
17231723

1724+
// Before newline is incorrect too for multiline strings. We need to
1725+
// check if the quotes on the first line are only followed by whitespace
1726+
1727+
// TODO: What about multiline? Should we move the assertion that it's before a newline here?
1728+
// The check in `re_lex_string_token_in_interpolation_element` whether it's positioned
1729+
// at a newline is also incorrect if the string is multiline as it doesn't check
1730+
// the end of the first line.
17241731
self.tokens
17251732
.re_lex_string_token_in_interpolation_element(string_kind);
17261733

‎crates/ruff_python_parser/tests/fixtures.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -277,8 +277,7 @@ fn extract_options(source: &str) -> Option<ParseOptions> {
277277
#[expect(clippy::print_stdout)]
278278
fn parser_quick_test() {
279279
let source = r#"
280-
f"hello
281-
"#;
280+
f"{""#;
282281

283282
let parsed = parse_unchecked(source, ParseOptions::from(Mode::Module));
284283

‎crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_unclosed_lbrace.py.snap‎

Lines changed: 11 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -206,41 +206,28 @@ Module(
206206
Expr(
207207
StmtExpr {
208208
node_index: NodeIndex(None),
209-
range: 29..38,
209+
range: 29..37,
210210
value: FString(
211211
ExprFString {
212212
node_index: NodeIndex(None),
213-
range: 29..38,
213+
range: 29..37,
214214
value: FStringValue {
215215
inner: Single(
216216
FString(
217217
FString {
218-
range: 29..38,
218+
range: 29..37,
219219
node_index: NodeIndex(None),
220220
elements: [
221221
Interpolation(
222222
InterpolatedElement {
223-
range: 33..38,
223+
range: 33..34,
224224
node_index: NodeIndex(None),
225-
expression: StringLiteral(
226-
ExprStringLiteral {
225+
expression: Name(
226+
ExprName {
227227
node_index: NodeIndex(None),
228-
range: 34..38,
229-
value: StringLiteralValue {
230-
inner: Single(
231-
StringLiteral {
232-
range: 34..38,
233-
node_index: NodeIndex(None),
234-
value: "\n",
235-
flags: StringLiteralFlags {
236-
quote_style: Double,
237-
prefix: Empty,
238-
triple_quoted: true,
239-
unclosed: true,
240-
},
241-
},
242-
),
243-
},
228+
range: 34..34,
229+
id: Name(""),
230+
ctx: Invalid,
244231
},
245232
),
246233
debug_text: None,
@@ -253,7 +240,7 @@ Module(
253240
quote_style: Double,
254241
prefix: Regular,
255242
triple_quoted: true,
256-
unclosed: true,
243+
unclosed: false,
257244
},
258245
},
259246
),
@@ -309,12 +296,5 @@ Module(
309296
3 | f"{foo="
310297
4 | f"{"
311298
5 | f"""{"""
312-
| ^^^ Syntax Error: missing closing quote in string literal
313-
|
314-
315-
316-
|
317-
4 | f"{"
318-
5 | f"""{"""
319-
| ^ Syntax Error: f-string: unterminated string
299+
| ^^^ Syntax Error: Expected an expression
320300
|

‎crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__fstring_format_spec_1.py.snap‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -513,5 +513,5 @@ Module(
513513
|
514514
11 | f'middle {'string':\\\
515515
12 | 'format spec'}
516-
| ^^ Syntax Error: Got unexpected string
516+
| ^^ Syntax Error: missing closing quote in string literal
517517
|

‎crates/ruff_python_parser/tests/snapshots/invalid_syntax@t_string_unclosed_lbrace.py.snap‎

Lines changed: 11 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -198,40 +198,27 @@ Module(
198198
Expr(
199199
StmtExpr {
200200
node_index: NodeIndex(None),
201-
range: 73..82,
201+
range: 73..81,
202202
value: TString(
203203
ExprTString {
204204
node_index: NodeIndex(None),
205-
range: 73..82,
205+
range: 73..81,
206206
value: TStringValue {
207207
inner: Single(
208208
TString {
209-
range: 73..82,
209+
range: 73..81,
210210
node_index: NodeIndex(None),
211211
elements: [
212212
Interpolation(
213213
InterpolatedElement {
214-
range: 77..82,
214+
range: 77..78,
215215
node_index: NodeIndex(None),
216-
expression: StringLiteral(
217-
ExprStringLiteral {
216+
expression: Name(
217+
ExprName {
218218
node_index: NodeIndex(None),
219-
range: 78..82,
220-
value: StringLiteralValue {
221-
inner: Single(
222-
StringLiteral {
223-
range: 78..82,
224-
node_index: NodeIndex(None),
225-
value: "\n",
226-
flags: StringLiteralFlags {
227-
quote_style: Double,
228-
prefix: Empty,
229-
triple_quoted: true,
230-
unclosed: true,
231-
},
232-
},
233-
),
234-
},
219+
range: 78..78,
220+
id: Name(""),
221+
ctx: Invalid,
235222
},
236223
),
237224
debug_text: None,
@@ -244,7 +231,7 @@ Module(
244231
quote_style: Double,
245232
prefix: Regular,
246233
triple_quoted: true,
247-
unclosed: true,
234+
unclosed: false,
248235
},
249236
},
250237
),
@@ -301,12 +288,5 @@ Module(
301288
4 | t"{foo="
302289
5 | t"{"
303290
6 | t"""{"""
304-
| ^^^ Syntax Error: missing closing quote in string literal
305-
|
306-
307-
308-
|
309-
5 | t"{"
310-
6 | t"""{"""
311-
| ^ Syntax Error: t-string: unterminated string
291+
| ^^^ Syntax Error: Expected an expression
312292
|

0 commit comments

Comments
 (0)
⚡