Skip to content

Commit acc5662

Browse files
ntBrentBre
authored
[syntax-errors] Allow yield in base classes and annotations (#17206)
Summary -- This PR fixes the issue pointed out by @JelleZijlstra in #17101 (comment). Namely, I conflated two very different errors from CPython: ```pycon >>> def m[T](x: (yield from 1)): ... File "<python-input-310>", line 1 def m[T](x: (yield from 1)): ... ^^^^^^^^^^^^ SyntaxError: yield expression cannot be used within the definition of a generic >>> def m(x: (yield from 1)): ... File "<python-input-311>", line 1 def m(x: (yield from 1)): ... ^^^^^^^^^^^^ SyntaxError: 'yield from' outside function >>> def outer(): ... def m(x: (yield from 1)): ... ... >>> ``` I thought the second error was the same as the first, but `yield` (and `yield from`) is actually valid in this position when inside a function scope. The same is true for base classes, as pointed out in the original comment. We don't currently raise an error for `yield` outside of a function, but that should be handled separately. On the upside, this had the benefit of removing the `InvalidExpressionPosition::BaseClass` variant and the `allow_named_expr` field from the visitor because they were both no longer used. Test Plan -- Updated inline tests.
1 parent 33a56f1 commit acc5662

16 files changed

Lines changed: 1074 additions & 679 deletions

‎crates/ruff_linter/src/checkers/ast/mod.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -587,6 +587,10 @@ impl SemanticSyntaxContext for Checker<'_> {
587587
fn source(&self) -> &str {
588588
self.source()
589589
}
590+
591+
fn future_annotations_or_stub(&self) -> bool {
592+
self.semantic.future_annotations_or_stub()
593+
}
590594
}
591595

592596
impl<'a> Visitor<'a> for Checker<'a> {

‎crates/ruff_python_parser/resources/inline/err/invalid_annotation_class.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
11
class F[T](y := list): ...
2-
class G((yield 1)): ...
3-
class H((yield from 1)): ...
42
class I[T]((yield 1)): ...
53
class J[T]((yield from 1)): ...
64
class K[T: (yield 1)]: ... # yield in TypeVar

‎crates/ruff_python_parser/resources/inline/err/invalid_annotation_function.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,9 @@
11
def f[T]() -> (y := 3): ...
22
def g[T](arg: (x := 1)): ...
33
def h[T](x: (yield 1)): ...
4-
def i(x: (yield 1)): ...
54
def j[T]() -> (yield 1): ...
6-
def k() -> (yield 1): ...
75
def l[T](x: (yield from 1)): ...
8-
def m(x: (yield from 1)): ...
96
def n[T]() -> (yield from 1): ...
10-
def o() -> (yield from 1): ...
117
def p[T: (yield 1)](): ... # yield in TypeVar bound
128
def q[T = (yield 1)](): ... # yield in TypeVar default
139
def r[*Ts = (yield 1)](): ... # yield in TypeVarTuple default

‎crates/ruff_python_parser/resources/inline/err/invalid_annotation_function_py314.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# parse_options: {"target-version": "3.14"}
2+
def f() -> (y := 3): ...
3+
def g(arg: (x := 1)): ...
4+
def outer():
5+
def i(x: (yield 1)): ...
6+
def k() -> (yield 1): ...
7+
def m(x: (yield from 1)): ...
8+
def o() -> (yield from 1): ...

‎crates/ruff_python_parser/resources/inline/ok/valid_annotation_class.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,4 @@
11
class F(y := list): ...
2+
def f():
3+
class G((yield 1)): ...
4+
class H((yield from 1)): ...

‎crates/ruff_python_parser/resources/inline/ok/valid_annotation_function.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,7 @@
11
def f() -> (y := 3): ...
22
def g(arg: (x := 1)): ...
3+
def outer():
4+
def i(x: (yield 1)): ...
5+
def k() -> (yield 1): ...
6+
def m(x: (yield from 1)): ...
7+
def o() -> (yield from 1): ...

‎crates/ruff_python_parser/src/semantic_errors.rs‎

Lines changed: 30 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -128,18 +128,29 @@ impl SemanticSyntaxChecker {
128128
// test_ok valid_annotation_function
129129
// def f() -> (y := 3): ...
130130
// def g(arg: (x := 1)): ...
131+
// def outer():
132+
// def i(x: (yield 1)): ...
133+
// def k() -> (yield 1): ...
134+
// def m(x: (yield from 1)): ...
135+
// def o() -> (yield from 1): ...
136+
137+
// test_err invalid_annotation_function_py314
138+
// # parse_options: {"target-version": "3.14"}
139+
// def f() -> (y := 3): ...
140+
// def g(arg: (x := 1)): ...
141+
// def outer():
142+
// def i(x: (yield 1)): ...
143+
// def k() -> (yield 1): ...
144+
// def m(x: (yield from 1)): ...
145+
// def o() -> (yield from 1): ...
131146

132147
// test_err invalid_annotation_function
133148
// def f[T]() -> (y := 3): ...
134149
// def g[T](arg: (x := 1)): ...
135150
// def h[T](x: (yield 1)): ...
136-
// def i(x: (yield 1)): ...
137151
// def j[T]() -> (yield 1): ...
138-
// def k() -> (yield 1): ...
139152
// def l[T](x: (yield from 1)): ...
140-
// def m(x: (yield from 1)): ...
141153
// def n[T]() -> (yield from 1): ...
142-
// def o() -> (yield from 1): ...
143154
// def p[T: (yield 1)](): ... # yield in TypeVar bound
144155
// def q[T = (yield 1)](): ... # yield in TypeVar default
145156
// def r[*Ts = (yield 1)](): ... # yield in TypeVarTuple default
@@ -148,19 +159,20 @@ impl SemanticSyntaxChecker {
148159
// def u[T = (x := 1)](): ... # named expr in TypeVar default
149160
// def v[*Ts = (x := 1)](): ... # named expr in TypeVarTuple default
150161
// def w[**Ts = (x := 1)](): ... # named expr in ParamSpec default
151-
let is_generic = type_params.is_some();
152162
let mut visitor = InvalidExpressionVisitor {
153-
allow_named_expr: !is_generic,
154163
position: InvalidExpressionPosition::TypeAnnotation,
155164
ctx,
156165
};
157166
if let Some(type_params) = type_params {
158167
visitor.visit_type_params(type_params);
159168
}
160-
if is_generic {
169+
// the __future__ annotation error takes precedence over the generic error
170+
if ctx.future_annotations_or_stub() || ctx.python_version() > PythonVersion::PY313 {
171+
visitor.position = InvalidExpressionPosition::TypeAnnotation;
172+
} else if type_params.is_some() {
161173
visitor.position = InvalidExpressionPosition::GenericDefinition;
162174
} else {
163-
visitor.position = InvalidExpressionPosition::TypeAnnotation;
175+
return;
164176
}
165177
for param in parameters
166178
.iter()
@@ -173,36 +185,29 @@ impl SemanticSyntaxChecker {
173185
}
174186
}
175187
Stmt::ClassDef(ast::StmtClassDef {
176-
type_params,
188+
type_params: Some(type_params),
177189
arguments,
178190
..
179191
}) => {
180192
// test_ok valid_annotation_class
181193
// class F(y := list): ...
194+
// def f():
195+
// class G((yield 1)): ...
196+
// class H((yield from 1)): ...
182197

183198
// test_err invalid_annotation_class
184199
// class F[T](y := list): ...
185-
// class G((yield 1)): ...
186-
// class H((yield from 1)): ...
187200
// class I[T]((yield 1)): ...
188201
// class J[T]((yield from 1)): ...
189202
// class K[T: (yield 1)]: ... # yield in TypeVar
190203
// class L[T: (x := 1)]: ... # named expr in TypeVar
191-
let is_generic = type_params.is_some();
192204
let mut visitor = InvalidExpressionVisitor {
193-
allow_named_expr: !is_generic,
194205
position: InvalidExpressionPosition::TypeAnnotation,
195206
ctx,
196207
};
197-
if let Some(type_params) = type_params {
198-
visitor.visit_type_params(type_params);
199-
}
200-
if is_generic {
201-
visitor.position = InvalidExpressionPosition::GenericDefinition;
202-
} else {
203-
visitor.position = InvalidExpressionPosition::BaseClass;
204-
}
208+
visitor.visit_type_params(type_params);
205209
if let Some(arguments) = arguments {
210+
visitor.position = InvalidExpressionPosition::GenericDefinition;
206211
visitor.visit_arguments(arguments);
207212
}
208213
}
@@ -217,7 +222,6 @@ impl SemanticSyntaxChecker {
217222
// type Y = (yield 1) # yield in value
218223
// type Y = (x := 1) # named expr in value
219224
let mut visitor = InvalidExpressionVisitor {
220-
allow_named_expr: false,
221225
position: InvalidExpressionPosition::TypeAlias,
222226
ctx,
223227
};
@@ -625,12 +629,6 @@ impl Display for SemanticSyntaxError {
625629
write!(f, "cannot delete `__debug__` on Python {python_version} (syntax was removed in 3.9)")
626630
}
627631
},
628-
SemanticSyntaxErrorKind::InvalidExpression(
629-
kind,
630-
InvalidExpressionPosition::BaseClass,
631-
) => {
632-
write!(f, "{kind} cannot be used as a base class")
633-
}
634632
SemanticSyntaxErrorKind::InvalidExpression(kind, position) => {
635633
write!(f, "{kind} cannot be used within a {position}")
636634
}
@@ -858,7 +856,6 @@ pub enum InvalidExpressionPosition {
858856
TypeVarTupleDefault,
859857
ParamSpecDefault,
860858
TypeAnnotation,
861-
BaseClass,
862859
GenericDefinition,
863860
TypeAlias,
864861
}
@@ -872,7 +869,6 @@ impl Display for InvalidExpressionPosition {
872869
InvalidExpressionPosition::ParamSpecDefault => "ParamSpec default",
873870
InvalidExpressionPosition::TypeAnnotation => "type annotation",
874871
InvalidExpressionPosition::GenericDefinition => "generic definition",
875-
InvalidExpressionPosition::BaseClass => "base class",
876872
InvalidExpressionPosition::TypeAlias => "type alias",
877873
})
878874
}
@@ -1086,16 +1082,6 @@ impl<'a, Ctx: SemanticSyntaxContext> MatchPatternVisitor<'a, Ctx> {
10861082
}
10871083

10881084
struct InvalidExpressionVisitor<'a, Ctx> {
1089-
/// Allow named expressions (`x := ...`) to appear in annotations.
1090-
///
1091-
/// These are allowed in non-generic functions, for example:
1092-
///
1093-
/// ```python
1094-
/// def foo(arg: (x := int)): ... # ok
1095-
/// def foo[T](arg: (x := int)): ... # syntax error
1096-
/// ```
1097-
allow_named_expr: bool,
1098-
10991085
/// Context used for emitting errors.
11001086
ctx: &'a Ctx,
11011087

@@ -1108,7 +1094,7 @@ where
11081094
{
11091095
fn visit_expr(&mut self, expr: &Expr) {
11101096
match expr {
1111-
Expr::Named(ast::ExprNamed { range, .. }) if !self.allow_named_expr => {
1097+
Expr::Named(ast::ExprNamed { range, .. }) => {
11121098
SemanticSyntaxChecker::add_error(
11131099
self.ctx,
11141100
SemanticSyntaxErrorKind::InvalidExpression(
@@ -1166,6 +1152,9 @@ pub trait SemanticSyntaxContext {
11661152
/// Returns `true` if a module's docstring boundary has been passed.
11671153
fn seen_docstring_boundary(&self) -> bool;
11681154

1155+
/// Returns `true` if `__future__`-style type annotations are enabled.
1156+
fn future_annotations_or_stub(&self) -> bool;
1157+
11691158
/// The target Python version for detecting backwards-incompatible syntax changes.
11701159
fn python_version(&self) -> PythonVersion;
11711160

‎crates/ruff_python_parser/tests/fixtures.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -490,6 +490,10 @@ impl SemanticSyntaxContext for TestContext<'_> {
490490
false
491491
}
492492

493+
fn future_annotations_or_stub(&self) -> bool {
494+
false
495+
}
496+
493497
fn python_version(&self) -> PythonVersion {
494498
self.python_version
495499
}

‎crates/ruff_python_parser/tests/snapshots/invalid_syntax@function_def_invalid_return_expr.py.snap‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -179,13 +179,3 @@ Module(
179179
3 | def foo() -> yield x: ...
180180
| ^^^^^^^ Syntax Error: Yield expression cannot be used here
181181
|
182-
183-
184-
## Semantic Syntax Errors
185-
186-
|
187-
1 | def foo() -> *int: ...
188-
2 | def foo() -> (*int): ...
189-
3 | def foo() -> yield x: ...
190-
| ^^^^^^^ Syntax Error: yield expression cannot be used within a type annotation
191-
|

0 commit comments

Comments
 (0)
⚡