Add a recursion limit to the parser (#24810)

## Summary

Partial fix for #22930.

Without this malicious or machine generated code could cause a stack
overflow with something as simple as `'(' * 5000 + '1' + ')' * 5000`.

I decided to do the simplest thing and have a limit that's always
applied with a reasonable default. Since:
* the overhead of this check will be tiny
* it seems inconceivable that anyone will want to have no limit 

## Test Plan

PR includes tests.

---------

Co-authored-by: Charlie Marsh <charlie.r.marsh@gmail.com>

Author: samuelcolvin

crates/ruff_python_parser/src/error.rs

@@ -200,6 +200,9 @@ pub enum ParseErrorType {
     TStringError(InterpolatedStringErrorType),
     /// Parser encountered an error during lexing.
     Lexical(LexicalErrorType),
+
+    /// Parser aborted because [`crate::ParseOptions::max_recursion_depth`] was exceeded.
+    RecursionLimitExceeded,
 }
 
 impl ParseErrorType {
@@ -329,6 +332,7 @@ impl std::fmt::Display for ParseErrorType {
             ParseErrorType::UnexpectedExpressionToken => {
                 write!(f, "Unexpected token at the end of an expression")
             }
+            ParseErrorType::RecursionLimitExceeded => f.write_str("Source is too deeply nested"),
         }
     }
 }

crates/ruff_python_parser/src/lexer.rs

@@ -131,6 +131,12 @@ impl<'src> Lexer<'src> {
         self.current_range
     }
 
+    /// Returns the current parenthesis, bracket, and brace nesting level.
+    #[inline]
+    pub(crate) const fn nesting(&self) -> u32 {
+        self.nesting
+    }
+
     /// Returns the flags for the current token.
     pub(crate) fn current_flags(&self) -> TokenFlags {
         self.current_flags

crates/ruff_python_parser/src/parser/expression.rs

@@ -300,7 +300,22 @@ impl<'src> Parser<'src> {
                 BinaryLikeOperator::Binary(bin_op) => {
                     self.bump(TokenKind::from(bin_op));
 
-                    let right = self.parse_binary_expression_or_higher(new_precedence, context);
+                    let right = if new_precedence.is_right_associative() {
+                        // For right-associative operators (`**`), the right
+                        // operand recursion is unbounded in `a**a**a**...`,
+                        // and it bypasses the guard in `parse_lhs_expression`
+                        // (that scope is exited once the atom is parsed).
+                        if let Some(right) = self.with_recursion(|parser| {
+                            parser.parse_binary_expression_or_higher(new_precedence, context)
+                        }) {
+                            right
+                        } else {
+                            self.report_recursion_limit_exceeded(self.current_token_range());
+                            self.recursion_recovery_expr()
+                        }
+                    } else {
+                        self.parse_binary_expression_or_higher(new_precedence, context)
+                    };
 
                     Expr::BinOp(ast::ExprBinOp {
                         left: Box::new(left.expr),
@@ -330,8 +345,61 @@ impl<'src> Parser<'src> {
         left_precedence: OperatorPrecedence,
         context: ExpressionContext,
     ) -> ParsedExpr {
-        let start = self.node_start();
         let token = self.current_token_kind();
+        if !Self::token_starts_recursive_lhs(token) {
+            return self.parse_lhs_expression_inner(left_precedence, context, token);
+        }
+
+        if let Some(result) = self.with_recursion(|parser| {
+            parser.parse_lhs_expression_inner(left_precedence, context, token)
+        }) {
+            result
+        } else {
+            self.report_recursion_limit_exceeded(self.current_token_range());
+            self.recursion_recovery_expr()
+        }
+    }
+
+    /// Returns whether parsing an expression that starts with `token` can
+    /// immediately recurse through another expression parse.
+    #[inline]
+    fn token_starts_recursive_lhs(token: TokenKind) -> bool {
+        token.as_unary_operator().is_some()
+            || matches!(
+                token,
+                TokenKind::Star
+                    | TokenKind::Await
+                    | TokenKind::Lambda
+                    | TokenKind::Yield
+                    | TokenKind::FStringStart
+                    | TokenKind::TStringStart
+                    | TokenKind::Lpar
+                    | TokenKind::Lsqb
+                    | TokenKind::Lbrace
+            )
+    }
+
+    /// The standard expression-recovery node returned when the recursion
+    /// limit is exceeded: an empty `Name` with the `Invalid` context.
+    fn recursion_recovery_expr(&mut self) -> ParsedExpr {
+        ParsedExpr {
+            expr: Expr::Name(ast::ExprName {
+                range: self.missing_node_range(),
+                id: Name::empty(),
+                ctx: ExprContext::Invalid,
+                node_index: AtomicNodeIndex::NONE,
+            }),
+            is_parenthesized: false,
+        }
+    }
+
+    fn parse_lhs_expression_inner(
+        &mut self,
+        left_precedence: OperatorPrecedence,
+        context: ExpressionContext,
+        token: TokenKind,
+    ) -> ParsedExpr {
+        let start = self.node_start();
 
         if let Some(unary_op) = token.as_unary_operator() {
             let expr = self.parse_unary_expression(unary_op, context);
@@ -365,7 +433,7 @@ impl<'src> Parser<'src> {
             return Expr::UnaryOp(expr).into();
         }
 
-        match self.current_token_kind() {
+        match token {
             TokenKind::Star => {
                 let starred_expr = self.parse_starred_expression(context);
 
@@ -665,8 +733,20 @@ impl<'src> Parser<'src> {
     pub(super) fn parse_postfix_expression(&mut self, mut lhs: Expr, start: TextSize) -> Expr {
         loop {
             lhs = match self.current_token_kind() {
-                TokenKind::Lpar => Expr::Call(self.parse_call_expression(lhs, start)),
-                TokenKind::Lsqb => Expr::Subscript(self.parse_subscript_expression(lhs, start)),
+                TokenKind::Lpar => {
+                    if self.tokens.nesting() > self.max_nesting_depth {
+                        self.report_recursion_limit_exceeded(self.current_token_range());
+                        break lhs;
+                    }
+                    Expr::Call(self.parse_call_expression(lhs, start))
+                }
+                TokenKind::Lsqb => {
+                    if self.tokens.nesting() > self.max_nesting_depth {
+                        self.report_recursion_limit_exceeded(self.current_token_range());
+                        break lhs;
+                    }
+                    Expr::Subscript(self.parse_subscript_expression(lhs, start))
+                }
                 TokenKind::Dot => Expr::Attribute(self.parse_attribute_expression(lhs, start)),
                 _ => break lhs,
             };
@@ -1802,11 +1882,18 @@ impl<'src> Parser<'src> {
 
         let format_spec = if self.eat(TokenKind::Colon) {
             let spec_start = self.node_start();
-            let elements = self.parse_interpolated_string_elements(
-                flags,
-                InterpolatedStringElementsKind::FormatSpec(string_kind),
-                string_kind,
-            );
+            let elements = if let Some(elements) = self.with_recursion(|parser| {
+                parser.parse_interpolated_string_elements(
+                    flags,
+                    InterpolatedStringElementsKind::FormatSpec(string_kind),
+                    string_kind,
+                )
+            }) {
+                elements
+            } else {
+                self.report_recursion_limit_exceeded(self.current_token_range());
+                ast::InterpolatedStringElements::from(vec![])
+            };
             Some(Box::new(ast::InterpolatedStringFormatSpec {
                 range: self.node_range(spec_start),
                 elements,
@@ -2878,7 +2965,16 @@ impl<'src> Parser<'src> {
         // test_err lambda_body_with_yield_expr
         // lambda x: yield y
         // lambda x: yield from y
-        let body = self.parse_conditional_expression_or_higher();
+
+        // `lambda: lambda: lambda: ...` recurses through the lambda body at
+        // the conditional layer, bypassing the `parse_lhs_expression` guard.
+        let body =
+            if let Some(body) = self.with_recursion(Self::parse_conditional_expression_or_higher) {
+                body
+            } else {
+                self.report_recursion_limit_exceeded(self.current_token_range());
+                self.recursion_recovery_expr()
+            };
 
         ast::ExprLambda {
             body: Box::new(body.expr),
@@ -2902,7 +2998,17 @@ impl<'src> Parser<'src> {
 
         self.expect(TokenKind::Else);
 
-        let orelse = self.parse_conditional_expression_or_higher();
+        // `a if b else a if b else ...` recurses through `orelse` at the
+        // conditional layer, which is not covered by the `parse_lhs_expression`
+        // guard (that scope is released once each atom is parsed). Guard here.
+        let orelse = if let Some(orelse) =
+            self.with_recursion(Self::parse_conditional_expression_or_higher)
+        {
+            orelse
+        } else {
+            self.report_recursion_limit_exceeded(self.current_token_range());
+            self.recursion_recovery_expr()
+        };
 
         ast::ExprIf {
             body: Box::new(body),

crates/ruff_python_parser/src/parser/mod.rs

@@ -1,7 +1,6 @@
 use std::cmp::Ordering;
 
 use bitflags::bitflags;
-
 use ruff_python_ast::token::TokenKind;
 use ruff_python_ast::{AtomicNodeIndex, Mod, ModExpression, ModModule};
 use ruff_text_size::{Ranged, TextRange, TextSize};
@@ -56,6 +55,12 @@ pub(crate) struct Parser<'src> {
 
     /// The start offset in the source code from which to start parsing at.
     start_offset: TextSize,
+
+    /// Current parser recursion depth remaining before the depth limit is exceeded.
+    depth_remaining: u16,
+
+    /// Maximum lexer nesting depth before postfix calls and subscripts should stop recursing.
+    max_nesting_depth: u32,
 }
 
 impl<'src> Parser<'src> {
@@ -71,6 +76,8 @@ impl<'src> Parser<'src> {
         options: ParseOptions,
     ) -> Self {
         let tokens = TokenSource::from_source(source, options.mode, start_offset);
+        let depth_remaining = options.max_recursion_depth;
+        let max_nesting_depth = u32::from(options.max_recursion_depth.saturating_sub(2));
 
         Parser {
             options,
@@ -82,6 +89,38 @@ impl<'src> Parser<'src> {
             prev_token_end: TextSize::new(0),
             start_offset,
             current_token_id: TokenId::default(),
+            depth_remaining,
+            max_nesting_depth,
+        }
+    }
+
+    /// Runs `f` if the recursive parser depth limit has not been hit.
+    ///
+    /// # Note
+    ///
+    /// This recursion guard is a temporary fix for #22930.
+    #[must_use]
+    #[inline]
+    fn with_recursion<T>(&mut self, f: impl FnOnce(&mut Self) -> T) -> Option<T> {
+        if self.depth_remaining == 0 {
+            return None;
+        }
+
+        self.depth_remaining -= 1;
+        let result = f(self);
+        self.depth_remaining += 1;
+        Some(result)
+    }
+
+    #[cold]
+    #[inline(never)]
+    fn report_recursion_limit_exceeded<R: Ranged>(&mut self, ranged: R) {
+        self.add_error(ParseErrorType::RecursionLimitExceeded, ranged);
+        // Skip to end-of-file so outer parser frames unwind quickly and our
+        // `ParserProgress` infinite-loop guards don't fire when they see the
+        // same `(` / `[` etc. that this frame failed to consume.
+        while self.current_token_kind() != TokenKind::EndOfFile {
+            self.bump_any();
         }
     }
 

crates/ruff_python_parser/src/parser/options.rs

@@ -2,6 +2,20 @@ use ruff_python_ast::{PySourceType, PythonVersion};
 
 use crate::{AsMode, Mode};
 
+/// The default maximum recursion depth used by the parser.
+///
+/// Real-world Python rarely nests more than a handful of levels deep; this cap
+/// exists to keep the parser from overflowing the stack on adversarial or
+/// machine-generated input.
+///
+/// The default value mirrors CPython's `MAXSTACK` of 200 nested parentheses
+/// (`Parser/parser.c`): a one-statement module of the form `((((1))))` at
+/// depth 200 must parse, and one at depth 201 must fail. Each nesting level
+/// costs one `with_recursion` call, plus two framing calls (one for the
+/// surrounding statement and one for the innermost atom), so the cap is set
+/// to `200 + 2`.
+const DEFAULT_MAX_RECURSION_DEPTH: u16 = 202;
+
 /// Options for controlling how a source file is parsed.
 ///
 /// You can construct a [`ParseOptions`] directly from a [`Mode`]:
@@ -26,6 +40,11 @@ pub struct ParseOptions {
     pub(crate) mode: Mode,
     /// Target version for detecting version-related syntax errors.
     pub(crate) target_version: PythonVersion,
+    /// Maximum recursion depth for the parser. The parser aborts with a
+    /// [`crate::ParseErrorType::RecursionLimitExceeded`] error once this many
+    /// nested expression / statement / pattern nodes are on the parser's call
+    /// stack. Defaults to [`DEFAULT_MAX_RECURSION_DEPTH`].
+    pub(crate) max_recursion_depth: u16,
 }
 
 impl ParseOptions {
@@ -38,13 +57,25 @@ impl ParseOptions {
     pub fn target_version(&self) -> PythonVersion {
         self.target_version
     }
+
+    /// Set the maximum recursion depth for the parser.
+    #[must_use]
+    pub fn with_max_recursion_depth(mut self, depth: u16) -> Self {
+        self.max_recursion_depth = depth;
+        self
+    }
+
+    pub fn max_recursion_depth(&self) -> u16 {
+        self.max_recursion_depth
+    }
 }
 
 impl From<Mode> for ParseOptions {
     fn from(mode: Mode) -> Self {
         Self {
             mode,
             target_version: PythonVersion::default(),
+            max_recursion_depth: DEFAULT_MAX_RECURSION_DEPTH,
         }
     }
 }
@@ -54,6 +85,7 @@ impl From<PySourceType> for ParseOptions {
         Self {
             mode: source_type.as_mode(),
             target_version: PythonVersion::default(),
+            max_recursion_depth: DEFAULT_MAX_RECURSION_DEPTH,
         }
     }
 }

crates/ruff_python_parser/src/parser/pattern.rs

@@ -88,6 +88,28 @@ impl Parser<'_> {
     ///
     /// See: <https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-pattern>
     fn parse_match_pattern(&mut self, allow_star_pattern: AllowStarPattern) -> Pattern {
+        if let Some(result) =
+            self.with_recursion(|parser| parser.parse_match_pattern_inner(allow_star_pattern))
+        {
+            result
+        } else {
+            let range = self.missing_node_range();
+            self.report_recursion_limit_exceeded(self.current_token_range());
+            let invalid_node = Expr::Name(ast::ExprName {
+                range,
+                id: Name::empty(),
+                ctx: ExprContext::Invalid,
+                node_index: AtomicNodeIndex::NONE,
+            });
+            Pattern::MatchValue(ast::PatternMatchValue {
+                range: invalid_node.range(),
+                value: Box::new(invalid_node),
+                node_index: AtomicNodeIndex::NONE,
+            })
+        }
+    }
+
+    fn parse_match_pattern_inner(&mut self, allow_star_pattern: AllowStarPattern) -> Pattern {
         let start = self.node_start();
 
         // We don't yet know if it's an or pattern or an as pattern, so use whatever