[ty] fix bug in string annotations and clean up diagnostics (#22913)

This multiplication was always using only using the larger sub-ast size
value, which didn't do anything unsound but made sub-string annotations
become exhausted on files 32x smaller than expected (16k nodes instead
of 512k nodes).

Also I decided to do a cleanup pass on the diagnostics to make them more
precise and helpful.

Author: Gankra

crates/ruff_db/src/parsed.rs

@@ -60,15 +60,21 @@ pub fn parsed_string_annotation(
     indexed::ensure_indexed(&expr, string.node_index().load()).map_err(|err| {
         let message = match err {
             NodeIndexError::NoParent => {
-                "Internal Error: string annotation's parent had no NodeIndex".to_owned()
+                "internal error: string annotation's parent had no NodeIndex".to_owned()
             }
-            NodeIndexError::OutOfIndices => {
-                "File too long, ran out of encoding space for string annotations".to_owned()
+            NodeIndexError::TooNested => "too many levels of nested string annotations; remove the redundant nested quotes".to_owned(),
+            NodeIndexError::OverflowedIndices => {
+                "file too long for string annotations; either break up the file or don't use string annotations".to_owned()
             }
-            NodeIndexError::OutOfSubIndices => {
-                "Substring annotation is too complex, ran out of encoding space".to_owned()
+            NodeIndexError::OverflowedSubIndices => {
+                "file too long for nested string annotations; remove the redundant nested quotes".to_owned()
+            }
+            NodeIndexError::ExhaustedSubIndices => {
+                "string annotation is too long; consider introducing type aliases to simplify".to_owned()
+            }
+            NodeIndexError::ExhaustedSubSubIndices => {
+                "nested string annotation is too long; remove the redundant nested quotes".to_owned()
             }
-            NodeIndexError::TooNested => "Too many levels of nested string annotations".to_owned(),
         };
 
         ParseError {
@@ -222,7 +228,12 @@ mod indexed {
         AnyNodeRef::from(parsed.syntax()).visit_source_order(&mut visitor);
 
         if visitor.overflowed {
-            return Err(NodeIndexError::OutOfSubIndices);
+            let level = sub_ast_level(parent_index);
+            if level == 0 {
+                return Err(NodeIndexError::ExhaustedSubIndices);
+            } else {
+                return Err(NodeIndexError::ExhaustedSubSubIndices);
+            }
         }
 
         Ok(())

crates/ruff_python_ast/src/node_index.rs

@@ -78,11 +78,14 @@ impl NodeIndex {
     }
 }
 
+#[derive(Debug, Copy, Clone)]
 pub enum NodeIndexError {
-    TooNested,
-    OutOfSubIndices,
     NoParent,
-    OutOfIndices,
+    TooNested,
+    ExhaustedSubIndices,
+    ExhaustedSubSubIndices,
+    OverflowedIndices,
+    OverflowedSubIndices,
 }
 
 const MAX_LEVEL: u32 = 2;
@@ -94,7 +97,7 @@ const SUB_SUB_NODES: u32 = 8;
 pub const MAX_REAL_INDEX: u32 = (1 << LEVEL_SHIFT) - 1;
 
 /// sub-AST level is stored in the top two bits
-fn sub_ast_level(index: u32) -> u32 {
+pub fn sub_ast_level(index: u32) -> u32 {
     (index & LEVEL_MASK) >> LEVEL_SHIFT
 }
 
@@ -106,10 +109,10 @@ pub fn sub_indices(index: u32) -> Result<(u32, u32), NodeIndexError> {
     }
     let next_level = (level + 1) << LEVEL_SHIFT;
     let without_level = index & !LEVEL_MASK;
-    let nodes_in_level = if level == 0 {
-        SUB_NODES
+    let (nodes_in_level, error_kind) = if level == 0 {
+        (SUB_NODES, NodeIndexError::OverflowedIndices)
     } else if level == 1 {
-        SUB_SUB_NODES
+        (SUB_SUB_NODES, NodeIndexError::OverflowedSubIndices)
     } else {
         unreachable!(
             "Someone made a mistake updating the encoding of node indices: {index:08X} had level {level}"
@@ -119,10 +122,10 @@ pub fn sub_indices(index: u32) -> Result<(u32, u32), NodeIndexError> {
     // If this overflows the file has hundreds of thousands of lines of code,
     // but that *can* happen (we just can't support string annotations that deep)
     let sub_index_without_level = without_level
-        .checked_mul(SUB_NODES)
-        .ok_or(NodeIndexError::OutOfIndices)?;
+        .checked_mul(nodes_in_level)
+        .ok_or(error_kind)?;
     if sub_index_without_level > MAX_REAL_INDEX {
-        return Err(NodeIndexError::OutOfIndices);
+        return Err(error_kind);
     }
 
     let first_index = sub_index_without_level | next_level;

crates/ty_python_semantic/resources/mdtest/annotations/string.md

@@ -128,12 +128,12 @@ reveal_type(Aliases.not_forward)  # revealed: str
 ```py
 a: "int" = 1
 b: "'int'" = 1
-# error: [invalid-syntax-in-forward-annotation] "Too many levels of nested string annotations"
+# error: [invalid-syntax-in-forward-annotation] "too many levels of nested string annotations; remove the redundant nested quotes"
 c: """'"int"'""" = 1
 d: "Foo"
 # error: [invalid-assignment] "Object of type `Literal[1]` is not assignable to `Foo`"
 e: "Foo" = 1
-# error: [invalid-syntax-in-forward-annotation] "too complex"
+# error: [invalid-syntax-in-forward-annotation] "nested string annotation is too long; remove the redundant nested quotes"
 f: "'str | int | bool | Foo | Bar'" = 1
 
 class Foo: ...