Support fancy new syntax for variadic types (#16242)

This is the last significant thing I am aware of that is needed for PEP
646 support. After this and other currently open PRs are merged, I will
make an additional pass grepping for usual suspects and verifying we
didn't miss anything. Then we can flip the switch and announce this as
supported.

Author: ilevkivskyi

mypy/exprtotype.py

@@ -103,7 +103,10 @@ def expr_to_unanalyzed_type(
                 return expr_to_unanalyzed_type(args[0], options, allow_new_syntax, expr)
             else:
                 base.args = tuple(
-                    expr_to_unanalyzed_type(arg, options, allow_new_syntax, expr) for arg in args
+                    expr_to_unanalyzed_type(
+                        arg, options, allow_new_syntax, expr, allow_unpack=True
+                    )
+                    for arg in args
                 )
             if not base.args:
                 base.empty_tuple_index = True

mypy/fastparse.py

@@ -1762,7 +1762,6 @@ def __init__(
         self.override_column = override_column
         self.node_stack: list[AST] = []
         self.is_evaluated = is_evaluated
-        self.allow_unpack = False
 
     def convert_column(self, column: int) -> int:
         """Apply column override if defined; otherwise return column.
@@ -2039,19 +2038,14 @@ def visit_Attribute(self, n: Attribute) -> Type:
         else:
             return self.invalid_type(n)
 
-    # Used for Callable[[X *Ys, Z], R]
+    # Used for Callable[[X *Ys, Z], R] etc.
     def visit_Starred(self, n: ast3.Starred) -> Type:
         return UnpackType(self.visit(n.value), from_star_syntax=True)
 
     # List(expr* elts, expr_context ctx)
     def visit_List(self, n: ast3.List) -> Type:
         assert isinstance(n.ctx, ast3.Load)
-        old_allow_unpack = self.allow_unpack
-        # We specifically only allow starred expressions in a list to avoid
-        # confusing errors for top-level unpacks (e.g. in base classes).
-        self.allow_unpack = True
         result = self.translate_argument_list(n.elts)
-        self.allow_unpack = old_allow_unpack
         return result
 
 

mypy/messages.py

@@ -2516,6 +2516,8 @@ def format_literal_value(typ: LiteralType) -> str:
             # There are type arguments. Convert the arguments to strings.
             return f"{base_str}[{format_list(itype.args)}]"
     elif isinstance(typ, UnpackType):
+        if options.use_star_unpack():
+            return f"*{format(typ.type)}"
         return f"Unpack[{format(typ.type)}]"
     elif isinstance(typ, TypeVarType):
         # This is similar to non-generic instance types.

mypy/options.py

@@ -385,6 +385,9 @@ def use_or_syntax(self) -> bool:
             return not self.force_union_syntax
         return False
 
+    def use_star_unpack(self) -> bool:
+        return self.python_version >= (3, 11)
+
     # To avoid breaking plugin compatibility, keep providing new_semantic_analyzer
     @property
     def new_semantic_analyzer(self) -> bool:

mypy/semanal.py

@@ -1992,38 +1992,42 @@ def analyze_class_typevar_declaration(self, base: Type) -> tuple[TypeVarLikeList
         return None
 
     def analyze_unbound_tvar(self, t: Type) -> tuple[str, TypeVarLikeExpr] | None:
-        if not isinstance(t, UnboundType):
-            return None
-        unbound = t
-        sym = self.lookup_qualified(unbound.name, unbound)
+        if isinstance(t, UnpackType) and isinstance(t.type, UnboundType):
+            return self.analyze_unbound_tvar_impl(t.type, allow_tvt=True)
+        if isinstance(t, UnboundType):
+            sym = self.lookup_qualified(t.name, t)
+            if sym and sym.fullname in ("typing.Unpack", "typing_extensions.Unpack"):
+                inner_t = t.args[0]
+                if isinstance(inner_t, UnboundType):
+                    return self.analyze_unbound_tvar_impl(inner_t, allow_tvt=True)
+                return None
+            return self.analyze_unbound_tvar_impl(t)
+        return None
+
+    def analyze_unbound_tvar_impl(
+        self, t: UnboundType, allow_tvt: bool = False
+    ) -> tuple[str, TypeVarLikeExpr] | None:
+        sym = self.lookup_qualified(t.name, t)
         if sym and isinstance(sym.node, PlaceholderNode):
             self.record_incomplete_ref()
-        if sym and isinstance(sym.node, ParamSpecExpr):
+        if not allow_tvt and sym and isinstance(sym.node, ParamSpecExpr):
             if sym.fullname and not self.tvar_scope.allow_binding(sym.fullname):
                 # It's bound by our type variable scope
                 return None
-            return unbound.name, sym.node
-        if sym and sym.fullname in ("typing.Unpack", "typing_extensions.Unpack"):
-            inner_t = unbound.args[0]
-            if not isinstance(inner_t, UnboundType):
+            return t.name, sym.node
+        if allow_tvt and sym and isinstance(sym.node, TypeVarTupleExpr):
+            if sym.fullname and not self.tvar_scope.allow_binding(sym.fullname):
+                # It's bound by our type variable scope
                 return None
-            inner_unbound = inner_t
-            inner_sym = self.lookup_qualified(inner_unbound.name, inner_unbound)
-            if inner_sym and isinstance(inner_sym.node, PlaceholderNode):
-                self.record_incomplete_ref()
-            if inner_sym and isinstance(inner_sym.node, TypeVarTupleExpr):
-                if inner_sym.fullname and not self.tvar_scope.allow_binding(inner_sym.fullname):
-                    # It's bound by our type variable scope
-                    return None
-                return inner_unbound.name, inner_sym.node
-        if sym is None or not isinstance(sym.node, TypeVarExpr):
+            return t.name, sym.node
+        if sym is None or not isinstance(sym.node, TypeVarExpr) or allow_tvt:
             return None
         elif sym.fullname and not self.tvar_scope.allow_binding(sym.fullname):
             # It's bound by our type variable scope
             return None
         else:
             assert isinstance(sym.node, TypeVarExpr)
-            return unbound.name, sym.node
+            return t.name, sym.node
 
     def get_all_bases_tvars(
         self, base_type_exprs: list[Expression], removed: list[int]
@@ -5333,7 +5337,9 @@ def analyze_type_application_args(self, expr: IndexExpr) -> list[Type] | None:
                 has_param_spec = False
                 num_args = -1
         elif isinstance(base, RefExpr) and isinstance(base.node, TypeInfo):
-            allow_unpack = base.node.has_type_var_tuple_type
+            allow_unpack = (
+                base.node.has_type_var_tuple_type or base.node.fullname == "builtins.tuple"
+            )
             has_param_spec = base.node.has_param_spec_type
             num_args = len(base.node.type_vars)
         else:
@@ -5343,7 +5349,7 @@ def analyze_type_application_args(self, expr: IndexExpr) -> list[Type] | None:
 
         for item in items:
             try:
-                typearg = self.expr_to_unanalyzed_type(item)
+                typearg = self.expr_to_unanalyzed_type(item, allow_unpack=True)
             except TypeTranslationError:
                 self.fail("Type expected within [...]", expr)
                 return None
@@ -6608,8 +6614,10 @@ def type_analyzer(
         tpan.global_scope = not self.type and not self.function_stack
         return tpan
 
-    def expr_to_unanalyzed_type(self, node: Expression) -> ProperType:
-        return expr_to_unanalyzed_type(node, self.options, self.is_stub_file)
+    def expr_to_unanalyzed_type(self, node: Expression, allow_unpack: bool = False) -> ProperType:
+        return expr_to_unanalyzed_type(
+            node, self.options, self.is_stub_file, allow_unpack=allow_unpack
+        )
 
     def anal_type(
         self,

mypy/typeanal.py

@@ -964,7 +964,10 @@ def visit_unpack_type(self, t: UnpackType) -> Type:
         if not self.allow_unpack:
             self.fail(message_registry.INVALID_UNPACK_POSITION, t.type, code=codes.VALID_TYPE)
             return AnyType(TypeOfAny.from_error)
-        return UnpackType(self.anal_type(t.type), from_star_syntax=t.from_star_syntax)
+        self.allow_type_var_tuple = True
+        result = UnpackType(self.anal_type(t.type), from_star_syntax=t.from_star_syntax)
+        self.allow_type_var_tuple = False
+        return result
 
     def visit_parameters(self, t: Parameters) -> Type:
         raise NotImplementedError("ParamSpec literals cannot have unbound TypeVars")

test-data/unit/check-python311.test

@@ -77,3 +77,68 @@ async def coro() -> Generator[List[Any], None, None]:
 reveal_type(coro)  # N: Revealed type is "def () -> typing.Coroutine[Any, Any, typing.Generator[builtins.list[Any], None, None]]"
 [builtins fixtures/async_await.pyi]
 [typing fixtures/typing-async.pyi]
+
+[case testTypeVarTupleNewSyntaxAnnotations]
+Ints = tuple[int, int, int]
+x: tuple[str, *Ints]
+reveal_type(x)  # N: Revealed type is "Tuple[builtins.str, builtins.int, builtins.int, builtins.int]"
+y: tuple[int, *tuple[int, ...]]
+reveal_type(y)  # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]]]"
+[builtins fixtures/tuple.pyi]
+
+[case testTypeVarTupleNewSyntaxGenerics]
+from typing import Generic, TypeVar, TypeVarTuple
+
+T = TypeVar("T")
+Ts = TypeVarTuple("Ts")
+class C(Generic[T, *Ts]):
+    attr: tuple[int, *Ts, str]
+
+    def test(self) -> None:
+        reveal_type(self.attr)  # N: Revealed type is "Tuple[builtins.int, Unpack[Ts`2], builtins.str]"
+        self.attr = ci  # E: Incompatible types in assignment (expression has type "C[*Tuple[int, ...]]", variable has type "Tuple[int, *Ts, str]")
+    def meth(self, *args: *Ts) -> T: ...
+
+ci: C[*tuple[int, ...]]
+reveal_type(ci)  # N: Revealed type is "__main__.C[Unpack[builtins.tuple[builtins.int, ...]]]"
+reveal_type(ci.meth)  # N: Revealed type is "def (*args: builtins.int) -> builtins.int"
+c3: C[str, str, str]
+reveal_type(c3)  # N: Revealed type is "__main__.C[builtins.str, builtins.str, builtins.str]"
+
+A = C[int, *Ts]
+B = tuple[str, *tuple[str, str], str]
+z: A[*B]
+reveal_type(z)  # N: Revealed type is "__main__.C[builtins.int, builtins.str, builtins.str, builtins.str, builtins.str]"
+[builtins fixtures/tuple.pyi]
+
+[case testTypeVarTupleNewSyntaxCallables]
+from typing import Generic, overload, TypeVar
+
+T1 = TypeVar("T1")
+T2 = TypeVar("T2")
+class MyClass(Generic[T1, T2]):
+    @overload
+    def __init__(self: MyClass[None, None]) -> None: ...
+
+    @overload
+    def __init__(self: MyClass[T1, None], *types: *tuple[type[T1]]) -> None: ...
+
+    @overload
+    def __init__(self: MyClass[T1, T2], *types: *tuple[type[T1], type[T2]]) -> None: ...
+
+    def __init__(self: MyClass[T1, T2], *types: *tuple[type, ...]) -> None:
+        pass
+
+myclass = MyClass()
+reveal_type(myclass)  # N: Revealed type is "__main__.MyClass[None, None]"
+myclass1 = MyClass(float)
+reveal_type(myclass1)  # N: Revealed type is "__main__.MyClass[builtins.float, None]"
+myclass2 = MyClass(float, float)
+reveal_type(myclass2)  # N: Revealed type is "__main__.MyClass[builtins.float, builtins.float]"
+myclass3 = MyClass(float, float, float)  # E: No overload variant of "MyClass" matches argument types "Type[float]", "Type[float]", "Type[float]" \
+                                         # N: Possible overload variants: \
+                                         # N:     def [T1, T2] __init__(self) -> MyClass[None, None] \
+                                         # N:     def [T1, T2] __init__(self, Type[T1], /) -> MyClass[T1, None] \
+                                         # N:     def [T1, T2] __init__(Type[T1], Type[T2], /) -> MyClass[T1, T2]
+reveal_type(myclass3)  # N: Revealed type is "Any"
+[builtins fixtures/tuple.pyi]

test-data/unit/check-python312.test

@@ -41,8 +41,6 @@ type Alias2[**P] = Callable[P, int]  # E: PEP 695 type aliases are not yet suppo
                                      # E: Value of type "int" is not indexable \
                                      # E: Name "P" is not defined
 type Alias3[*Ts] = tuple[*Ts]  # E: PEP 695 type aliases are not yet supported \
-                               # E: Type expected within [...] \
-                               # E: The type "Type[Tuple[Any, ...]]" is not generic and not indexable \
                                # E: Name "Ts" is not defined
 
 class Cls1[T: int]: ...  # E: PEP 695 generics are not yet supported