More tweaks to support mypyc (#5513)

Author: msullivan

mypy/bogus_type.py

@@ -10,13 +10,16 @@
 For those cases some other technique should be used.
 """
 
+import sys
+
 from mypy_extensions import FlexibleAlias
 from typing import TypeVar, Any
 
 T = TypeVar('T')
 
-SUPPRESS_BOGUS_TYPES = False
-if SUPPRESS_BOGUS_TYPES:
+# This won't ever be true at runtime, but we consider it true during
+# mypyc compilations.
+if sys.platform == 'mypyc':
     Bogus = FlexibleAlias[T, Any]
 else:
     Bogus = FlexibleAlias[T, T]

mypy/checkexpr.py

@@ -720,8 +720,8 @@ def analyze_type_type_callee(self, item: Type, context: Context) -> Type:
                 res = res.copy_modified(from_type_type=True)
             return expand_type_by_instance(res, item)
         if isinstance(item, UnionType):
-            return UnionType([self.analyze_type_type_callee(item, context)
-                              for item in item.relevant_items()], item.line)
+            return UnionType([self.analyze_type_type_callee(tp, context)
+                              for tp in item.relevant_items()], item.line)
         if isinstance(item, TypeVarType):
             # Pretend we're calling the typevar's upper bound,
             # i.e. its constructor (a poor approximation for reality,

mypy/fastparse2.py

@@ -17,7 +17,12 @@
 from functools import wraps
 import sys
 
-from typing import Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, cast, List
+from typing import Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, Dict, cast, List
+MYPY = False
+if MYPY:
+    import typing  # for typing.Type, which conflicts with types.Type
+    from typing import ClassVar
+
 from mypy.sharedparse import (
     special_function_elide_names, argument_elide_name,
 )
@@ -193,7 +198,7 @@ def translate_stmt_list(self, l: Sequence[ast27.AST]) -> List[Statement]:
         ast27.BitXor: '^',
         ast27.BitAnd: '&',
         ast27.FloorDiv: '//'
-    }
+    }  # type: ClassVar[Dict[typing.Type[ast27.AST], str]]
 
     def from_operator(self, op: ast27.operator) -> str:
         op_name = ASTConverter.op_map.get(type(op))
@@ -215,7 +220,7 @@ def from_operator(self, op: ast27.operator) -> str:
         ast27.IsNot: 'is not',
         ast27.In: 'in',
         ast27.NotIn: 'not in'
-    }
+    }  # type: ClassVar[Dict[typing.Type[ast27.AST], str]]
 
     def from_comp_operator(self, op: ast27.cmpop) -> str:
         op_name = ASTConverter.comp_op_map.get(type(op))
@@ -240,7 +245,7 @@ def as_required_block(self, stmts: List[ast27.stmt], lineno: int) -> Block:
     def fix_function_overloads(self, stmts: List[Statement]) -> List[Statement]:
         ret = []  # type: List[Statement]
         current_overload = []  # type: List[OverloadPart]
-        current_overload_name = None
+        current_overload_name = None  # type: Optional[str]
         for stmt in stmts:
             if (current_overload_name is not None
                     and isinstance(stmt, (Decorator, FuncDef))

mypy/options.py

@@ -191,7 +191,7 @@ def __init__(self) -> None:
         self.use_builtins_fixtures = False
 
         # -- experimental options --
-        self.shadow_file = None  # type: Optional[List[Tuple[str, str]]]
+        self.shadow_file = None  # type: Optional[List[List[str]]]
         self.show_column_numbers = False  # type: bool
         self.dump_graph = False
         self.dump_deps = False

mypy/stubgen.py

@@ -55,6 +55,7 @@
 import mypy.parse
 import mypy.errors
 import mypy.traverser
+import mypy.util
 from mypy import defaults
 from mypy.nodes import (
     Expression, IntExpr, UnaryExpr, StrExpr, BytesExpr, NameExpr, FloatExpr, MemberExpr, TupleExpr,
@@ -818,28 +819,33 @@ def is_recorded_name(self, name: str) -> bool:
         return self.is_top_level() and name in self._toplevel_names
 
 
+class SelfTraverser(mypy.traverser.TraverserVisitor):
+    def __init__(self) -> None:
+        self.results = []  # type: List[Tuple[str, Expression]]
+
+    def visit_assignment_stmt(self, o: AssignmentStmt) -> None:
+        lvalue = o.lvalues[0]
+        if (isinstance(lvalue, MemberExpr) and
+                isinstance(lvalue.expr, NameExpr) and
+                lvalue.expr.name == 'self'):
+            self.results.append((lvalue.name, o.rvalue))
+
+
 def find_self_initializers(fdef: FuncBase) -> List[Tuple[str, Expression]]:
-    results = []  # type: List[Tuple[str, Expression]]
+    traverser = SelfTraverser()
+    fdef.accept(traverser)
+    return traverser.results
+
 
-    class SelfTraverser(mypy.traverser.TraverserVisitor):
-        def visit_assignment_stmt(self, o: AssignmentStmt) -> None:
-            lvalue = o.lvalues[0]
-            if (isinstance(lvalue, MemberExpr) and
-                    isinstance(lvalue.expr, NameExpr) and
-                    lvalue.expr.name == 'self'):
-                results.append((lvalue.name, o.rvalue))
+class ReturnSeeker(mypy.traverser.TraverserVisitor):
+    def __init__(self) -> None:
+        self.found = False
 
-    fdef.accept(SelfTraverser())
-    return results
+    def visit_return_stmt(self, o: ReturnStmt) -> None:
+        self.found = True
 
 
 def has_return_statement(fdef: FuncBase) -> bool:
-    class ReturnSeeker(mypy.traverser.TraverserVisitor):
-        def __init__(self) -> None:
-            self.found = False
-
-        def visit_return_stmt(self, o: ReturnStmt) -> None:
-            self.found = True
 
     seeker = ReturnSeeker()
     fdef.accept(seeker)

mypy/stubgenc.py

@@ -6,7 +6,7 @@
 import importlib
 import os.path
 import re
-from typing import List, Dict, Tuple, Optional
+from typing import List, Dict, Tuple, Optional, Mapping, Any
 from types import ModuleType
 
 from mypy.stubutil import is_c_module, write_header, infer_sig_from_docstring
@@ -137,7 +137,10 @@ def generate_c_type_stub(module: ModuleType,
                          sigs: Dict[str, str] = {},
                          class_sigs: Dict[str, str] = {},
                          ) -> None:
-    items = sorted(obj.__dict__.items(), key=lambda x: method_name_sort_key(x[0]))
+    # typeshed gives obj.__dict__ the not quite correct type Dict[str, Any]
+    # (it could be a mappingproxy!), which makes mypyc mad, so obfuscate it.
+    obj_dict = getattr(obj, '__dict__')  # type: Mapping[str, Any]
+    items = sorted(obj_dict.items(), key=lambda x: method_name_sort_key(x[0]))
     methods = []
     done = set()
     for attr, value in items:
@@ -151,7 +154,7 @@ def generate_c_type_stub(module: ModuleType,
                     self_var = 'self'
                 if attr == '__new__':
                     # TODO: We should support __new__.
-                    if '__init__' in obj.__dict__:
+                    if '__init__' in obj_dict:
                         # Avoid duplicate functions if both are present.
                         # But is there any case where .__new__() has a
                         # better signature than __init__() ?

mypy/test/testtransform.py

@@ -9,9 +9,8 @@
 )
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.config import test_temp_dir
+from mypy.test.visitors import TypeAssertTransformVisitor
 from mypy.errors import CompileError
-from mypy.treetransform import TransformVisitor
-from mypy.types import Type
 from mypy.options import Options
 
 
@@ -72,9 +71,3 @@ def test_transform(testcase: DataDrivenTestCase) -> None:
         testcase.output, a,
         'Invalid semantic analyzer output ({}, line {})'.format(testcase.file,
                                                                 testcase.line))
-
-
-class TypeAssertTransformVisitor(TransformVisitor):
-    def type(self, type: Type) -> Type:
-        assert type is not None
-        return type

mypy/test/testtypegen.py

@@ -2,18 +2,16 @@
 
 import re
 
-from typing import Set
-
 from mypy import build
 from mypy.build import BuildSource
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal
+from mypy.test.visitors import SkippedNodeSearcher, ignore_node
 from mypy.util import short_type
 from mypy.nodes import (
     NameExpr, TypeVarExpr, CallExpr, Expression, MypyFile, AssignmentStmt, IntExpr
 )
-from mypy.traverser import TraverserVisitor
 from mypy.errors import CompileError
 from mypy.options import Options
 
@@ -73,49 +71,3 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
             testcase.output, a,
             'Invalid type checker output ({}, line {})'.format(testcase.file,
                                                                testcase.line))
-
-
-class SkippedNodeSearcher(TraverserVisitor):
-    def __init__(self) -> None:
-        self.nodes = set()  # type: Set[Expression]
-        self.is_typing = False
-
-    def visit_mypy_file(self, f: MypyFile) -> None:
-        self.is_typing = f.fullname() == 'typing' or f.fullname() == 'builtins'
-        super().visit_mypy_file(f)
-
-    def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
-        if s.type or ignore_node(s.rvalue):
-            for lvalue in s.lvalues:
-                if isinstance(lvalue, NameExpr):
-                    self.nodes.add(lvalue)
-        super().visit_assignment_stmt(s)
-
-    def visit_name_expr(self, n: NameExpr) -> None:
-        self.skip_if_typing(n)
-
-    def visit_int_expr(self, n: IntExpr) -> None:
-        self.skip_if_typing(n)
-
-    def skip_if_typing(self, n: Expression) -> None:
-        if self.is_typing:
-            self.nodes.add(n)
-
-
-def ignore_node(node: Expression) -> bool:
-    """Return True if node is to be omitted from test case output."""
-
-    # We want to get rid of object() expressions in the typing module stub
-    # and also TypeVar(...) expressions. Since detecting whether a node comes
-    # from the typing module is not easy, we just to strip them all away.
-    if isinstance(node, TypeVarExpr):
-        return True
-    if isinstance(node, NameExpr) and node.fullname == 'builtins.object':
-        return True
-    if isinstance(node, NameExpr) and node.fullname == 'builtins.None':
-        return True
-    if isinstance(node, CallExpr) and (ignore_node(node.callee) or
-                                       node.analyzed):
-        return True
-
-    return False

mypy/test/visitors.py

@@ -0,0 +1,71 @@
+"""Visitor classes pulled out from different tests
+
+These are here because we don't currently support having interpreted
+classes subtype compiled ones but pytest grabs the python file
+even if the test was compiled.
+
+"""
+
+from typing import Set
+
+from mypy.nodes import (
+    NameExpr, TypeVarExpr, CallExpr, Expression, MypyFile, AssignmentStmt, IntExpr
+)
+from mypy.traverser import TraverserVisitor
+
+from mypy.treetransform import TransformVisitor
+from mypy.types import Type
+
+
+# from testtypegen
+class SkippedNodeSearcher(TraverserVisitor):
+    def __init__(self) -> None:
+        self.nodes = set()  # type: Set[Expression]
+        self.is_typing = False
+
+    def visit_mypy_file(self, f: MypyFile) -> None:
+        self.is_typing = f.fullname() == 'typing' or f.fullname() == 'builtins'
+        super().visit_mypy_file(f)
+
+    def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
+        if s.type or ignore_node(s.rvalue):
+            for lvalue in s.lvalues:
+                if isinstance(lvalue, NameExpr):
+                    self.nodes.add(lvalue)
+        super().visit_assignment_stmt(s)
+
+    def visit_name_expr(self, n: NameExpr) -> None:
+        self.skip_if_typing(n)
+
+    def visit_int_expr(self, n: IntExpr) -> None:
+        self.skip_if_typing(n)
+
+    def skip_if_typing(self, n: Expression) -> None:
+        if self.is_typing:
+            self.nodes.add(n)
+
+
+def ignore_node(node: Expression) -> bool:
+    """Return True if node is to be omitted from test case output."""
+
+    # We want to get rid of object() expressions in the typing module stub
+    # and also TypeVar(...) expressions. Since detecting whether a node comes
+    # from the typing module is not easy, we just to strip them all away.
+    if isinstance(node, TypeVarExpr):
+        return True
+    if isinstance(node, NameExpr) and node.fullname == 'builtins.object':
+        return True
+    if isinstance(node, NameExpr) and node.fullname == 'builtins.None':
+        return True
+    if isinstance(node, CallExpr) and (ignore_node(node.callee) or
+                                       node.analyzed):
+        return True
+
+    return False
+
+
+# from testtransform
+class TypeAssertTransformVisitor(TransformVisitor):
+    def type(self, type: Type) -> Type:
+        assert type is not None
+        return type

mypy/treetransform.py

@@ -105,7 +105,7 @@ def visit_func_def(self, node: FuncDef) -> FuncDef:
         new = FuncDef(node.name(),
                       [self.copy_argument(arg) for arg in node.arguments],
                       self.block(node.body),
-                      cast(FunctionLike, self.optional_type(node.type)))
+                      cast(Optional[FunctionLike], self.optional_type(node.type)))
 
         self.copy_function_attributes(new, node)
 
@@ -131,8 +131,8 @@ def visit_func_def(self, node: FuncDef) -> FuncDef:
 
     def visit_lambda_expr(self, node: LambdaExpr) -> LambdaExpr:
         new = LambdaExpr([self.copy_argument(arg) for arg in node.arguments],
-                       self.block(node.body),
-                       cast(FunctionLike, self.optional_type(node.type)))
+                         self.block(node.body),
+                         cast(Optional[FunctionLike], self.optional_type(node.type)))
         self.copy_function_attributes(new, node)
         return new