Fix RecursionError when checking an Instance with a self-referential __call__ attribute against a Callable type

mypy/subtypes.py

@@ -118,6 +118,11 @@ def __init__(
         self.erase_instances = erase_instances
         self.keep_erased_types = keep_erased_types
         self.options = options
+        # Classes whose "__call__" attribute is currently being followed while deciding
+        # whether an Instance is a subtype of a Callable/Overloaded type. Used to break
+        # infinite recursion when "__call__" refers back to its own class, e.g.
+        # "class C: __call__: C" (see #22398). Shared across a single subtype query.
+        self.resolving_call_types: set[TypeInfo] = set()
 
     def check_context(self, proper_subtype: bool) -> None:
         # Historically proper and non-proper subtypes were defined using different helpers
@@ -658,7 +663,17 @@ def visit_instance(self, left: Instance) -> bool:
             # Special case: Instance can be a subtype of Callable / Overloaded.
             call = find_member("__call__", left, left, is_operator=True)
             if call:
-                return self._is_subtype(call, right)
+                if left.type in self.subtype_context.resolving_call_types:
+                    # "__call__" (transitively) refers back to its own class, e.g.
+                    # "class C: __call__: C". Following it would recurse forever, and
+                    # such an instance cannot be called at runtime either, so it is
+                    # not a subtype of this Callable/Overloaded type (see #22398).
+                    return False
+                self.subtype_context.resolving_call_types.add(left.type)
+                try:
+                    return self._is_subtype(call, right)
+                finally:
+                    self.subtype_context.resolving_call_types.discard(left.type)
             return False
         else:
             return False

test-data/unit/check-classes.test

@@ -3410,6 +3410,48 @@ if int():
     b = a(a)  # E: Incompatible types in assignment (expression has type "A", variable has type "B")
 
 
+[case testSubtypeOfCallableWithSelfReferentialCallAttribute]
+from typing import Callable, Generic, Optional, TypeVar
+from typing_extensions import Self
+
+T = TypeVar("T")
+
+class C:
+    __call__: C
+class D:
+    __call__: Optional[D]
+class S:
+    __call__: Self
+class G(Generic[T]):
+    __call__: G[list[T]]
+class X:
+    __call__: Y
+class Y:
+    __call__: X
+class OK:
+    def __call__(self) -> int: return 1
+
+def h(cb: Callable[[], int]) -> None: ...
+
+h(C())  # E: Argument 1 to "h" has incompatible type "C"; expected "Callable[[], int]" \
+       # N: "C.__call__" has type "C"
+h(D())  # E: Argument 1 to "h" has incompatible type "D"; expected "Callable[[], int]" \
+       # N: "D.__call__" has type "D | None"
+h(S())  # E: Argument 1 to "h" has incompatible type "S"; expected "Callable[[], int]" \
+       # N: "S.__call__" has type "S"
+h(G[int]())  # E: Argument 1 to "h" has incompatible type "G[int]"; expected "Callable[[], int]" \
+            # N: "G[int].__call__" has type "G[list[int]]"
+h(X())  # E: Argument 1 to "h" has incompatible type "X"; expected "Callable[[], int]" \
+       # N: "X.__call__" has type "Y"
+h(Y())  # E: Argument 1 to "h" has incompatible type "Y"; expected "Callable[[], int]" \
+       # N: "Y.__call__" has type "X"
+h(OK())
+x: Callable[[], int] = C()  # E: Incompatible types in assignment (expression has type "C", variable has type "Callable[[], int]") \
+                          # N: "C.__call__" has type "C"
+y: Callable[[], int] = OK()
+[builtins fixtures/list.pyi]
+
+
 -- __new__
 -- --------