Better handling of hidden builtins

pybind11_stubgen/parser/mixins/fix.py

@@ -40,7 +40,6 @@
 
 
 class RemoveSelfAnnotation(IParser):
-
     __any_t_name = QualifiedName.from_str("Any")
     __typing_any_t_name = QualifiedName.from_str("typing.Any")
 
@@ -266,19 +265,54 @@ def handle_module(
 class FixBuiltinTypes(IParser):
     _any_type = QualifiedName.from_str("typing.Any")
 
+    _hidden_builtins = {
+        getattr(types, name).__qualname__: name
+        for name in dir(types)
+        if isinstance(getattr(types, name), type)
+        and getattr(getattr(types, name), "__module__", None)
+        == "builtins"  # defined in types, but reports `builtins`
+        and not hasattr(builtins, name)  # not actually available in `builtins`
+    }
+    """Types by their real name that are available in the `types` module,
+    but report `builtins` at runtime."""
+
+    _hidden_builtins_overrides = {
+        "function": "typing.Callable",
+        "builtin_function_or_method": "typing.Callable",
+    }
+    """Manual overrides for builtin types."""
+
     def handle_type(self, type_: type) -> QualifiedName:
         if type_.__qualname__ == "PyCapsule" and type_.__module__ == "builtins":
             return self._any_type
 
         result = super().handle_type(type_)
 
         if result[0] == "builtins":
-            if result[1] == "NoneType":
-                return QualifiedName((Identifier("None"),))
-            if result[1] in ("function", "builtin_function_or_method"):
-                callable_t = self.parse_annotation_str("typing.Callable")
-                assert isinstance(callable_t, ResolvedType)
-                return callable_t.name
+            typename = result[1]
+
+            if typename == "NoneType":
+                return QualifiedName(
+                    (Identifier("None"),)
+                )  # just print None instead of types.NoneType
+
+            # some types (e.g. `types.MappingProxyType`) report a wrong qualname
+            #  at runtime, and module == `builtins`
+            # we collect these upfront and translate their "builtin" name to the
+            #  importable one
+            hidden_builtin = self._hidden_builtins.get(typename)
+            if hidden_builtin is not None:
+                # some of these types are better described via the `typing`
+                # special forms e.g. types.FunctionType -> typing.Callable,
+                # so we use the override name
+                hidden_builtin_override = self._hidden_builtins_overrides.get(typename)
+
+                annotation = hidden_builtin_override or "types.%s" % hidden_builtin
+
+                override_t = self.parse_annotation_str(annotation)
+                assert isinstance(override_t, ResolvedType)
+                return override_t.name
+
             return QualifiedName(result[1:])
 
         return result

tests/py-demo/bindings/src/main.cpp

@@ -15,4 +15,5 @@ PYBIND11_MODULE(_bindings, m) {
     bind_stl_bind_module(m.def_submodule("stl_bind"));
     bind_typing_module(m.def_submodule("typing"));
     bind_values_module(m.def_submodule("values"));
-}
+    bind_hidden_builtins_module(m.def_submodule("hidden_builtins"));
+}

tests/py-demo/bindings/src/modules.h

@@ -20,3 +20,4 @@ void bind_stl_module(py::module&& m);
 void bind_stl_bind_module(py::module&& m);
 void bind_typing_module(py::module&& m);
 void bind_values_module(py::module&& m);
+void bind_hidden_builtins_module(py::module&& m);

tests/py-demo/bindings/src/modules/hidden_builtins.cpp

@@ -0,0 +1,18 @@
+#include "modules.h"
+
+#include <pybind11/pytypes.h>
+
+namespace hidden_builtins_detail {
+
+};
+
+void bind_hidden_builtins_module(py::module&& m) {
+  py::dict readonly_dict{};
+
+  auto types_m = py::module::import("types");
+  auto mappingproxy_t = types_m.attr("MappingProxyType");
+  auto none_t = types_m.attr("NoneType");
+
+  m.attr("mapping_proxy") = mappingproxy_t(readonly_dict);
+  m.attr("none") = none_t();
+}

tests/stubs/python-3.11/pybind11-v2.13/numpy-array-wrap-with-annotated/demo/__init__.pyi

@@ -7,6 +7,7 @@ from demo._bindings import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -27,6 +28,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",

tests/stubs/python-3.11/pybind11-v2.13/numpy-array-wrap-with-annotated/demo/_bindings/__init__.pyi

@@ -7,6 +7,7 @@ from . import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -24,6 +25,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",

tests/stubs/python-3.11/pybind11-v2.13/numpy-array-wrap-with-annotated/demo/_bindings/hidden_builtins.pyi

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+import types
+
+__all__: list[str] = ["mapping_proxy", "none"]
+mapping_proxy: types.MappingProxyType  # value = mappingproxy({})
+none = None

tests/stubs/python-3.11/pybind11-v2.13/numpy-array-wrap-with-annotated/demo/core.pyi

@@ -7,6 +7,7 @@ from demo._bindings import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -24,6 +25,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",

tests/stubs/python-3.11/pybind11-v3.0/numpy-array-wrap-with-annotated/demo/__init__.pyi

@@ -7,6 +7,7 @@ from demo._bindings import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -27,6 +28,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",

tests/stubs/python-3.11/pybind11-v3.0/numpy-array-wrap-with-annotated/demo/_bindings/__init__.pyi

@@ -7,6 +7,7 @@ from . import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -24,6 +25,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",

tests/stubs/python-3.11/pybind11-v3.0/numpy-array-wrap-with-annotated/demo/_bindings/hidden_builtins.pyi

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+import types
+
+__all__: list[str] = ["mapping_proxy", "none"]
+mapping_proxy: types.MappingProxyType  # value = mappingproxy({})
+none = None

tests/stubs/python-3.11/pybind11-v3.0/numpy-array-wrap-with-annotated/demo/core.pyi

@@ -7,6 +7,7 @@ from demo._bindings import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -24,6 +25,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",

tests/stubs/python-3.12/pybind11-v2.11/numpy-array-wrap-with-annotated/demo/__init__.pyi

@@ -7,6 +7,7 @@ from demo._bindings import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -27,6 +28,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",

tests/stubs/python-3.12/pybind11-v2.11/numpy-array-wrap-with-annotated/demo/_bindings/__init__.pyi

@@ -7,6 +7,7 @@ from . import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -24,6 +25,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",

tests/stubs/python-3.12/pybind11-v2.11/numpy-array-wrap-with-annotated/demo/_bindings/hidden_builtins.pyi

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+import types
+
+__all__: list[str] = ["mapping_proxy", "none"]
+mapping_proxy: types.MappingProxyType  # value = mappingproxy({})
+none = None

tests/stubs/python-3.12/pybind11-v2.11/numpy-array-wrap-with-annotated/demo/core.pyi

@@ -7,6 +7,7 @@ from demo._bindings import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -24,6 +25,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",

tests/stubs/python-3.12/pybind11-v2.12/numpy-array-wrap-with-annotated/demo/__init__.pyi

@@ -7,6 +7,7 @@ from demo._bindings import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -27,6 +28,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",

tests/stubs/python-3.12/pybind11-v2.12/numpy-array-wrap-with-annotated/demo/_bindings/__init__.pyi

@@ -7,6 +7,7 @@ from . import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -24,6 +25,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",

tests/stubs/python-3.12/pybind11-v2.12/numpy-array-wrap-with-annotated/demo/_bindings/hidden_builtins.pyi

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+import types
+
+__all__: list[str] = ["mapping_proxy", "none"]
+mapping_proxy: types.MappingProxyType  # value = mappingproxy({})
+none = None

tests/stubs/python-3.12/pybind11-v2.12/numpy-array-wrap-with-annotated/demo/core.pyi

@@ -7,6 +7,7 @@ from demo._bindings import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -24,6 +25,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",

tests/stubs/python-3.12/pybind11-v2.13/numpy-array-use-type-var/demo/__init__.pyi

@@ -7,6 +7,7 @@ from demo._bindings import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -27,6 +28,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",

tests/stubs/python-3.12/pybind11-v2.13/numpy-array-use-type-var/demo/_bindings/__init__.pyi

@@ -7,6 +7,7 @@ from . import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -24,6 +25,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",

tests/stubs/python-3.12/pybind11-v2.13/numpy-array-use-type-var/demo/_bindings/hidden_builtins.pyi

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+import types
+
+__all__: list[str] = ["mapping_proxy", "none"]
+mapping_proxy: types.MappingProxyType  # value = mappingproxy({})
+none = None

tests/stubs/python-3.12/pybind11-v2.13/numpy-array-use-type-var/demo/core.pyi

@@ -7,6 +7,7 @@ from demo._bindings import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -24,6 +25,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",

tests/stubs/python-3.12/pybind11-v2.13/numpy-array-wrap-with-annotated/demo/__init__.pyi

@@ -7,6 +7,7 @@ from demo._bindings import (
     enum,
     flawed_bindings,
     functions,
+    hidden_builtins,
     issues,
     methods,
     numpy,
@@ -27,6 +28,7 @@ __all__: list[str] = [
     "enum",
     "flawed_bindings",
     "functions",
+    "hidden_builtins",
     "issues",
     "methods",
     "numpy",