From c43ff6fa845c6c060844314a21cd0b7102c00b7d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 15:55:52 -0400 Subject: [PATCH 01/45] chore: fail fast and clearly when Py_LIMITED_API is defined Py_LIMITED_API is not supported yet; a build with it defined failed with pages of unrelated errors. Add a single #error in common.h, and reserve a bit for Py_LIMITED_API in the precompiled configuration check so later stable-ABI work can switch it on. See https://github.com/pybind/pybind11/discussions/6104 (B1). Assisted-by: ClaudeCode:claude-fable-5-1 --- docs/compiling.rst | 5 +++-- include/pybind11/detail/common.h | 5 +++++ include/pybind11/detail/internals.h | 16 +++++++++++----- tools/check_inl_headers.py | 1 + 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/docs/compiling.rst b/docs/compiling.rst index a81b16fead..e926c2cc67 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -435,8 +435,9 @@ Requirements and caveats: macros ``PYBIND11_INTERNALS_VERSION``, ``Py_GIL_DISABLED``, ``PYBIND11_SIMPLE_GIL_MANAGEMENT``, ``PYBIND11_DETAILED_ERROR_MESSAGES`` (defaults on in debug builds), - ``PYBIND11_HAS_SUBINTERPRETER_SUPPORT``, and - ``PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET``. A mismatch produces one + ``PYBIND11_HAS_SUBINTERPRETER_SUPPORT``, + ``PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET``, and + ``Py_LIMITED_API``. A mismatch produces one readable undefined symbol at link time referencing ``pybind11_precompiled_config``. * Configuration macros that only change code inside the library (for example diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 142c23c772..a86187c76f 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -14,6 +14,11 @@ # error "PYTHON < 3.9 IS UNSUPPORTED. pybind11 v3.0 was the last to support Python 3.8." #endif +#if defined(Py_LIMITED_API) +# error \ + "pybind11 does not support Py_LIMITED_API (the Python stable ABI) yet. Track progress at https://github.com/pybind/pybind11/discussions/6104" +#endif + // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // See also: https://github.com/python/cpython/blob/HEAD/Include/patchlevel.h /* -- start version constants -- */ diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index cf546db43f..abac3efb35 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -79,18 +79,24 @@ # else # define PYBIND11_PRECOMPILED_CFG_TD 0 # endif +# if defined(Py_LIMITED_API) +# define PYBIND11_PRECOMPILED_CFG_LA 1 +# else +# define PYBIND11_PRECOMPILED_CFG_LA 0 +# endif // PYBIND11_CONCAT does not macro-expand its arguments (## suppresses expansion). -# define PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de, si, td) \ - pybind11_precompiled_config_v##v##_gd##gd##_sg##sg##_de##de##_si##si##_td##td -# define PYBIND11_PRECOMPILED_CONFIG_NAME(v, gd, sg, de, si, td) \ - PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de, si, td) +# define PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de, si, td, la) \ + pybind11_precompiled_config_v##v##_gd##gd##_sg##sg##_de##de##_si##si##_td##td##_la##la +# define PYBIND11_PRECOMPILED_CONFIG_NAME(v, gd, sg, de, si, td, la) \ + PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de, si, td, la) # define PYBIND11_PRECOMPILED_CONFIG_CHECK \ PYBIND11_PRECOMPILED_CONFIG_NAME(PYBIND11_INTERNALS_VERSION, \ PYBIND11_PRECOMPILED_CFG_GD, \ PYBIND11_PRECOMPILED_CFG_SG, \ PYBIND11_PRECOMPILED_CFG_DE, \ PYBIND11_PRECOMPILED_CFG_SI, \ - PYBIND11_PRECOMPILED_CFG_TD) + PYBIND11_PRECOMPILED_CFG_TD, \ + PYBIND11_PRECOMPILED_CFG_LA) # define PYBIND11_PRECOMPILED_CONFIG_GUARD \ ::pybind11::detail::PYBIND11_PRECOMPILED_CONFIG_CHECK(); #else diff --git a/tools/check_inl_headers.py b/tools/check_inl_headers.py index 98e5d458bc..db95d85fb0 100755 --- a/tools/check_inl_headers.py +++ b/tools/check_inl_headers.py @@ -45,6 +45,7 @@ "PYBIND11_INTERNALS_VERSION", "PYBIND11_SIMPLE_GIL_MANAGEMENT", "Py_GIL_DISABLED", + "Py_LIMITED_API", } # Only change code inside the library; documented in docs/compiling.rst. From a07e4ed760190e23f9419fabd55047cc773d6bb7 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 15:57:29 -0400 Subject: [PATCH 02/45] chore: add a stable-ABI tag to PYBIND11_INTERNALS_ID and PYBIND11_MODULE_LOCAL_ID Modules built with Py_LIMITED_API will use a different internals layout, so they must not share internals with regular modules in one process. The tag is empty today and becomes "_stable" under Py_LIMITED_API. It is not part of PYBIND11_PLATFORM_ABI_ID, so cpp_conduit keeps bridging both worlds. See https://github.com/pybind/pybind11/discussions/6104 (B2). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/internals.h | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index abac3efb35..60f3cfcae3 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -499,13 +499,26 @@ struct native_enum_record { static const char *attribute_name() { return "__pybind11_native_enum__"; } }; +// Modules built for the Python stable ABI (Py_LIMITED_API) use a different `internals` layout +// (opaque TSS keys, different type-object access), so they must not share internals with +// regular modules in the same process. The tag keeps the two universes apart. It is deliberately +// not part of PYBIND11_PLATFORM_ABI_ID: the C++ ABI is the same, so the cpp_conduit protocol +// still bridges the two. +#if defined(Py_LIMITED_API) +# define PYBIND11_INTERNALS_SABI_TAG "_stable" +#else +# define PYBIND11_INTERNALS_SABI_TAG "" +#endif + #define PYBIND11_INTERNALS_ID \ "__pybind11_internals_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID "__" + PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE \ + PYBIND11_PLATFORM_ABI_ID PYBIND11_INTERNALS_SABI_TAG "__" #define PYBIND11_MODULE_LOCAL_ID \ "__pybind11_module_local_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID "__" + PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE \ + PYBIND11_PLATFORM_ABI_ID PYBIND11_INTERNALS_SABI_TAG "__" /// We use this to figure out if there are or have been multiple subinterpreters active at any /// point. This must never go from true to false while any interpreter may be running in any From 8d8fe02e58fd7b3dbeaa11c22b0e9d4584ce2f60 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 15:58:53 -0400 Subject: [PATCH 03/45] refactor: add detail::get_bases()/get_mro() and use them instead of tp_bases/tp_mro Direct reads of tp_bases/tp_mro are not possible under the Python stable ABI. Route all reads through two small helpers so the limited-API branch can be added in one place later. See https://github.com/pybind/pybind11/discussions/6104 (B3). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 4 ++-- include/pybind11/detail/type_caster_base-inl.h | 10 +++++++--- include/pybind11/pybind11-inl.h | 3 +-- include/pybind11/pytypes.h | 11 +++++++++++ 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 57d12fee44..67bd5901cd 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -268,7 +268,7 @@ PYBIND11_INLINE void traverse_offset_bases(void *valueptr, const detail::type_info *tinfo, instance *self, bool (*f)(void * /*parentptr*/, instance * /*self*/)) { - for (handle h : reinterpret_borrow(tinfo->type->tp_bases)) { + for (handle h : get_bases(tinfo->type)) { if (auto *parent_tinfo = get_type_info(reinterpret_cast(h.ptr()))) { for (auto &c : parent_tinfo->implicit_casts) { if (c.first == tinfo->cpptype) { @@ -564,7 +564,7 @@ PYBIND11_INLINE void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { extern "C" PYBIND11_INLINE int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int flags) { // Look for a `get_buffer` implementation in this type's info or any bases (following MRO). type_info *tinfo = nullptr; - for (auto type : reinterpret_borrow(Py_TYPE(obj)->tp_mro)) { + for (auto type : get_mro(Py_TYPE(obj))) { tinfo = get_type_info((PyTypeObject *) type.ptr()); if (tinfo && tinfo->get_buffer) { break; diff --git a/include/pybind11/detail/type_caster_base-inl.h b/include/pybind11/detail/type_caster_base-inl.h index 892b2c55b9..cad39fb397 100644 --- a/include/pybind11/detail/type_caster_base-inl.h +++ b/include/pybind11/detail/type_caster_base-inl.h @@ -85,7 +85,7 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void all_type_info_populate(PyTypeObject *t, std::vector &bases) { assert(bases.empty()); std::vector check; - for (handle parent : reinterpret_borrow(t->tp_bases)) { + for (handle parent : get_bases(t)) { check.push_back(reinterpret_cast(parent.ptr())); } auto const &type_dict = get_internals().registered_types_py; @@ -118,9 +118,13 @@ all_type_info_populate(PyTypeObject *t, std::vector &bases) { all_type_info_add_base_most_derived_first(bases, tinfo); } } - } else if (type->tp_bases) { + } else { // It's some python type, so keep follow its bases classes to look for one or more // registered types + tuple parents = get_bases(type); + if (parents.empty()) { + continue; + } if (i + 1 == check.size()) { // When we're at the end, we can pop off the current element to avoid growing // `check` when adding just one base (which is typical--i.e. when there is no @@ -128,7 +132,7 @@ all_type_info_populate(PyTypeObject *t, std::vector &bases) { check.pop_back(); i--; } - for (handle parent : reinterpret_borrow(type->tp_bases)) { + for (handle parent : parents) { check.push_back(reinterpret_cast(parent.ptr())); } } diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index 24e1359220..1363be35be 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -1337,8 +1337,7 @@ PYBIND11_INLINE void generic_type::initialize(const type_record &rec) { } PYBIND11_INLINE void generic_type::mark_parents_nonsimple(PyTypeObject *value) { - auto t = reinterpret_borrow(value->tp_bases); - for (handle h : t) { + for (handle h : get_bases(value)) { auto *tinfo2 = get_type_info(reinterpret_cast(h.ptr())); if (tinfo2) { tinfo2->simple_type = false; diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 88de78d024..021bcc95d5 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -2465,6 +2465,17 @@ PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator>>=, PyNumber_InPlaceRshift) #undef PYBIND11_MATH_OPERATOR_BINARY_INPLACE // Meant to return a Python str, but this is not checked. +/// `type.__bases__` as a tuple (empty if the type has no bases tuple yet). +inline tuple get_bases(PyTypeObject *type) { + if (type->tp_bases == nullptr) { + return tuple(); + } + return reinterpret_borrow(type->tp_bases); +} + +/// `type.__mro__` as a tuple. +inline tuple get_mro(PyTypeObject *type) { return reinterpret_borrow(type->tp_mro); } + inline object get_module_name_if_available(handle scope) { if (scope) { if (hasattr(scope, "__module__")) { From ac39419f1931d6c97cd2059d45e5c599caf6e469 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:01:04 -0400 Subject: [PATCH 04/45] refactor: add detail::get_tp_name()/tp_name_equals() and use them instead of raw tp_name reads Direct reads of tp_name are not possible under the Python stable ABI. obj_class_name() now returns std::string. See https://github.com/pybind/pybind11/discussions/6104 (B4). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/cast.h | 8 +++----- include/pybind11/detail/class-inl.h | 6 +++--- .../pybind11/detail/function_record_pyobject.h | 5 ++--- include/pybind11/pytypes-inl.h | 15 ++------------- include/pybind11/pytypes.h | 15 ++++++++++++--- include/pybind11/stl.h | 3 +-- 6 files changed, 23 insertions(+), 29 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 7448677928..436761c3c3 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -477,10 +477,9 @@ class type_caster { private: // Test if an object is a NumPy boolean (without fetching the type). static bool is_numpy_bool(handle object) { - const char *type_name = Py_TYPE(object.ptr())->tp_name; // Name changed to `numpy.bool` in NumPy 2, `numpy.bool_` is needed for 1.x support - return std::strcmp("numpy.bool", type_name) == 0 - || std::strcmp("numpy.bool_", type_name) == 0; + return tp_name_equals(Py_TYPE(object.ptr()), "numpy.bool") + || tp_name_equals(Py_TYPE(object.ptr()), "numpy.bool_"); } }; @@ -1172,8 +1171,7 @@ std::weak_ptr potentially_slicing_weak_ptr(handle obj) { if (caster.load(obj, /*convert=*/true)) { return caster.potentially_slicing_weak_ptr(); } - const char *obj_type_name = detail::obj_class_name(obj.ptr()); - throw type_error("\"" + std::string(obj_type_name) + throw type_error("\"" + detail::obj_class_name(obj.ptr()) + "\" object is not convertible to std::weak_ptr (with T = " + type_id() + ")"); } diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 67bd5901cd..708f0ea035 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -21,13 +21,13 @@ PYBIND11_NAMESPACE_BEGIN(detail) PYBIND11_INLINE std::string get_fully_qualified_tp_name(PyTypeObject *type) { #if !defined(PYPY_VERSION) - return type->tp_name; + return get_tp_name(type); #else auto module_name = handle((PyObject *) type).attr("__module__").cast(); if (module_name == PYBIND11_BUILTINS_MODULE) - return type->tp_name; + return get_tp_name(type); else - return std::move(module_name) + "." + type->tp_name; + return std::move(module_name) + "." + get_tp_name(type); #endif } diff --git a/include/pybind11/detail/function_record_pyobject.h b/include/pybind11/detail/function_record_pyobject.h index 42b0e96be7..38ee61d5f0 100644 --- a/include/pybind11/detail/function_record_pyobject.h +++ b/include/pybind11/detail/function_record_pyobject.h @@ -115,9 +115,8 @@ inline bool is_function_record_PyObject(PyObject *obj) { return true; } // This works across extension modules. Note that tp_name is versioned. - if (strcmp(obj_type->tp_name, function_record_PyTypeObject_methods::tp_qualname_impl) == 0 - || strcmp(obj_type->tp_name, function_record_PyTypeObject_methods::tp_plainname_impl) - == 0) { + if (tp_name_equals(obj_type, function_record_PyTypeObject_methods::tp_qualname_impl) + || tp_name_equals(obj_type, function_record_PyTypeObject_methods::tp_plainname_impl)) { return true; } return false; diff --git a/include/pybind11/pytypes-inl.h b/include/pybind11/pytypes-inl.h index f2eea1070f..0dd6965624 100644 --- a/include/pybind11/pytypes-inl.h +++ b/include/pybind11/pytypes-inl.h @@ -28,13 +28,7 @@ PYBIND11_INLINE error_fetch_and_normalize::error_fetch_and_normalize(const char + " called while " "Python error indicator not set."); } - const char *exc_type_name_orig = detail::obj_class_name(m_type.ptr()); - if (exc_type_name_orig == nullptr) { - pybind11_fail("Internal error: " + std::string(called) - + " failed to obtain the name " - "of the original active exception type."); - } - m_lazy_error_string = exc_type_name_orig; + m_lazy_error_string = detail::obj_class_name(m_type.ptr()); #if PY_VERSION_HEX >= 0x030C0000 // The presence of __notes__ is likely due to exception normalization // errors, although that is not necessarily true, therefore insert a @@ -56,12 +50,7 @@ PYBIND11_INLINE error_fetch_and_normalize::error_fetch_and_normalize(const char + " failed to normalize the " "active exception."); } - const char *exc_type_name_norm = detail::obj_class_name(m_type.ptr()); - if (exc_type_name_norm == nullptr) { - pybind11_fail("Internal error: " + std::string(called) - + " failed to obtain the name " - "of the normalized active exception type."); - } + std::string exc_type_name_norm = detail::obj_class_name(m_type.ptr()); if (exc_type_name_norm != m_lazy_error_string) { std::string msg = std::string(called) + ": MISMATCH of original and normalized " diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 021bcc95d5..d12a260791 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -500,12 +500,21 @@ T reinterpret_steal(handle h) { PYBIND11_NAMESPACE_BEGIN(detail) +/// The `tp_name` of a type: `module.Name` for static and pybind11 types, `Name` for Python +/// heap types. +inline std::string get_tp_name(PyTypeObject *type) { return type->tp_name; } + +/// True if `get_tp_name(type) == name`, without allocating in the common case. +inline bool tp_name_equals(PyTypeObject *type, const char *name) { + return std::strcmp(type->tp_name, name) == 0; +} + // Equivalent to obj.__class__.__name__ (or obj.__name__ if obj is a class). -inline const char *obj_class_name(PyObject *obj) { +inline std::string obj_class_name(PyObject *obj) { if (PyType_Check(obj)) { - return reinterpret_cast(obj)->tp_name; + return get_tp_name(reinterpret_cast(obj)); } - return Py_TYPE(obj)->tp_name; + return get_tp_name(Py_TYPE(obj)); } std::string error_string(); diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 01be0b47c6..f573e566d3 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -79,9 +79,8 @@ inline bool object_is_instance_with_one_of_tp_names(PyObject *obj, if (PyType_Check(obj)) { return false; } - const char *obj_tp_name = Py_TYPE(obj)->tp_name; for (const auto *tp_name : tp_names) { - if (std::strcmp(obj_tp_name, tp_name) == 0) { + if (tp_name_equals(Py_TYPE(obj), tp_name)) { return true; } } From e95a78a0b1e45beb90a728832dde6b79a5b91844 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:04:21 -0400 Subject: [PATCH 05/45] fix: use the internals registry in type_is_managed_by_our_internals() Comparing tp_new against this module's pybind11_object_new fails for types whose pybind11_object base was created by another module (the inherited pointer belongs to that module's copy). The registry lookup, which was the PyPy path, is correct across modules and does not read type-object fields. See https://github.com/pybind/pybind11/discussions/6104 (B5). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/cpp_conduit-inl.h | 15 ++++++++------- include/pybind11/detail/cpp_conduit.h | 3 --- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/include/pybind11/detail/cpp_conduit-inl.h b/include/pybind11/detail/cpp_conduit-inl.h index 60d0ef215e..6ec0fb4120 100644 --- a/include/pybind11/detail/cpp_conduit-inl.h +++ b/include/pybind11/detail/cpp_conduit-inl.h @@ -19,14 +19,15 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) +// True for types registered with these internals, and for types that derive from one and have +// been seen by all_type_info() already. The registry lookup is what every module sharing the +// internals agrees on; comparing tp_new against this module's pybind11_object_new would fail +// for types whose base was created by another module. PYBIND11_INLINE bool type_is_managed_by_our_internals(PyTypeObject *type_obj) { -#if defined(PYPY_VERSION) - auto &internals = get_internals(); - return bool(internals.registered_types_py.find(type_obj) - != internals.registered_types_py.end()); -#else - return (type_obj->tp_new == pybind11_object_new); -#endif + return with_internals([type_obj](internals &internals) { + auto it = internals.registered_types_py.find(type_obj); + return it != internals.registered_types_py.end() && !it->second.empty(); + }); } PYBIND11_INLINE bool is_instance_method_of_type(PyTypeObject *type_obj, PyObject *attr_name) { diff --git a/include/pybind11/detail/cpp_conduit.h b/include/pybind11/detail/cpp_conduit.h index 9560270782..ada5b607af 100644 --- a/include/pybind11/detail/cpp_conduit.h +++ b/include/pybind11/detail/cpp_conduit.h @@ -12,9 +12,6 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) -// Forward declaration needed here: Refactoring opportunity. -extern "C" PyObject *pybind11_object_new(PyTypeObject *type, PyObject *, PyObject *); - bool type_is_managed_by_our_internals(PyTypeObject *type_obj); bool is_instance_method_of_type(PyTypeObject *type_obj, PyObject *attr_name); From c7d6c4e440717b42b11134dfc108076d78f35cad Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:05:37 -0400 Subject: [PATCH 06/45] refactor: walk tracebacks by attribute in error_already_set::what() Read tb_next, tb_frame, f_back, co_filename and co_name through attribute access and use the public PyFrame_GetCode/PyFrame_GetLineNumber functions, instead of poking traceback, frame and code object struct fields, which are not part of the stable ABI. See https://github.com/pybind/pybind11/discussions/6104 (B6). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/pytypes-inl.h | 36 +++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/include/pybind11/pytypes-inl.h b/include/pybind11/pytypes-inl.h index 0dd6965624..29f05c34d0 100644 --- a/include/pybind11/pytypes-inl.h +++ b/include/pybind11/pytypes-inl.h @@ -136,30 +136,38 @@ PYBIND11_INLINE std::string error_fetch_and_normalize::format_value_and_trace() bool have_trace = false; if (m_trace) { #if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) - auto *tb = reinterpret_cast(m_trace.ptr()); + // Attribute access instead of struct access: the traceback, frame and code object + // layouts are not part of the stable ABI. Returns a null object on failure. + auto get_attr = [](handle obj, const char *name) { + auto value = reinterpret_steal(PyObject_GetAttrString(obj.ptr(), name)); + if (!value) { + PyErr_Clear(); + } + return value; + }; // Get the deepest trace possible. - while (tb->tb_next) { - tb = tb->tb_next; + object tb = m_trace; + for (object next = get_attr(tb, "tb_next"); next && !next.is_none(); + next = get_attr(tb, "tb_next")) { + tb = std::move(next); } - PyFrameObject *frame = tb->tb_frame; - Py_XINCREF(frame); + object frame = get_attr(tb, "tb_frame"); result += "\n\nAt:\n"; - while (frame) { - PyCodeObject *f_code = PyFrame_GetCode(frame); - int lineno = PyFrame_GetLineNumber(frame); + while (frame && !frame.is_none()) { + auto *frame_ptr = reinterpret_cast(frame.ptr()); + auto f_code = reinterpret_steal( + reinterpret_cast(PyFrame_GetCode(frame_ptr))); + int lineno = PyFrame_GetLineNumber(frame_ptr); result += " "; - result += handle(f_code->co_filename).cast(); + result += get_attr(f_code, "co_filename").cast(); result += '('; result += std::to_string(lineno); result += "): "; - result += handle(f_code->co_name).cast(); + result += get_attr(f_code, "co_name").cast(); result += '\n'; - Py_DECREF(f_code); - auto *b_frame = PyFrame_GetBack(frame); - Py_DECREF(frame); - frame = b_frame; + frame = get_attr(frame, "f_back"); } have_trace = true; From c3c93a80301f28a512c23a9e10cd109c518081b5 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:06:48 -0400 Subject: [PATCH 07/45] refactor: add detail::type_lookup() and use it instead of _PyType_Lookup _PyType_Lookup is a private CPython function and is not available under the stable ABI. Route the three users (metaclass getattro/setattro and the cpp_conduit instance-method check) through one helper. See https://github.com/pybind/pybind11/discussions/6104 (B7). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 19 +++++++++---------- include/pybind11/detail/cpp_conduit-inl.h | 4 ++-- include/pybind11/pytypes.h | 6 ++++++ 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 708f0ea035..74755dc1e5 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -113,24 +113,24 @@ class pybind11_static_property(property): #endif // PYPY extern "C" PYBIND11_INLINE int pybind11_meta_setattro(PyObject *obj, PyObject *name, PyObject *value) { - // Use `_PyType_Lookup()` instead of `PyObject_GetAttr()` in order to get the raw + // Use `type_lookup()` instead of `PyObject_GetAttr()` in order to get the raw // descriptor (`property`) instead of calling `tp_descr_get` (`property.__get__()`). - PyObject *descr = _PyType_Lookup((PyTypeObject *) obj, name); + object descr = type_lookup((PyTypeObject *) obj, name); // The following assignment combinations are possible: // 1. `Type.static_prop = value` --> descr_set: `Type.static_prop.__set__(value)` // 2. `Type.static_prop = other_static_prop` --> setattro: replace existing `static_prop` // 3. `Type.regular_attribute = value` --> setattro: regular attribute assignment auto *const static_prop = (PyObject *) get_internals().static_property_type; - const auto call_descr_set = (descr != nullptr) && (value != nullptr) - && (PyObject_IsInstance(descr, static_prop) != 0) + const auto call_descr_set = descr && (value != nullptr) + && (PyObject_IsInstance(descr.ptr(), static_prop) != 0) && (PyObject_IsInstance(value, static_prop) == 0); if (call_descr_set) { // Call `static_property.__set__()` instead of replacing the `static_property`. #if !defined(PYPY_VERSION) - return Py_TYPE(descr)->tp_descr_set(descr, obj, value); + return Py_TYPE(descr.ptr())->tp_descr_set(descr.ptr(), obj, value); #else - if (PyObject *result = PyObject_CallMethod(descr, "__set__", "OO", obj, value)) { + if (PyObject *result = PyObject_CallMethod(descr.ptr(), "__set__", "OO", obj, value)) { Py_DECREF(result); return 0; } else { @@ -144,10 +144,9 @@ pybind11_meta_setattro(PyObject *obj, PyObject *name, PyObject *value) { } extern "C" PYBIND11_INLINE PyObject *pybind11_meta_getattro(PyObject *obj, PyObject *name) { - PyObject *descr = _PyType_Lookup((PyTypeObject *) obj, name); - if (descr && PyInstanceMethod_Check(descr)) { - Py_INCREF(descr); - return descr; + object descr = type_lookup((PyTypeObject *) obj, name); + if (descr && PyInstanceMethod_Check(descr.ptr())) { + return descr.release().ptr(); } return PyType_Type.tp_getattro(obj, name); } diff --git a/include/pybind11/detail/cpp_conduit-inl.h b/include/pybind11/detail/cpp_conduit-inl.h index 6ec0fb4120..733b180cfa 100644 --- a/include/pybind11/detail/cpp_conduit-inl.h +++ b/include/pybind11/detail/cpp_conduit-inl.h @@ -31,8 +31,8 @@ PYBIND11_INLINE bool type_is_managed_by_our_internals(PyTypeObject *type_obj) { } PYBIND11_INLINE bool is_instance_method_of_type(PyTypeObject *type_obj, PyObject *attr_name) { - PyObject *descr = _PyType_Lookup(type_obj, attr_name); - return ((descr != nullptr) && PyInstanceMethod_Check(descr)); + object descr = type_lookup(type_obj, attr_name); + return descr && PyInstanceMethod_Check(descr.ptr()); } PYBIND11_INLINE object try_get_cpp_conduit_method(PyObject *obj) { diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index d12a260791..2dd157b28c 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -2485,6 +2485,12 @@ inline tuple get_bases(PyTypeObject *type) { /// `type.__mro__` as a tuple. inline tuple get_mro(PyTypeObject *type) { return reinterpret_borrow(type->tp_mro); } +/// Look `name` up along the MRO of `type` without invoking descriptors, like `_PyType_Lookup`. +/// Returns a null object if the attribute is not found; never raises. +inline object type_lookup(PyTypeObject *type, handle name) { + return reinterpret_borrow(_PyType_Lookup(type, name.ptr())); +} + inline object get_module_name_if_available(handle scope) { if (scope) { if (hasattr(scope, "__module__")) { From 7b35af837c2de5049bc3aa48bf822eda2a22248c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:07:38 -0400 Subject: [PATCH 08/45] refactor: use PyThreadState_GetInterpreter() instead of tstate->interp PyThreadState is opaque under the stable ABI; the accessor function is public since Python 3.9. See https://github.com/pybind/pybind11/discussions/6104 (B8). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/internals.h | 8 ++++---- include/pybind11/subinterpreter.h | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 60f3cfcae3..59162eb08d 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -220,7 +220,7 @@ inline PyThreadState *get_thread_state_unchecked() { inline PyInterpreterState *get_interpreter_state_unchecked() { auto *tstate = get_thread_state_unchecked(); - return tstate ? tstate->interp : nullptr; + return tstate ? PyThreadState_GetInterpreter(tstate) : nullptr; } object get_python_state_dict(); @@ -664,7 +664,7 @@ class internals_pp_manager { // internals_pp so that it can be pulled from the interpreter's state dict. That is // slow, so we use the current PyThreadState to check if it is necessary. auto *tstate = get_thread_state_unchecked(); - if (!tstate || tstate->interp != last_istate_tls()) { + if (!tstate || PyThreadState_GetInterpreter(tstate) != last_istate_tls()) { gil_scoped_acquire_simple gil; if (!tstate) { tstate = get_thread_state_unchecked(); @@ -672,7 +672,7 @@ class internals_pp_manager { // Update the cache only on success; a stale interp with a null pp would make // later calls return nullptr. auto *pp = get_or_create_pp_in_state_dict(); - last_istate_tls() = tstate->interp; + last_istate_tls() = PyThreadState_GetInterpreter(tstate); internals_p_tls() = pp; } return internals_p_tls(); @@ -702,7 +702,7 @@ class internals_pp_manager { if (has_seen_non_main_interpreter()) { auto *tstate = get_thread_state_unchecked(); // this could be called without an active interpreter, just use what was cached - if (!tstate || tstate->interp == last_istate_tls()) { + if (!tstate || PyThreadState_GetInterpreter(tstate) == last_istate_tls()) { auto tpp = internals_p_tls(); { std::lock_guard lock(pp_set_mutex_); diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index f1e4e44361..2ad90bfe5b 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -119,7 +119,7 @@ class subinterpreter { } // upon success, the new interpreter is activated in this thread - result.istate_ = result.creation_tstate_->interp; + result.istate_ = PyThreadState_GetInterpreter(result.creation_tstate_); detail::has_seen_non_main_interpreter() = true; detail::get_internals(); // initialize internals.tstate, amongst other things... @@ -184,7 +184,8 @@ class subinterpreter { old_tstate = PyThreadState_Swap(destroy_tstate); #endif - bool switch_back = (old_tstate != nullptr) && old_tstate->interp != istate_; + bool switch_back + = (old_tstate != nullptr) && PyThreadState_GetInterpreter(old_tstate) != istate_; // Internals always exists in the subinterpreter, this class enforces it when it creates // the subinterpreter. Even if it didn't, this only creates the pointer-to-pointer, not the From c800cc1487ac2d827b9fdaa5571f6d870f9476ca Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:11:21 -0400 Subject: [PATCH 09/45] refactor: add PYBIND11_HAS_DIRECT_STRUCT_ACCESS for the struct-access fallbacks Gate the PyPy fallbacks that exist because cpyext does not expose CPython struct layouts (static-property __set__, bool conversion, tuple/list fast iterators) on one semantic macro. The stable ABI needs the same fallbacks. See https://github.com/pybind/pybind11/discussions/6104 (B9). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/cast.h | 4 ++-- include/pybind11/detail/class-inl.h | 2 +- include/pybind11/detail/common.h | 7 +++++++ include/pybind11/pytypes.h | 2 +- tools/check_inl_headers.py | 4 +++- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 436761c3c3..75fbcf8b27 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -447,8 +447,8 @@ class type_caster { if (src.is_none()) { res = 0; // None is implicitly converted to False } -#if defined(PYPY_VERSION) - // On PyPy, check that "__bool__" attr exists +#if !defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) + // Check that the "__bool__" attr exists else if (hasattr(src, PYBIND11_BOOL_ATTR)) { res = PyObject_IsTrue(src.ptr()); } diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 74755dc1e5..1cff264780 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -127,7 +127,7 @@ pybind11_meta_setattro(PyObject *obj, PyObject *name, PyObject *value) { && (PyObject_IsInstance(value, static_prop) == 0); if (call_descr_set) { // Call `static_property.__set__()` instead of replacing the `static_property`. -#if !defined(PYPY_VERSION) +#if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) return Py_TYPE(descr.ptr())->tp_descr_set(descr.ptr(), obj, value); #else if (PyObject *result = PyObject_CallMethod(descr.ptr(), "__set__", "OO", obj, value)) { diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index a86187c76f..0c438bb219 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -242,6 +242,13 @@ # define PYBIND11_SIMPLE_GIL_MANAGEMENT #endif +// Direct access to CPython object struct fields and type slots (tp_*, nb_*, ob_item, ...) is +// available. Not on PyPy (cpyext emulates only part of the layouts) and not under the stable +// ABI (the structs are opaque). Code that is gated off this falls back to the public API. +#if !defined(PYPY_VERSION) && !defined(Py_LIMITED_API) +# define PYBIND11_HAS_DIRECT_STRUCT_ACCESS +#endif + #include #include #include diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 2dd157b28c..77766034f9 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1169,7 +1169,7 @@ class dict_readonly { }; PYBIND11_NAMESPACE_END(iterator_policies) -#if !defined(PYPY_VERSION) +#if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) using tuple_iterator = generic_iterator; using list_iterator = generic_iterator; #else diff --git a/tools/check_inl_headers.py b/tools/check_inl_headers.py index db95d85fb0..a81082c6cc 100755 --- a/tools/check_inl_headers.py +++ b/tools/check_inl_headers.py @@ -37,10 +37,12 @@ "__clang__", } -# Encoded in PYBIND11_PRECOMPILED_CONFIG_CHECK. +# Encoded in PYBIND11_PRECOMPILED_CONFIG_CHECK (PYBIND11_HAS_DIRECT_STRUCT_ACCESS through the +# Py_LIMITED_API bit and the environment). GUARDED_MACROS = { "PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET", "PYBIND11_DETAILED_ERROR_MESSAGES", + "PYBIND11_HAS_DIRECT_STRUCT_ACCESS", "PYBIND11_HAS_SUBINTERPRETER_SUPPORT", "PYBIND11_INTERNALS_VERSION", "PYBIND11_SIMPLE_GIL_MANAGEMENT", From cb3611c89eba1ce0e9640f86b8f0daec9275418d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:13:09 -0400 Subject: [PATCH 10/45] refactor: replace PyTuple/PyList/PyByteArray/PyCFunction macro accessors Use the function forms (available under the stable ABI) at the non-hot sites, and route the dispatcher and tuple/list caster sites through small detail:: wrappers that keep the macro forms when struct access is available. See https://github.com/pybind/pybind11/discussions/6104 (B10). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/cast.h | 4 ++-- include/pybind11/functional.h | 2 +- include/pybind11/pybind11-inl.h | 22 +++++++++++---------- include/pybind11/pybind11.h | 8 ++++---- include/pybind11/pytypes-inl.h | 2 +- include/pybind11/pytypes.h | 35 +++++++++++++++++++++++++++++---- include/pybind11/stl.h | 4 ++-- 7 files changed, 53 insertions(+), 24 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 75fbcf8b27..79f8df7301 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -849,7 +849,7 @@ class tuple_caster { tuple result(size); int counter = 0; for (auto &entry : entries) { - PyTuple_SET_ITEM(result.ptr(), counter++, entry.release().ptr()); + tuple_set_item(result.ptr(), counter++, entry.release().ptr()); } return result.release(); } @@ -1959,7 +1959,7 @@ typing::Tuple make_tuple(Args &&...args_) { tuple result(size); int counter = 0; for (auto &arg_value : args) { - PyTuple_SET_ITEM(result.ptr(), counter++, arg_value.release().ptr()); + detail::tuple_set_item(result.ptr(), counter++, arg_value.release().ptr()); } PYBIND11_WARNING_PUSH #ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING diff --git a/include/pybind11/functional.h b/include/pybind11/functional.h index c06fabe0d6..cfa0a88a08 100644 --- a/include/pybind11/functional.h +++ b/include/pybind11/functional.h @@ -82,7 +82,7 @@ struct type_caster> { captured variables), in which case the roundtrip can be avoided. */ if (auto cfunc = func.cpp_function()) { - auto *cfunc_self = PyCFunction_GET_SELF(cfunc.ptr()); + auto *cfunc_self = PyCFunction_GetSelf(cfunc.ptr()); if (cfunc_self == nullptr) { PyErr_Clear(); } else { diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index 1363be35be..ce8a286f6e 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -482,7 +482,7 @@ PYBIND11_INLINE function get_type_override(const void *this_ptr, # else PyObject *co_varnames = PyObject_GetAttrString((PyObject *) f_code, "co_varnames"); # endif - PyObject *self_arg = PyTuple_GET_ITEM(co_varnames, 0); + PyObject *self_arg = PyTuple_GetItem(co_varnames, 0); Py_DECREF(co_varnames); PyObject *self_caller = dict_getitem(locals, self_arg); Py_DECREF(locals); @@ -611,9 +611,9 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u detail::function_record *chain = nullptr, *chain_start = rec; if (rec->sibling) { if (PyCFunction_Check(rec->sibling.ptr())) { - auto *self = PyCFunction_GET_SELF(rec->sibling.ptr()); + auto *self = PyCFunction_GetSelf(rec->sibling.ptr()); if (self == nullptr) { - pybind11_fail("initialize_generic: Unexpected nullptr from PyCFunction_GET_SELF"); + pybind11_fail("initialize_generic: Unexpected nullptr from PyCFunction_GetSelf"); } chain = detail::function_record_ptr_from_PyObject(self); if (chain && !chain->scope.is(rec->scope)) { @@ -674,8 +674,8 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u // chain. chain_start = rec; rec->next = chain; - auto *py_func_rec = reinterpret_cast( - PyCFunction_GET_SELF(m_ptr)); + auto *py_func_rec + = reinterpret_cast(PyCFunction_GetSelf(m_ptr)); py_func_rec->cpp_func_rec = unique_rec.release(); guarded_strdup.release(); } else { @@ -976,7 +976,7 @@ PYBIND11_INLINE PyObject *cpp_function::dispatcher(PyObject *self, // 2. Check kwargs and, failing that, defaults that may help complete the list small_vector used_kwargs( - kwnames_in ? static_cast(PyTuple_GET_SIZE(kwnames_in)) : 0, false); + kwnames_in ? static_cast(detail::tuple_size(kwnames_in)) : 0, false); size_t used_kwargs_count = 0; if (args_copied < num_args) { for (; args_copied < num_args; ++args_copied) { @@ -1052,7 +1052,8 @@ PYBIND11_INLINE PyObject *cpp_function::dispatcher(PyObject *self, // Cast values into handles before indexing into kwargs to ensure // well-defined evaluation order (MSVC C4866). handle arg_in_arr = args_in_arr[n_args_in + i], - kwname = PyTuple_GET_ITEM(kwnames_in, i); + kwname + = detail::tuple_get_item(kwnames_in, static_cast(i)); kwargs[kwname] = arg_in_arr; } } @@ -1206,19 +1207,20 @@ PYBIND11_INLINE PyObject *cpp_function::dispatcher(PyObject *self, msg += ""; } } - if (kwnames_in && PyTuple_GET_SIZE(kwnames_in) > 0) { + if (kwnames_in && detail::tuple_size(kwnames_in) > 0) { if (some_args) { msg += "; "; } msg += "kwargs: "; bool first = true; - for (size_t i = 0; i < static_cast(PyTuple_GET_SIZE(kwnames_in)); ++i) { + for (size_t i = 0; i < static_cast(detail::tuple_size(kwnames_in)); ++i) { if (first) { first = false; } else { msg += ", "; } - msg += reinterpret_borrow(PyTuple_GET_ITEM(kwnames_in, i)); + msg += reinterpret_borrow( + detail::tuple_get_item(kwnames_in, static_cast(i))); msg += '='; try { msg += pybind11::repr(args_in_arr[n_args_in + i]); diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 8cf3f9c4bb..0d48897a3e 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -562,15 +562,15 @@ class cpp_function : public function { * second lexicographic check. This is wildly expensive for huge argument lists, but those * are incredibly rare so we optimize for the vastly common case of just a couple of args. */ - auto n = PyTuple_GET_SIZE(haystack); + auto n = detail::tuple_size(haystack); auto s = reinterpret_steal(PyUnicode_InternFromString(needle)); for (ssize_t i = 0; i < n; ++i) { - if (PyTuple_GET_ITEM(haystack, i) == s.ptr()) { + if (detail::tuple_get_item(haystack, i) == s.ptr()) { return i; } } for (ssize_t i = 0; i < n; ++i) { - if (PyUnicode_Compare(PyTuple_GET_ITEM(haystack, i), s.ptr()) == 0) { + if (PyUnicode_Compare(detail::tuple_get_item(haystack, i), s.ptr()) == 0) { return i; } } @@ -1851,7 +1851,7 @@ class class_ : public detail::generic_type { return nullptr; } - handle func_self = PyCFunction_GET_SELF(h.ptr()); + handle func_self = PyCFunction_GetSelf(h.ptr()); if (!func_self) { throw error_already_set(); } diff --git a/include/pybind11/pytypes-inl.h b/include/pybind11/pytypes-inl.h index 29f05c34d0..699fb5323d 100644 --- a/include/pybind11/pytypes-inl.h +++ b/include/pybind11/pytypes-inl.h @@ -105,7 +105,7 @@ PYBIND11_INLINE std::string error_fetch_and_normalize::format_value_and_trace() } else { result += "\n__notes__ (len=" + std::to_string(len_notes) + "):"; for (ssize_t i = 0; i < len_notes; i++) { - PyObject *note = PyList_GET_ITEM(notes.ptr(), i); + PyObject *note = PyList_GetItem(notes.ptr(), i); auto note_bytes = reinterpret_steal( PyUnicode_AsEncodedString(note, "utf-8", "backslashreplace")); if (!note_bytes) { diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 77766034f9..ab7c36aa64 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -509,6 +509,33 @@ inline bool tp_name_equals(PyTypeObject *type, const char *name) { return std::strcmp(type->tp_name, name) == 0; } +// Unchecked tuple/list element access for hot paths. Function forms under the stable ABI. +#if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) +inline ssize_t tuple_size(PyObject *tup) { return PyTuple_GET_SIZE(tup); } +inline PyObject *tuple_get_item(PyObject *tup, ssize_t i) { return PyTuple_GET_ITEM(tup, i); } +/// Steals a reference to `item`; `tup` must be a fresh tuple. +inline void tuple_set_item(PyObject *tup, ssize_t i, PyObject *item) { + PyTuple_SET_ITEM(tup, i, item); +} +inline ssize_t list_size(PyObject *lst) { return PyList_GET_SIZE(lst); } +inline PyObject *list_get_item(PyObject *lst, ssize_t i) { return PyList_GET_ITEM(lst, i); } +/// Steals a reference to `item`; `lst` must be a fresh list. +inline void list_set_item(PyObject *lst, ssize_t i, PyObject *item) { + PyList_SET_ITEM(lst, i, item); +} +#else +inline ssize_t tuple_size(PyObject *tup) { return PyTuple_Size(tup); } +inline PyObject *tuple_get_item(PyObject *tup, ssize_t i) { return PyTuple_GetItem(tup, i); } +inline void tuple_set_item(PyObject *tup, ssize_t i, PyObject *item) { + PyTuple_SetItem(tup, i, item); +} +inline ssize_t list_size(PyObject *lst) { return PyList_Size(lst); } +inline PyObject *list_get_item(PyObject *lst, ssize_t i) { return PyList_GetItem(lst, i); } +inline void list_set_item(PyObject *lst, ssize_t i, PyObject *item) { + PyList_SetItem(lst, i, item); +} +#endif + // Equivalent to obj.__class__.__name__ (or obj.__name__ if obj is a class). inline std::string obj_class_name(PyObject *obj) { if (PyType_Check(obj)) { @@ -1649,8 +1676,8 @@ class bytearray : public object { size_t size() const { return static_cast(PyByteArray_Size(m_ptr)); } explicit operator std::string() const { - char *buffer = PyByteArray_AS_STRING(m_ptr); - ssize_t size = PyByteArray_GET_SIZE(m_ptr); + char *buffer = PyByteArray_AsString(m_ptr); + ssize_t size = PyByteArray_Size(m_ptr); return std::string(buffer, static_cast(size)); } }; @@ -1973,7 +2000,7 @@ class tuple : public object { return object::operator[](std::forward(o)); } detail::tuple_iterator begin() const { return {*this, 0}; } - detail::tuple_iterator end() const { return {*this, PyTuple_GET_SIZE(m_ptr)}; } + detail::tuple_iterator end() const { return {*this, detail::tuple_size(m_ptr)}; } }; // We need to put this into a separate function because the Intel compiler @@ -2063,7 +2090,7 @@ class list : public object { return object::operator[](std::forward(o)); } detail::list_iterator begin() const { return {*this, 0}; } - detail::list_iterator end() const { return {*this, PyList_GET_SIZE(m_ptr)}; } + detail::list_iterator end() const { return {*this, detail::list_size(m_ptr)}; } template void append(T &&val) /* py-non-const */ { if (PyList_Append(m_ptr, detail::object_or_cast(std::forward(val)).ptr()) != 0) { diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index f573e566d3..7a2ac29c7c 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -351,7 +351,7 @@ struct list_caster { if (!value_) { return handle(); } - PyList_SET_ITEM(l.ptr(), index++, value_.release().ptr()); // steals a reference + list_set_item(l.ptr(), index++, value_.release().ptr()); // steals a reference } return l.release(); } @@ -461,7 +461,7 @@ struct array_caster { if (!value_) { return handle(); } - PyList_SET_ITEM(l.ptr(), index++, value_.release().ptr()); // steals a reference + list_set_item(l.ptr(), index++, value_.release().ptr()); // steals a reference } return l.release(); } From 600c4a48cd7d2f1451ff22bb7c1c9525f2b58ba5 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:16:51 -0400 Subject: [PATCH 11/45] feat: add the PYBIND11_TYPE_CREATION_VIA_SPEC opt-in macro and CI job The macro will select PyType_FromMetaclass-based type creation on CPython 3.12+. It is a no-op for now; this adds the switch, its precompiled config-check bit, a CMake option, and a CI matrix that builds with it on. See https://github.com/pybind/pybind11/discussions/6104 (C1). Assisted-by: ClaudeCode:claude-fable-5-1 --- .github/workflows/ci.yml | 28 ++++++++++++++++++++++++++++ CMakeLists.txt | 5 +++++ docs/compiling.rst | 3 ++- include/pybind11/detail/common.h | 10 ++++++++++ include/pybind11/detail/internals.h | 16 +++++++++++----- tools/check_inl_headers.py | 1 + 6 files changed, 57 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41e21dc129..b47e5c5120 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,6 +99,34 @@ jobs: python-version: ${{ matrix.python-version }} cmake-args: -DPYBIND11_TEST_PRECOMPILE=ON + # Opt-in PyType_FromMetaclass type creation (the future default on CPython 3.12+ + # and the only path under the stable ABI). + type-creation-via-spec: + if: github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + include: + - runs-on: ubuntu-latest + python-version: '3.12' + cmake-args: -DPYBIND11_TYPE_CREATION_VIA_SPEC=ON + - runs-on: ubuntu-latest + python-version: '3.14' + cmake-args: -DPYBIND11_TYPE_CREATION_VIA_SPEC=ON -DPYBIND11_TEST_PRECOMPILE=ON + - runs-on: macos-latest + python-version: '3.13' + cmake-args: -DPYBIND11_TYPE_CREATION_VIA_SPEC=ON + - runs-on: windows-2022 + python-version: '3.13' + cmake-args: -DPYBIND11_TYPE_CREATION_VIA_SPEC=ON + + name: 📐 + uses: ./.github/workflows/reusable-standard.yml + with: + runs-on: ${{ matrix.runs-on }} + python-version: ${{ matrix.python-version }} + cmake-args: ${{ matrix.cmake-args }} + standard-large: if: github.event.pull_request.draft == false strategy: diff --git a/CMakeLists.txt b/CMakeLists.txt index 46f0a1f11d..024626e5d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,6 +99,8 @@ option(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION "To enforce that a handle_type_name<> specialization exists" OFF) option(PYBIND11_SIMPLE_GIL_MANAGEMENT "Use simpler GIL management logic that does not support disassociation" OFF) +option(PYBIND11_TYPE_CREATION_VIA_SPEC + "Create type objects with PyType_FromMetaclass (CPython 3.12+ only, opt-in preview)" OFF) set(PYBIND11_INTERNALS_VERSION "" CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.") @@ -323,6 +325,9 @@ if(NOT TARGET pybind11_headers) if(PYBIND11_SIMPLE_GIL_MANAGEMENT) target_compile_definitions(pybind11_headers INTERFACE PYBIND11_SIMPLE_GIL_MANAGEMENT) endif() + if(PYBIND11_TYPE_CREATION_VIA_SPEC) + target_compile_definitions(pybind11_headers INTERFACE PYBIND11_TYPE_CREATION_VIA_SPEC) + endif() else() # It is invalid to install a target twice, too. set(PYBIND11_INSTALL OFF) diff --git a/docs/compiling.rst b/docs/compiling.rst index e926c2cc67..9e61949b02 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -436,7 +436,8 @@ Requirements and caveats: ``PYBIND11_SIMPLE_GIL_MANAGEMENT``, ``PYBIND11_DETAILED_ERROR_MESSAGES`` (defaults on in debug builds), ``PYBIND11_HAS_SUBINTERPRETER_SUPPORT``, - ``PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET``, and + ``PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET``, + ``PYBIND11_TYPE_CREATION_VIA_SPEC``, and ``Py_LIMITED_API``. A mismatch produces one readable undefined symbol at link time referencing ``pybind11_precompiled_config``. diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 0c438bb219..8ec9e0a875 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -242,6 +242,16 @@ # define PYBIND11_SIMPLE_GIL_MANAGEMENT #endif +// Opt-in: create the pybind11 type objects (metaclass, static property, instance base and all +// bound classes) through PyType_FromMetaclass() with PyType_Spec instead of filling in raw +// PyHeapTypeObject fields. Requires CPython 3.12+. This is the only type-creation path under the +// stable ABI and is planned to become the default on CPython 3.12+. +#if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +# if PY_VERSION_HEX < 0x030C0000 || defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) +# error "PYBIND11_TYPE_CREATION_VIA_SPEC requires CPython 3.12 or newer." +# endif +#endif + // Direct access to CPython object struct fields and type slots (tp_*, nb_*, ob_item, ...) is // available. Not on PyPy (cpyext emulates only part of the layouts) and not under the stable // ABI (the structs are opaque). Code that is gated off this falls back to the public API. diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 59162eb08d..02906d0573 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -84,11 +84,16 @@ # else # define PYBIND11_PRECOMPILED_CFG_LA 0 # endif +# if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +# define PYBIND11_PRECOMPILED_CFG_TS 1 +# else +# define PYBIND11_PRECOMPILED_CFG_TS 0 +# endif // PYBIND11_CONCAT does not macro-expand its arguments (## suppresses expansion). -# define PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de, si, td, la) \ - pybind11_precompiled_config_v##v##_gd##gd##_sg##sg##_de##de##_si##si##_td##td##_la##la -# define PYBIND11_PRECOMPILED_CONFIG_NAME(v, gd, sg, de, si, td, la) \ - PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de, si, td, la) +# define PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de, si, td, la, ts) \ + pybind11_precompiled_config_v##v##_gd##gd##_sg##sg##_de##de##_si##si##_td##td##_la##la##_ts##ts +# define PYBIND11_PRECOMPILED_CONFIG_NAME(v, gd, sg, de, si, td, la, ts) \ + PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de, si, td, la, ts) # define PYBIND11_PRECOMPILED_CONFIG_CHECK \ PYBIND11_PRECOMPILED_CONFIG_NAME(PYBIND11_INTERNALS_VERSION, \ PYBIND11_PRECOMPILED_CFG_GD, \ @@ -96,7 +101,8 @@ PYBIND11_PRECOMPILED_CFG_DE, \ PYBIND11_PRECOMPILED_CFG_SI, \ PYBIND11_PRECOMPILED_CFG_TD, \ - PYBIND11_PRECOMPILED_CFG_LA) + PYBIND11_PRECOMPILED_CFG_LA, \ + PYBIND11_PRECOMPILED_CFG_TS) # define PYBIND11_PRECOMPILED_CONFIG_GUARD \ ::pybind11::detail::PYBIND11_PRECOMPILED_CONFIG_CHECK(); #else diff --git a/tools/check_inl_headers.py b/tools/check_inl_headers.py index a81082c6cc..eb2ca87ad3 100755 --- a/tools/check_inl_headers.py +++ b/tools/check_inl_headers.py @@ -46,6 +46,7 @@ "PYBIND11_HAS_SUBINTERPRETER_SUPPORT", "PYBIND11_INTERNALS_VERSION", "PYBIND11_SIMPLE_GIL_MANAGEMENT", + "PYBIND11_TYPE_CREATION_VIA_SPEC", "Py_GIL_DISABLED", "Py_LIMITED_API", } From 7f952674000130ae5ea1820de7547f1bffd130e4 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:19:00 -0400 Subject: [PATCH 12/45] refactor: forward to type/property slots through PyType_GetSlot() The metaclass and static-property implementations called the tp_* fields of PyType_Type and PyProperty_Type directly. Look the slots up once per module with PyType_GetSlot(), which works on static types since 3.10 and is part of the stable ABI; older versions and PyPy keep the direct reads. See https://github.com/pybind/pybind11/discussions/6104 (C2). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 40 ++++++++++++++++++++++++----- include/pybind11/detail/class.h | 8 ++++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 1cff264780..a1d5badd3e 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -36,16 +36,44 @@ PYBIND11_INLINE PyTypeObject *type_incref(PyTypeObject *type) { return type; } +// Slots of the static base types that the pybind11 metaclass and static property type forward +// to. PyType_GetSlot() works on static types since Python 3.10 and is part of the stable ABI; +// the function-local caches make each lookup happen once per module. Below 3.10 (and on PyPy, +// where PyType_GetSlot on static types is not reliable) the struct fields are read directly. +#if PY_VERSION_HEX >= 0x030A0000 && !defined(PYPY_VERSION) +# define PYBIND11_BASE_TYPE_SLOT(type, slot_id, field, slot_type) \ + static const auto cached = reinterpret_cast(PyType_GetSlot(&type, slot_id)); \ + return cached; +#else +# define PYBIND11_BASE_TYPE_SLOT(type, slot_id, field, slot_type) return type.field; +#endif + +PYBIND11_INLINE ternaryfunc type_type_call(){PYBIND11_BASE_TYPE_SLOT( + PyType_Type, Py_tp_call, tp_call, ternaryfunc)} PYBIND11_INLINE setattrofunc + type_type_setattro(){PYBIND11_BASE_TYPE_SLOT( + PyType_Type, Py_tp_setattro, tp_setattro, setattrofunc)} PYBIND11_INLINE getattrofunc + type_type_getattro(){PYBIND11_BASE_TYPE_SLOT( + PyType_Type, Py_tp_getattro, tp_getattro, getattrofunc)} PYBIND11_INLINE destructor + type_type_dealloc(){PYBIND11_BASE_TYPE_SLOT( + PyType_Type, Py_tp_dealloc, tp_dealloc, destructor)} PYBIND11_INLINE descrgetfunc + property_type_descr_get(){PYBIND11_BASE_TYPE_SLOT( + PyProperty_Type, Py_tp_descr_get, tp_descr_get, descrgetfunc)} PYBIND11_INLINE descrsetfunc + property_type_descr_set() { + PYBIND11_BASE_TYPE_SLOT(PyProperty_Type, Py_tp_descr_set, tp_descr_set, descrsetfunc) +} + +#undef PYBIND11_BASE_TYPE_SLOT + #if !defined(PYPY_VERSION) extern "C" PYBIND11_INLINE PyObject * pybind11_static_get(PyObject *self, PyObject * /*ob*/, PyObject *cls) { - return PyProperty_Type.tp_descr_get(self, cls, cls); + return property_type_descr_get()(self, cls, cls); } extern "C" PYBIND11_INLINE int pybind11_static_set(PyObject *self, PyObject *obj, PyObject *value) { PyObject *cls = PyType_Check(obj) ? obj : (PyObject *) Py_TYPE(obj); - return PyProperty_Type.tp_descr_set(self, cls, value); + return property_type_descr_set()(self, cls, value); } PYBIND11_INLINE PyTypeObject *make_static_property_type() { @@ -139,7 +167,7 @@ pybind11_meta_setattro(PyObject *obj, PyObject *name, PyObject *value) { #endif } else { // Replace existing attribute. - return PyType_Type.tp_setattro(obj, name, value); + return type_type_setattro()(obj, name, value); } } @@ -148,14 +176,14 @@ extern "C" PYBIND11_INLINE PyObject *pybind11_meta_getattro(PyObject *obj, PyObj if (descr && PyInstanceMethod_Check(descr.ptr())) { return descr.release().ptr(); } - return PyType_Type.tp_getattro(obj, name); + return type_type_getattro()(obj, name); } extern "C" PYBIND11_INLINE PyObject * pybind11_meta_call(PyObject *type, PyObject *args, PyObject *kwargs) { // use the default metaclass call to create/initialize the object - PyObject *self = PyType_Type.tp_call(type, args, kwargs); + PyObject *self = type_type_call()(type, args, kwargs); if (self == nullptr) { return nullptr; } @@ -220,7 +248,7 @@ extern "C" PYBIND11_INLINE void pybind11_meta_dealloc(PyObject *obj) { } }); - PyType_Type.tp_dealloc(obj); + type_type_dealloc()(obj); } PYBIND11_INLINE PyTypeObject *make_default_metaclass() { diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 74e5ef0593..29f78af390 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -37,6 +37,14 @@ std::string get_fully_qualified_tp_name(PyTypeObject *type); PyTypeObject *type_incref(PyTypeObject *type); +/// Slots of `type` and `property` that pybind11's own types forward to. +ternaryfunc type_type_call(); +setattrofunc type_type_setattro(); +getattrofunc type_type_getattro(); +destructor type_type_dealloc(); +descrgetfunc property_type_descr_get(); +descrsetfunc property_type_descr_set(); + #if !defined(PYPY_VERSION) /// `pybind11_static_property.__get__()`: Always pass the class instead of the instance. From cfb5a74f780d41f6070bdab9842b63ed86e1307b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:23:21 -0400 Subject: [PATCH 13/45] feat: spec-based pybind11_static_property behind PYBIND11_TYPE_CREATION_VIA_SPEC Create the static-property type with PyType_FromMetaclass() when the opt-in macro is defined. The __dict__ getset is shared with the legacy path through dynamic_attr_getset(). See https://github.com/pybind/pybind11/discussions/6104 (C3). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 51 ++++++++++++++++++++++++----- include/pybind11/detail/class.h | 3 ++ 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index a1d5badd3e..e59012d81e 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -76,6 +76,34 @@ pybind11_static_set(PyObject *self, PyObject *obj, PyObject *value) { return property_type_descr_set()(self, cls, value); } +# if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) + +PYBIND11_INLINE PyTypeObject *make_static_property_type() { + // Since Python-3.12 property-derived types are required to have dynamic attributes (to set + // `__doc__`), hence the GC and dict slots. + static PyType_Slot slots[] = {{Py_tp_descr_get, reinterpret_cast(pybind11_static_get)}, + {Py_tp_descr_set, reinterpret_cast(pybind11_static_set)}, + {Py_tp_traverse, reinterpret_cast(pybind11_traverse)}, + {Py_tp_clear, reinterpret_cast(pybind11_clear)}, + {Py_tp_getset, reinterpret_cast(dynamic_attr_getset())}, + {0, nullptr}}; + static PyType_Spec spec + = {PYBIND11_DUMMY_MODULE_NAME ".pybind11_static_property", + 0, // inherit from property + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MANAGED_DICT, + slots}; + PyObject *type = PyType_FromMetaclass( + nullptr, nullptr, &spec, reinterpret_cast(&PyProperty_Type)); + if (!type) { + pybind11_fail("make_static_property_type(): failure in PyType_FromMetaclass(): " + + error_string()); + } + return reinterpret_cast(type); +} + +# else // legacy: fill in a PyHeapTypeObject by hand + PYBIND11_INLINE PyTypeObject *make_static_property_type() { constexpr auto *name = "pybind11_static_property"; auto name_obj = reinterpret_steal(PYBIND11_FROM_STRING(name)); @@ -90,9 +118,9 @@ PYBIND11_INLINE PyTypeObject *make_static_property_type() { } heap_type->ht_name = name_obj.inc_ref().ptr(); -# ifdef PYBIND11_BUILTIN_QUALNAME +# ifdef PYBIND11_BUILTIN_QUALNAME heap_type->ht_qualname = name_obj.inc_ref().ptr(); -# endif +# endif auto *type = &heap_type->ht_type; type->tp_name = name; @@ -101,11 +129,11 @@ PYBIND11_INLINE PyTypeObject *make_static_property_type() { type->tp_descr_get = pybind11_static_get; type->tp_descr_set = pybind11_static_set; -# if PY_VERSION_HEX >= 0x030C0000 +# if PY_VERSION_HEX >= 0x030C0000 // Since Python-3.12 property-derived types are required to // have dynamic attributes (to set `__doc__`) enable_dynamic_attributes(heap_type); -# endif +# endif if (PyType_Ready(type) < 0) { pybind11_fail("make_static_property_type(): failure in PyType_Ready()!"); @@ -117,6 +145,8 @@ PYBIND11_INLINE PyTypeObject *make_static_property_type() { return type; } +# endif // PYBIND11_TYPE_CREATION_VIA_SPEC + #else // PYPY PYBIND11_INLINE PyTypeObject *make_static_property_type() { auto d = dict(); @@ -570,6 +600,13 @@ extern "C" PYBIND11_INLINE int pybind11_clear(PyObject *self) { return 0; } +PYBIND11_INLINE PyGetSetDef *dynamic_attr_getset() { + static PyGetSetDef getset[] + = {{"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, nullptr, nullptr}, + {nullptr, nullptr, nullptr, nullptr, nullptr}}; + return getset; +} + PYBIND11_INLINE void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; @@ -581,11 +618,7 @@ PYBIND11_INLINE void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { #endif type->tp_traverse = pybind11_traverse; type->tp_clear = pybind11_clear; - - static PyGetSetDef getset[] - = {{"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, nullptr, nullptr}, - {nullptr, nullptr, nullptr, nullptr, nullptr}}; - type->tp_getset = getset; + type->tp_getset = dynamic_attr_getset(); } extern "C" PYBIND11_INLINE int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int flags) { diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 29f78af390..acce47ad55 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -151,6 +151,9 @@ extern "C" int pybind11_traverse(PyObject *self, visitproc visit, void *arg); /// dynamic_attr: Allow the GC to clear the dictionary. extern "C" int pybind11_clear(PyObject *self); +/// The `__dict__` descriptor for types with dynamic attributes. +PyGetSetDef *dynamic_attr_getset(); + /// Give instances of this type a `__dict__` and opt into garbage collection. void enable_dynamic_attributes(PyHeapTypeObject *heap_type); From b2649fdc1b05e492edf6536dd236efdca7759c28 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:24:55 -0400 Subject: [PATCH 14/45] feat: spec-based pybind11_type metaclass behind PYBIND11_TYPE_CREATION_VIA_SPEC See https://github.com/pybind/pybind11/discussions/6104 (C4). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 31 +++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index e59012d81e..2baadba3f0 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -281,6 +281,31 @@ extern "C" PYBIND11_INLINE void pybind11_meta_dealloc(PyObject *obj) { type_type_dealloc()(obj); } +#if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) + +PYBIND11_INLINE PyTypeObject *make_default_metaclass() { + static PyType_Slot slots[] + = {{Py_tp_call, reinterpret_cast(pybind11_meta_call)}, + {Py_tp_setattro, reinterpret_cast(pybind11_meta_setattro)}, + {Py_tp_getattro, reinterpret_cast(pybind11_meta_getattro)}, + {Py_tp_dealloc, reinterpret_cast(pybind11_meta_dealloc)}, + {0, nullptr}}; + static PyType_Spec spec = {PYBIND11_DUMMY_MODULE_NAME ".pybind11_type", + 0, // inherit from type + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + slots}; + PyObject *type = PyType_FromMetaclass( + nullptr, nullptr, &spec, reinterpret_cast(&PyType_Type)); + if (!type) { + pybind11_fail("make_default_metaclass(): failure in PyType_FromMetaclass(): " + + error_string()); + } + return reinterpret_cast(type); +} + +#else // legacy: fill in a PyHeapTypeObject by hand + PYBIND11_INLINE PyTypeObject *make_default_metaclass() { constexpr auto *name = "pybind11_type"; auto name_obj = reinterpret_steal(PYBIND11_FROM_STRING(name)); @@ -295,9 +320,9 @@ PYBIND11_INLINE PyTypeObject *make_default_metaclass() { } heap_type->ht_name = name_obj.inc_ref().ptr(); -#ifdef PYBIND11_BUILTIN_QUALNAME +# ifdef PYBIND11_BUILTIN_QUALNAME heap_type->ht_qualname = name_obj.inc_ref().ptr(); -#endif +# endif auto *type = &heap_type->ht_type; type->tp_name = name; @@ -321,6 +346,8 @@ PYBIND11_INLINE PyTypeObject *make_default_metaclass() { return type; } +#endif // PYBIND11_TYPE_CREATION_VIA_SPEC + PYBIND11_INLINE void traverse_offset_bases(void *valueptr, const detail::type_info *tinfo, instance *self, From d6e3f6480c55816ed12f21c56a07008fc7f8419f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:26:25 -0400 Subject: [PATCH 15/45] feat: spec-based pybind11_object base behind PYBIND11_TYPE_CREATION_VIA_SPEC Weak-reference support moves from tp_weaklistoffset to a __weaklistoffset__ member, which is how PyType_Spec expresses it. See https://github.com/pybind/pybind11/discussions/6104 (C5). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 36 +++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 2baadba3f0..d2c73ab867 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -560,6 +560,36 @@ extern "C" PYBIND11_INLINE void pybind11_object_dealloc(PyObject *self) { Py_DECREF(type); } +#if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) + +PYBIND11_INLINE PyObject *make_object_base_type(PyTypeObject *metaclass) { + /* Support weak references (needed for the keep_alive feature) */ + static PyMemberDef members[] = { + {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(instance, weakrefs), Py_READONLY, nullptr}, + {nullptr, 0, 0, 0, nullptr}}; + static PyType_Slot slots[] + = {{Py_tp_new, reinterpret_cast(pybind11_object_new)}, + {Py_tp_init, reinterpret_cast(pybind11_object_init)}, + {Py_tp_dealloc, reinterpret_cast(pybind11_object_dealloc)}, + {Py_tp_members, reinterpret_cast(members)}, + {0, nullptr}}; + static PyType_Spec spec = {PYBIND11_DUMMY_MODULE_NAME ".pybind11_object", + static_cast(sizeof(instance)), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + slots}; + PyObject *type = PyType_FromMetaclass( + metaclass, nullptr, &spec, reinterpret_cast(&PyBaseObject_Type)); + if (!type) { + pybind11_fail("make_object_base_type(): failure in PyType_FromMetaclass(): " + + error_string()); + } + assert(!PyType_HasFeature(reinterpret_cast(type), Py_TPFLAGS_HAVE_GC)); + return type; +} + +#else // legacy: fill in a PyHeapTypeObject by hand + PYBIND11_INLINE PyObject *make_object_base_type(PyTypeObject *metaclass) { constexpr auto *name = "pybind11_object"; auto name_obj = reinterpret_steal(PYBIND11_FROM_STRING(name)); @@ -574,9 +604,9 @@ PYBIND11_INLINE PyObject *make_object_base_type(PyTypeObject *metaclass) { } heap_type->ht_name = name_obj.inc_ref().ptr(); -#ifdef PYBIND11_BUILTIN_QUALNAME +# ifdef PYBIND11_BUILTIN_QUALNAME heap_type->ht_qualname = name_obj.inc_ref().ptr(); -#endif +# endif auto *type = &heap_type->ht_type; type->tp_name = name; @@ -602,6 +632,8 @@ PYBIND11_INLINE PyObject *make_object_base_type(PyTypeObject *metaclass) { return reinterpret_cast(heap_type); } +#endif // PYBIND11_TYPE_CREATION_VIA_SPEC + extern "C" PYBIND11_INLINE int pybind11_traverse(PyObject *self, visitproc visit, void *arg) { #if PY_VERSION_HEX >= 0x030D0000 int ret = PyObject_VisitManagedDict(self, visit, arg); From 0528f8a919e2da43251d4febbcbb5f9ccb65871e Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:31:50 -0400 Subject: [PATCH 16/45] feat: spec-based make_new_python_type() behind PYBIND11_TYPE_CREATION_VIA_SPEC Bound classes are created with PyType_FromMetaclass(): bases are passed as a tuple, the docstring through Py_tp_doc (no manual tp_doc allocation), dynamic attributes through Py_TPFLAGS_MANAGED_DICT plus traverse/clear slots, the buffer protocol through Py_bf_* slots, and py::is_final by omitting Py_TPFLAGS_BASETYPE. __qualname__ is set after creation. The hand-filled path stays as make_new_python_type_legacy() and is still used for py::custom_type_setup, metaclasses with a custom tp_new, and a py::metaclass less derived than pybind11_type, which PyType_FromMetaclass refuses. See https://github.com/pybind/pybind11/discussions/6104 (C6). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 96 +++++++++++++++++++++++++++++ include/pybind11/detail/class.h | 5 ++ 2 files changed, 101 insertions(+) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index d2c73ab867..a63c95097f 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -791,7 +791,103 @@ PYBIND11_INLINE void enable_buffer_protocol(PyHeapTypeObject *heap_type) { heap_type->as_buffer.bf_releasebuffer = pybind11_releasebuffer; } +#if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) + +PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { + auto &internals = get_internals(); + auto *metaclass = rec.metaclass.ptr() ? reinterpret_cast(rec.metaclass.ptr()) + : internals.default_metaclass; + + // PyType_FromMetaclass() cannot run a pre-PyType_Ready callback, rejects metaclasses with a + // custom tp_new, and (correctly) refuses a metaclass that is less derived than the one of + // the base; the legacy path accepts all three. + if (rec.custom_type_setup_callback + || PyType_GetSlot(metaclass, Py_tp_new) != PyType_GetSlot(&PyType_Type, Py_tp_new) + || (metaclass != internals.default_metaclass + && !PyType_IsSubtype(metaclass, internals.default_metaclass))) { + return make_new_python_type_legacy(rec); + } + + object module_ = get_module_name_if_available(rec.scope); + // Persistent: the type keeps pointing at it as tp_name. PyType_FromMetaclass() derives + // __module__ from the part before the last dot and warns if there is none. + const auto *full_name + = c_str((module_ ? str(module_).cast() : PYBIND11_DUMMY_MODULE_NAME) + "." + + rec.name); + + auto bases = tuple(rec.bases); + object base; // a single base or a tuple of bases + if (bases.empty()) { + base = reinterpret_borrow(internals.instance_base); + } else if (bases.size() == 1) { + base = bases[0]; + } else { + base = std::move(bases); + } + + std::vector slots; + /* Don't inherit base __init__ */ + slots.push_back({Py_tp_init, reinterpret_cast(pybind11_object_init)}); + if (rec.doc && options::show_user_defined_docstrings()) { + slots.push_back({Py_tp_doc, const_cast(rec.doc)}); + } + unsigned int flags = Py_TPFLAGS_DEFAULT; + if (!rec.is_final) { + flags |= Py_TPFLAGS_BASETYPE; + } + if (rec.dynamic_attr) { + flags |= Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MANAGED_DICT; + slots.push_back({Py_tp_traverse, reinterpret_cast(pybind11_traverse)}); + slots.push_back({Py_tp_clear, reinterpret_cast(pybind11_clear)}); + slots.push_back({Py_tp_getset, reinterpret_cast(dynamic_attr_getset())}); + } + if (rec.buffer_protocol) { + slots.push_back({Py_bf_getbuffer, reinterpret_cast(pybind11_getbuffer)}); + slots.push_back({Py_bf_releasebuffer, reinterpret_cast(pybind11_releasebuffer)}); + } + slots.push_back({0, nullptr}); + + PyType_Spec spec = {full_name, + 0, // inherit the instance layout from the base + 0, + flags, + slots.data()}; + PyObject *type = PyType_FromMetaclass(metaclass, nullptr, &spec, base.ptr()); + if (!type) { + pybind11_fail(std::string(rec.name) + ": PyType_FromMetaclass failed: " + error_string()); + } + assert(!rec.dynamic_attr + || PyType_HasFeature(reinterpret_cast(type), Py_TPFLAGS_HAVE_GC)); + + /* Register type with the parent scope */ + if (rec.scope) { + setattr(rec.scope, rec.name, type); + } else { + Py_INCREF(type); // Keep it alive forever (reference leak) + } + + if (module_) { // Needed by pydoc + setattr(type, "__module__", module_); + } + if (rec.scope && !PyModule_Check(rec.scope.ptr()) && hasattr(rec.scope, "__qualname__")) { + setattr(type, + "__qualname__", + reinterpret_steal(PyUnicode_FromFormat( + "%U.%s", rec.scope.attr("__qualname__").ptr(), rec.name))); + } + + return type; +} + +#else + PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { + return make_new_python_type_legacy(rec); +} + +#endif // PYBIND11_TYPE_CREATION_VIA_SPEC + +PYBIND11_INLINE PyObject *make_new_python_type_legacy(const type_record &rec) { auto name = reinterpret_steal(PYBIND11_FROM_STRING(rec.name)); auto qualname = name; diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index acce47ad55..bd8ef6ba5d 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -170,6 +170,11 @@ void enable_buffer_protocol(PyHeapTypeObject *heap_type); Return value: New reference. */ PyObject *make_new_python_type(const type_record &rec); +/// The hand-filled PyHeapTypeObject path. With PYBIND11_TYPE_CREATION_VIA_SPEC it is still used +/// for `py::custom_type_setup` and for metaclasses with a custom `tp_new` (which +/// PyType_FromMetaclass rejects). +PyObject *make_new_python_type_legacy(const type_record &rec); + PYBIND11_WARNING_POP PYBIND11_NAMESPACE_END(detail) From 408c543d48e0b682fb648355571cafdc1e9025de Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:33:47 -0400 Subject: [PATCH 17/45] refactor: wrap instance tp_alloc/tp_free and the __dict__ slot pointer type_alloc()/type_free() use PyType_GetSlot() when direct slot access is unavailable; instance_dict_ptr() is the single place that reads the instance dictionary pointer. See https://github.com/pybind/pybind11/discussions/6104 (C7). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 28 +++++++++++++++++++++++----- include/pybind11/detail/class.h | 7 +++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index a63c95097f..091e543c93 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -429,6 +429,24 @@ PYBIND11_INLINE bool deregister_instance(instance *self, void *valptr, const typ return ret; } +PYBIND11_INLINE PyObject *type_alloc(PyTypeObject *type) { +#if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) + return type->tp_alloc(type, 0); +#else + return reinterpret_cast(PyType_GetSlot(type, Py_tp_alloc))(type, 0); +#endif +} + +PYBIND11_INLINE void type_free(PyTypeObject *type, PyObject *self) { +#if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) + type->tp_free(self); +#else + reinterpret_cast(PyType_GetSlot(type, Py_tp_free))(self); +#endif +} + +PYBIND11_INLINE PyObject **instance_dict_ptr(PyObject *self) { return _PyObject_GetDictPtr(self); } + PYBIND11_INLINE PyObject *make_new_instance(PyTypeObject *type) { #if defined(PYPY_VERSION) // PyPy gets tp_basicsize wrong (issue 2482) under multiple inheritance when the first @@ -438,7 +456,7 @@ PYBIND11_INLINE PyObject *make_new_instance(PyTypeObject *type) { type->tp_basicsize = instance_size; } #endif - PyObject *self = type->tp_alloc(type, 0); + PyObject *self = type_alloc(type); auto *inst = reinterpret_cast(self); // Allocate the value/holder internals: inst->allocate_layout(); @@ -520,7 +538,7 @@ PYBIND11_INLINE void clear_instance(PyObject *self) { PyObject_ClearWeakRefs(self); } - PyObject **dict_ptr = _PyObject_GetDictPtr(self); + PyObject **dict_ptr = instance_dict_ptr(self); if (dict_ptr) { Py_CLEAR(*dict_ptr); } @@ -553,7 +571,7 @@ extern "C" PYBIND11_INLINE void pybind11_object_dealloc(PyObject *self) { clear_instance(self); - type->tp_free(self); + type_free(type, self); // This was not needed before Python 3.8 (Python issue 35810) // https://github.com/pybind/pybind11/issues/1946 @@ -641,7 +659,7 @@ extern "C" PYBIND11_INLINE int pybind11_traverse(PyObject *self, visitproc visit return ret; } #else - PyObject *&dict = *_PyObject_GetDictPtr(self); + PyObject *&dict = *instance_dict_ptr(self); Py_VISIT(dict); #endif // https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_traverse @@ -653,7 +671,7 @@ extern "C" PYBIND11_INLINE int pybind11_clear(PyObject *self) { #if PY_VERSION_HEX >= 0x030D0000 PyObject_ClearManagedDict(self); #else - PyObject *&dict = *_PyObject_GetDictPtr(self); + PyObject *&dict = *instance_dict_ptr(self); Py_CLEAR(dict); #endif return 0; diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index bd8ef6ba5d..7cee3c8d54 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -112,6 +112,13 @@ void register_instance(instance *self, void *valptr, const type_info *tinfo); bool deregister_instance(instance *self, void *valptr, const type_info *tinfo); +/// `type->tp_alloc(type, 0)` / `type->tp_free(self)`, also without direct slot access. +PyObject *type_alloc(PyTypeObject *type); +void type_free(PyTypeObject *type, PyObject *self); + +/// Pointer to the `__dict__` slot of an instance, or nullptr if its type has none. +PyObject **instance_dict_ptr(PyObject *self); + /// Instance creation function for all pybind11 types. It allocates the internal instance layout /// for holding C++ objects and holders. Allocation is done lazily (the first time the instance is /// cast to a reference or pointer), and initialization is done by an `__init__` function. From fd57174eedd0f234938e7b01303b9e6af9ef4cdb Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 17:15:33 -0400 Subject: [PATCH 18/45] feat: support the Python stable ABI (Py_LIMITED_API >= 0x030C0000) Replace the Py_LIMITED_API #error with real gates and the missing pieces: - Require CPython 3.12+, reject PyPy/GraalPy and free-threaded builds, force PYBIND11_SIMPLE_GIL_MANAGEMENT and PYBIND11_TYPE_CREATION_VIA_SPEC, disable subinterpreter support; embed.h and chrono.h #error, and py::custom_type_setup is unavailable. - Heap-allocated Py_tss_t keys (the struct is opaque). - dynamic_attr types and pybind11_static_property append the __dict__ slot to the base layout (no Py_TPFLAGS_MANAGED_DICT); the offset is cached in type_info because _PyObject_GetDictPtr is unavailable. - A pybind11-provided instancemethod type (shared through the internals) replaces PyInstanceMethod_Type; bound methods go through types.MethodType. - Attribute-based fallbacks for tp_name, __bases__/__mro__, type lookup, frame inspection, docstrings on PyCFunctions, dict.setdefault, length hints, generators, complex numbers, and eval/exec (Py_CompileString). - The import-time version check accepts any interpreter from the Py_LIMITED_API version on. Modules built this way run unchanged on every CPython from the targeted version on. See docs/advanced/stable_abi.rst (next commits) for the feature matrix. See https://github.com/pybind/pybind11/discussions/6104 (D1). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/attr.h | 8 + include/pybind11/cast.h | 2 +- include/pybind11/chrono.h | 5 + include/pybind11/complex.h | 27 +- include/pybind11/detail/class-inl.h | 316 ++++++++++++++++-- include/pybind11/detail/class.h | 26 ++ include/pybind11/detail/common.h | 105 ++++-- include/pybind11/detail/init.h | 15 +- include/pybind11/detail/internals.h | 71 ++-- .../pybind11/detail/type_caster_base-inl.h | 15 +- include/pybind11/detail/type_caster_base.h | 2 +- include/pybind11/embed.h | 3 + include/pybind11/eval.h | 51 ++- include/pybind11/gil_simple.h | 2 + include/pybind11/pybind11-inl.h | 52 ++- include/pybind11/pybind11.h | 12 +- include/pybind11/pytypes.h | 157 ++++++++- include/pybind11/stl.h | 9 +- tests/env.py | 7 + tests/pybind11_tests.cpp | 6 + tests/test_call_policies.cpp | 3 + tests/test_call_policies.py | 3 +- tests/test_kwargs_and_defaults.cpp | 4 +- tests/test_methods_and_attributes.py | 3 + 24 files changed, 785 insertions(+), 119 deletions(-) diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index ebac419663..048d165987 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -97,6 +97,7 @@ struct metaclass { explicit metaclass(handle value) : value(value) {} }; +#if !defined(Py_LIMITED_API) /// Specifies a custom callback with signature `void (PyHeapTypeObject*)` that /// may be used to customize the Python type. /// @@ -106,6 +107,8 @@ struct metaclass { /// work with later versions of pybind11. You may wish to consult the /// implementation of `make_new_python_type` in `detail/classes.h` to understand /// the context in which the callback will be run. +/// +/// Not available under the stable ABI (`Py_LIMITED_API`): PyHeapTypeObject is opaque there. struct custom_type_setup { using callback = std::function; @@ -113,6 +116,7 @@ struct custom_type_setup { callback value; }; +#endif /// Annotation that marks a class as local to the module: struct module_local { @@ -327,7 +331,9 @@ struct type_record { handle metaclass; /// Custom type setup. +#if !defined(Py_LIMITED_API) custom_type_setup::callback custom_type_setup_callback; +#endif /// Multiple inheritance marker bool multiple_inheritance : 1; @@ -567,12 +573,14 @@ struct process_attribute : process_attribute_default static void init(const dynamic_attr &, type_record *r) { r->dynamic_attr = true; } }; +#if !defined(Py_LIMITED_API) template <> struct process_attribute { static void init(const custom_type_setup &value, type_record *r) { r->custom_type_setup_callback = value.value; } }; +#endif template <> struct process_attribute : process_attribute_default { diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 79f8df7301..a4cdcb1390 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -2413,7 +2413,7 @@ unpacking_collector collect_arguments(Args &&...args) { template template object object_api::operator()(Args &&...args) const { -#ifndef NDEBUG +#if !defined(NDEBUG) && !defined(Py_LIMITED_API) // PyGILState_Check: not in the stable ABI if (!PyGILState_Check()) { pybind11_fail("pybind11::object_api<>::operator() PyGILState_Check() failure."); } diff --git a/include/pybind11/chrono.h b/include/pybind11/chrono.h index 668e458e99..29188511ef 100644 --- a/include/pybind11/chrono.h +++ b/include/pybind11/chrono.h @@ -12,6 +12,11 @@ #include "pybind11.h" +#if defined(Py_LIMITED_API) +# error \ + "pybind11/chrono.h is not available under the stable ABI (Py_LIMITED_API): the datetime C API accesses struct fields." +#endif + #include #include #include diff --git a/include/pybind11/complex.h b/include/pybind11/complex.h index 9d0812f09f..574cf8d5ef 100644 --- a/include/pybind11/complex.h +++ b/include/pybind11/complex.h @@ -57,10 +57,10 @@ class type_caster> { return false; } handle src_or_index = src; + object index; // PyPy: 7.3.7's 3.8 does not implement PyLong_*'s __index__ calls. // The same logic is used in numeric_caster for ints and floats #if defined(PYPY_VERSION) - object index; if (PYBIND11_INDEX_CHECK(src.ptr())) { index = reinterpret_steal(PyNumber_Index(src.ptr())); if (!index) { @@ -72,12 +72,37 @@ class type_caster> { } } #endif +#if !defined(Py_LIMITED_API) Py_complex result = PyComplex_AsCComplex(src_or_index.ptr()); if (result.real == -1.0 && PyErr_Occurred()) { PyErr_Clear(); return false; } value = std::complex((T) result.real, (T) result.imag); +#else + // Py_complex is not part of the stable ABI. Before 3.13, PyComplex_RealAsDouble() does + // not call __complex__ itself. + if (!PyComplex_Check(src_or_index.ptr()) && hasattr(src_or_index, "__complex__")) { + index = reinterpret_steal( + PyObject_CallMethod(src_or_index.ptr(), "__complex__", nullptr)); + if (!index) { + PyErr_Clear(); + return false; + } + src_or_index = index; + } + double real = PyComplex_RealAsDouble(src_or_index.ptr()); + if (real == -1.0 && PyErr_Occurred()) { + PyErr_Clear(); + return false; + } + double imag = PyComplex_ImagAsDouble(src_or_index.ptr()); + if (imag == -1.0 && PyErr_Occurred()) { + PyErr_Clear(); + return false; + } + value = std::complex((T) real, (T) imag); +#endif return true; } diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 091e543c93..445b986db1 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -16,6 +16,8 @@ #include "class.h" +#include + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) @@ -32,7 +34,7 @@ PYBIND11_INLINE std::string get_fully_qualified_tp_name(PyTypeObject *type) { } PYBIND11_INLINE PyTypeObject *type_incref(PyTypeObject *type) { - Py_INCREF(type); + Py_INCREF(reinterpret_cast(type)); return type; } @@ -78,9 +80,31 @@ pybind11_static_set(PyObject *self, PyObject *obj, PyObject *value) { # if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +# if defined(Py_LIMITED_API) +// Without Py_TPFLAGS_MANAGED_DICT the `__dict__` slot follows the property object. These two +// functions find it through the type's `__dictoffset__`. +PYBIND11_INLINE PyObject **static_property_dict_ptr(PyObject *self) { + static const Py_ssize_t offset = handle(reinterpret_cast(Py_TYPE(self))) + .attr("__dictoffset__") + .cast(); + return reinterpret_cast(reinterpret_cast(self) + offset); +} +extern "C" PYBIND11_INLINE int +pybind11_static_property_traverse(PyObject *self, visitproc visit, void *arg) { + Py_VISIT(*static_property_dict_ptr(self)); + Py_VISIT(Py_TYPE(self)); + return 0; +} +extern "C" PYBIND11_INLINE int pybind11_static_property_clear(PyObject *self) { + Py_CLEAR(*static_property_dict_ptr(self)); + return 0; +} +# endif + PYBIND11_INLINE PyTypeObject *make_static_property_type() { // Since Python-3.12 property-derived types are required to have dynamic attributes (to set // `__doc__`), hence the GC and dict slots. +# if !defined(Py_LIMITED_API) static PyType_Slot slots[] = {{Py_tp_descr_get, reinterpret_cast(pybind11_static_get)}, {Py_tp_descr_set, reinterpret_cast(pybind11_static_set)}, {Py_tp_traverse, reinterpret_cast(pybind11_traverse)}, @@ -93,6 +117,26 @@ PYBIND11_INLINE PyTypeObject *make_static_property_type() { 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MANAGED_DICT, slots}; +# else + // Append the `__dict__` slot to the property layout. (PEP 697 relative offsets are not + // applied to `__dictoffset__` on 3.12, so the offset is absolute.) + const auto dictoffset = type_generic_getattr(&PyProperty_Type, "__basicsize__").cast(); + PyMemberDef members[] = {{"__dictoffset__", Py_T_PYSSIZET, dictoffset, Py_READONLY, nullptr}, + {nullptr, 0, 0, 0, nullptr}}; + PyType_Slot slots[] + = {{Py_tp_descr_get, reinterpret_cast(pybind11_static_get)}, + {Py_tp_descr_set, reinterpret_cast(pybind11_static_set)}, + {Py_tp_traverse, reinterpret_cast(pybind11_static_property_traverse)}, + {Py_tp_clear, reinterpret_cast(pybind11_static_property_clear)}, + {Py_tp_getset, reinterpret_cast(dynamic_attr_getset())}, + {Py_tp_members, reinterpret_cast(members)}, + {0, nullptr}}; + PyType_Spec spec = {PYBIND11_DUMMY_MODULE_NAME ".pybind11_static_property", + dictoffset + static_cast(sizeof(PyObject *)), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + slots}; +# endif PyObject *type = PyType_FromMetaclass( nullptr, nullptr, &spec, reinterpret_cast(&PyProperty_Type)); if (!type) { @@ -203,7 +247,7 @@ pybind11_meta_setattro(PyObject *obj, PyObject *name, PyObject *value) { extern "C" PYBIND11_INLINE PyObject *pybind11_meta_getattro(PyObject *obj, PyObject *name) { object descr = type_lookup((PyTypeObject *) obj, name); - if (descr && PyInstanceMethod_Check(descr.ptr())) { + if (descr && PYBIND11_INSTANCE_METHOD_CHECK(descr.ptr())) { return descr.release().ptr(); } return type_type_getattro()(obj, name); @@ -429,6 +473,161 @@ PYBIND11_INLINE bool deregister_instance(instance *self, void *valptr, const typ return ret; } +#if defined(Py_LIMITED_API) + +// Stand-in for PyInstanceMethod_Type, which the stable ABI does not export. Like CPython's +// instancemethod: `__get__` binds through types.MethodType, calls and unknown attributes go to +// the wrapped function. The type is shared through the internals. +struct instancemethod_object { + PyObject_HEAD + PyObject *func; +}; + +PYBIND11_INLINE PyTypeObject *get_bound_method_type() { + static PyTypeObject *const type = [] { + auto types = reinterpret_steal(PyImport_ImportModule("types")); + if (!types) { + throw error_already_set(); + } + // Static builtin type: keep the reference forever. + return reinterpret_cast( + types.attr("MethodType").cast().release().ptr()); + }(); + return type; +} + +PYBIND11_INLINE bool is_bound_method(PyObject *obj) { + return Py_TYPE(obj) == get_bound_method_type(); +} + +PYBIND11_INLINE PyObject *bound_method_function(PyObject *obj) { + PyObject *func = PyObject_GetAttrString(obj, "__func__"); + if (func == nullptr) { + PyErr_Clear(); + return nullptr; + } + Py_DECREF(func); // the method object keeps the function alive + return func; +} + +extern "C" PYBIND11_INLINE PyObject * +instancemethod_descr_get(PyObject *self, PyObject *obj, PyObject * /*type*/) { + if (obj == nullptr || obj == Py_None) { + Py_INCREF(self); + return self; + } + return PyObject_CallFunctionObjArgs(reinterpret_cast(get_bound_method_type()), + reinterpret_cast(self)->func, + obj, + nullptr); +} + +extern "C" PYBIND11_INLINE PyObject * +instancemethod_call(PyObject *self, PyObject *args, PyObject *kwargs) { + return PyObject_Call(reinterpret_cast(self)->func, args, kwargs); +} + +extern "C" PYBIND11_INLINE PyObject *instancemethod_getattro(PyObject *self, PyObject *name) { + // Descriptors of the instancemethod type itself (`__func__`, `__class__`, ...) win; + // everything else (`__name__`, `__doc__`, `__module__`, ...) comes from the function. + object descr = type_lookup(Py_TYPE(self), name); + if (descr) { + auto get = reinterpret_cast( + PyType_GetSlot(Py_TYPE(descr.ptr()), Py_tp_descr_get)); + if (get != nullptr) { + return get(descr.ptr(), self, reinterpret_cast(Py_TYPE(self))); + } + } + return PyObject_GetAttr(reinterpret_cast(self)->func, name); +} + +extern "C" PYBIND11_INLINE PyObject *instancemethod_repr(PyObject *self) { + PyObject *func = reinterpret_cast(self)->func; + auto name = reinterpret_steal(PyObject_GetAttrString(func, "__name__")); + if (!name) { + PyErr_Clear(); + return PyUnicode_FromFormat("", self); + } + return PyUnicode_FromFormat("", name.ptr(), self); +} + +extern "C" PYBIND11_INLINE int +instancemethod_traverse(PyObject *self, visitproc visit, void *arg) { + Py_VISIT(reinterpret_cast(self)->func); + Py_VISIT(Py_TYPE(self)); + return 0; +} + +extern "C" PYBIND11_INLINE int instancemethod_clear(PyObject *self) { + Py_CLEAR(reinterpret_cast(self)->func); + return 0; +} + +extern "C" PYBIND11_INLINE void instancemethod_dealloc(PyObject *self) { + PyTypeObject *type = Py_TYPE(self); + PyObject_GC_UnTrack(self); + Py_CLEAR(reinterpret_cast(self)->func); + type_free(type, self); + Py_DECREF(reinterpret_cast(type)); +} + +PYBIND11_INLINE PyTypeObject *get_instancemethod_type() { + return with_internals([](internals &internals) { + if (internals.instancemethod_type == nullptr) { + static PyMemberDef members[] = {{"__func__", + Py_T_OBJECT_EX, + offsetof(instancemethod_object, func), + Py_READONLY, + nullptr}, + {nullptr, 0, 0, 0, nullptr}}; + static PyType_Slot slots[] + = {{Py_tp_descr_get, reinterpret_cast(instancemethod_descr_get)}, + {Py_tp_call, reinterpret_cast(instancemethod_call)}, + {Py_tp_getattro, reinterpret_cast(instancemethod_getattro)}, + {Py_tp_repr, reinterpret_cast(instancemethod_repr)}, + {Py_tp_traverse, reinterpret_cast(instancemethod_traverse)}, + {Py_tp_clear, reinterpret_cast(instancemethod_clear)}, + {Py_tp_dealloc, reinterpret_cast(instancemethod_dealloc)}, + {Py_tp_members, reinterpret_cast(members)}, + {0, nullptr}}; + static PyType_Spec spec + = {PYBIND11_DUMMY_MODULE_NAME ".instancemethod", + static_cast(sizeof(instancemethod_object)), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION, + slots}; + PyObject *type = PyType_FromSpec(&spec); + if (type == nullptr) { + pybind11_fail("get_instancemethod_type(): failure in PyType_FromSpec(): " + + error_string()); + } + internals.instancemethod_type = reinterpret_cast(type); + } + return internals.instancemethod_type; + }); +} + +PYBIND11_INLINE bool is_instancemethod(PyObject *obj) { + return Py_TYPE(obj) == get_instancemethod_type(); +} + +PYBIND11_INLINE PyObject *instancemethod_function(PyObject *obj) { + return reinterpret_cast(obj)->func; +} + +PYBIND11_INLINE PyObject *instancemethod_new(PyObject *func) { + PyTypeObject *type = get_instancemethod_type(); + PyObject *self = type_alloc(type); + if (self == nullptr) { + return nullptr; + } + Py_INCREF(func); + reinterpret_cast(self)->func = func; + return self; +} + +#endif // Py_LIMITED_API + PYBIND11_INLINE PyObject *type_alloc(PyTypeObject *type) { #if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) return type->tp_alloc(type, 0); @@ -445,7 +644,22 @@ PYBIND11_INLINE void type_free(PyTypeObject *type, PyObject *self) { #endif } -PYBIND11_INLINE PyObject **instance_dict_ptr(PyObject *self) { return _PyObject_GetDictPtr(self); } +PYBIND11_INLINE PyObject **instance_dict_ptr(PyObject *self) { +#if !defined(Py_LIMITED_API) + return _PyObject_GetDictPtr(self); +#else + // Only `py::dynamic_attr()` types have a `__dict__` slot pybind11 manages (recorded in the + // type_info of the pybind11 type that introduced it); Python subclasses that add a managed + // dict clear it themselves before calling the base tp_dealloc. + for (const auto *tinfo : all_type_info(Py_TYPE(self))) { + if (tinfo->dictoffset > 0) { + return reinterpret_cast(reinterpret_cast(self) + + tinfo->dictoffset); + } + } + return nullptr; +#endif +} PYBIND11_INLINE PyObject *make_new_instance(PyTypeObject *type) { #if defined(PYPY_VERSION) @@ -558,7 +772,7 @@ extern "C" PYBIND11_INLINE void pybind11_object_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); } -#if PY_VERSION_HEX >= 0x030D0000 +#if PY_VERSION_HEX >= 0x030D0000 && !defined(Py_LIMITED_API) // PyObject_ClearManagedDict() is available from Python 3.13+. It must be // called before tp_free() because on Python 3.14+ tp_free no longer // implicitly clears the managed dict, which would abandon the refcounts of @@ -575,7 +789,7 @@ extern "C" PYBIND11_INLINE void pybind11_object_dealloc(PyObject *self) { // This was not needed before Python 3.8 (Python issue 35810) // https://github.com/pybind/pybind11/issues/1946 - Py_DECREF(type); + Py_DECREF(reinterpret_cast(type)); } #if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) @@ -653,14 +867,15 @@ PYBIND11_INLINE PyObject *make_object_base_type(PyTypeObject *metaclass) { #endif // PYBIND11_TYPE_CREATION_VIA_SPEC extern "C" PYBIND11_INLINE int pybind11_traverse(PyObject *self, visitproc visit, void *arg) { -#if PY_VERSION_HEX >= 0x030D0000 +#if PY_VERSION_HEX >= 0x030D0000 && !defined(Py_LIMITED_API) int ret = PyObject_VisitManagedDict(self, visit, arg); if (ret) { return ret; } #else - PyObject *&dict = *instance_dict_ptr(self); - Py_VISIT(dict); + if (PyObject **dict = instance_dict_ptr(self)) { + Py_VISIT(*dict); + } #endif // https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_traverse Py_VISIT(Py_TYPE(self)); @@ -668,11 +883,12 @@ extern "C" PYBIND11_INLINE int pybind11_traverse(PyObject *self, visitproc visit } extern "C" PYBIND11_INLINE int pybind11_clear(PyObject *self) { -#if PY_VERSION_HEX >= 0x030D0000 +#if PY_VERSION_HEX >= 0x030D0000 && !defined(Py_LIMITED_API) PyObject_ClearManagedDict(self); #else - PyObject *&dict = *instance_dict_ptr(self); - Py_CLEAR(dict); + if (PyObject **dict = instance_dict_ptr(self)) { + Py_CLEAR(*dict); + } #endif return 0; } @@ -684,19 +900,21 @@ PYBIND11_INLINE PyGetSetDef *dynamic_attr_getset() { return getset; } +#if !defined(Py_LIMITED_API) PYBIND11_INLINE void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; -#ifdef PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET +# ifdef PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET type->tp_dictoffset = type->tp_basicsize; // place dict at the end type->tp_basicsize += (ssize_t) sizeof(PyObject *); // and allocate enough space for it -#else +# else type->tp_flags |= Py_TPFLAGS_MANAGED_DICT; -#endif +# endif type->tp_traverse = pybind11_traverse; type->tp_clear = pybind11_clear; type->tp_getset = dynamic_attr_getset(); } +#endif // !Py_LIMITED_API extern "C" PYBIND11_INLINE int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int flags) { // Look for a `get_buffer` implementation in this type's info or any bases (following MRO). @@ -802,12 +1020,14 @@ extern "C" PYBIND11_INLINE void pybind11_releasebuffer(PyObject *, Py_buffer *vi delete (buffer_info *) view->internal; } +#if !defined(Py_LIMITED_API) PYBIND11_INLINE void enable_buffer_protocol(PyHeapTypeObject *heap_type) { heap_type->ht_type.tp_as_buffer = &heap_type->as_buffer; heap_type->as_buffer.bf_getbuffer = pybind11_getbuffer; heap_type->as_buffer.bf_releasebuffer = pybind11_releasebuffer; } +#endif // !Py_LIMITED_API #if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) @@ -819,12 +1039,22 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { // PyType_FromMetaclass() cannot run a pre-PyType_Ready callback, rejects metaclasses with a // custom tp_new, and (correctly) refuses a metaclass that is less derived than the one of // the base; the legacy path accepts all three. +# if !defined(Py_LIMITED_API) if (rec.custom_type_setup_callback || PyType_GetSlot(metaclass, Py_tp_new) != PyType_GetSlot(&PyType_Type, Py_tp_new) || (metaclass != internals.default_metaclass && !PyType_IsSubtype(metaclass, internals.default_metaclass))) { return make_new_python_type_legacy(rec); } +# else + // Without the legacy path, a less derived py::metaclass() (e.g. `type`) is resolved to the + // most derived one by PyType_FromMetaclass(), i.e. the class gets pybind11_type. + if (PyType_GetSlot(metaclass, Py_tp_new) != PyType_GetSlot(&PyType_Type, Py_tp_new)) { + pybind11_fail(std::string(rec.name) + + ": py::metaclass() must not define __new__ under the stable ABI " + "(Py_LIMITED_API)"); + } +# endif object module_ = get_module_name_if_available(rec.scope); // Persistent: the type keeps pointing at it as tp_name. PyType_FromMetaclass() derives @@ -834,14 +1064,10 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { + rec.name); auto bases = tuple(rec.bases); - object base; // a single base or a tuple of bases if (bases.empty()) { - base = reinterpret_borrow(internals.instance_base); - } else if (bases.size() == 1) { - base = bases[0]; - } else { - base = std::move(bases); + bases = make_tuple(handle(internals.instance_base)); } + object base = bases.size() == 1 ? bases[0].cast() : static_cast(bases); std::vector slots; /* Don't inherit base __init__ */ @@ -853,8 +1079,34 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { if (!rec.is_final) { flags |= Py_TPFLAGS_BASETYPE; } + int basicsize = 0; // inherit the instance layout from the base +# if defined(Py_LIMITED_API) + // No Py_TPFLAGS_MANAGED_DICT in the stable ABI: the `__dict__` slot is appended to the base + // layout unless a base has one already. (PEP 697 relative offsets are not applied to + // `__dictoffset__` on 3.12, so the offset is absolute.) + PyMemberDef dict_member[] = {{"__dictoffset__", Py_T_PYSSIZET, 0, Py_READONLY, nullptr}, + {nullptr, 0, 0, 0, nullptr}}; +# endif if (rec.dynamic_attr) { - flags |= Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MANAGED_DICT; + flags |= Py_TPFLAGS_HAVE_GC; +# if !defined(Py_LIMITED_API) + flags |= Py_TPFLAGS_MANAGED_DICT; +# else + bool base_has_dict = false; + int base_basicsize = 0; + for (handle b : bases) { + auto *b_type = reinterpret_cast(b.ptr()); + base_has_dict + |= type_generic_getattr(b_type, "__dictoffset__").cast() != 0; + base_basicsize = std::max(base_basicsize, + type_generic_getattr(b_type, "__basicsize__").cast()); + } + if (!base_has_dict) { + dict_member[0].offset = base_basicsize; + basicsize = base_basicsize + static_cast(sizeof(PyObject *)); + slots.push_back({Py_tp_members, reinterpret_cast(dict_member)}); + } +# endif slots.push_back({Py_tp_traverse, reinterpret_cast(pybind11_traverse)}); slots.push_back({Py_tp_clear, reinterpret_cast(pybind11_clear)}); slots.push_back({Py_tp_getset, reinterpret_cast(dynamic_attr_getset())}); @@ -865,11 +1117,7 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { } slots.push_back({0, nullptr}); - PyType_Spec spec = {full_name, - 0, // inherit the instance layout from the base - 0, - flags, - slots.data()}; + PyType_Spec spec = {full_name, basicsize, 0, flags, slots.data()}; PyObject *type = PyType_FromMetaclass(metaclass, nullptr, &spec, base.ptr()); if (!type) { pybind11_fail(std::string(rec.name) + ": PyType_FromMetaclass failed: " + error_string()); @@ -905,6 +1153,7 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { #endif // PYBIND11_TYPE_CREATION_VIA_SPEC +#if !defined(Py_LIMITED_API) PYBIND11_INLINE PyObject *make_new_python_type_legacy(const type_record &rec) { auto name = reinterpret_steal(PYBIND11_FROM_STRING(rec.name)); @@ -916,20 +1165,20 @@ PYBIND11_INLINE PyObject *make_new_python_type_legacy(const type_record &rec) { object module_ = get_module_name_if_available(rec.scope); const auto *full_name = c_str( -#if !defined(PYPY_VERSION) +# if !defined(PYPY_VERSION) module_ ? str(module_).cast() + "." + rec.name : -#endif +# endif rec.name); char *tp_doc = nullptr; if (rec.doc && options::show_user_defined_docstrings()) { /* Allocate memory for docstring (Python will free this later on) */ size_t size = std::strlen(rec.doc) + 1; -#if PY_VERSION_HEX >= 0x030D0000 +# if PY_VERSION_HEX >= 0x030D0000 tp_doc = static_cast(PyMem_MALLOC(size)); -#else +# else tp_doc = (char *) PyObject_MALLOC(size); -#endif +# endif std::memcpy((void *) tp_doc, rec.doc, size); } @@ -950,9 +1199,9 @@ PYBIND11_INLINE PyObject *make_new_python_type_legacy(const type_record &rec) { } heap_type->ht_name = name.release().ptr(); -#ifdef PYBIND11_BUILTIN_QUALNAME +# ifdef PYBIND11_BUILTIN_QUALNAME heap_type->ht_qualname = qualname.inc_ref().ptr(); -#endif +# endif auto *type = &heap_type->ht_type; type->tp_name = full_name; @@ -1011,6 +1260,7 @@ PYBIND11_INLINE PyObject *make_new_python_type_legacy(const type_record &rec) { return reinterpret_cast(type); } +#endif // !Py_LIMITED_API PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 7cee3c8d54..416849d8fc 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -47,6 +47,12 @@ descrsetfunc property_type_descr_set(); #if !defined(PYPY_VERSION) +# if defined(Py_LIMITED_API) +PyObject **static_property_dict_ptr(PyObject *self); +extern "C" int pybind11_static_property_traverse(PyObject *self, visitproc visit, void *arg); +extern "C" int pybind11_static_property_clear(PyObject *self); +# endif + /// `pybind11_static_property.__get__()`: Always pass the class instead of the instance. extern "C" PyObject *pybind11_static_get(PyObject *self, PyObject * /*ob*/, PyObject *cls); @@ -116,6 +122,20 @@ bool deregister_instance(instance *self, void *valptr, const type_info *tinfo); PyObject *type_alloc(PyTypeObject *type); void type_free(PyTypeObject *type, PyObject *self); +#if defined(Py_LIMITED_API) +/// Stand-ins for the instancemethod and method C APIs (see detail/class-inl.h). The +/// `is_*`/`*_function` functions are also declared in pytypes.h. +PyTypeObject *get_bound_method_type(); +PyTypeObject *get_instancemethod_type(); +extern "C" PyObject *instancemethod_descr_get(PyObject *self, PyObject *obj, PyObject *type); +extern "C" PyObject *instancemethod_call(PyObject *self, PyObject *args, PyObject *kwargs); +extern "C" PyObject *instancemethod_getattro(PyObject *self, PyObject *name); +extern "C" PyObject *instancemethod_repr(PyObject *self); +extern "C" int instancemethod_traverse(PyObject *self, visitproc visit, void *arg); +extern "C" int instancemethod_clear(PyObject *self); +extern "C" void instancemethod_dealloc(PyObject *self); +#endif + /// Pointer to the `__dict__` slot of an instance, or nullptr if its type has none. PyObject **instance_dict_ptr(PyObject *self); @@ -161,8 +181,10 @@ extern "C" int pybind11_clear(PyObject *self); /// The `__dict__` descriptor for types with dynamic attributes. PyGetSetDef *dynamic_attr_getset(); +#if !defined(Py_LIMITED_API) /// Give instances of this type a `__dict__` and opt into garbage collection. void enable_dynamic_attributes(PyHeapTypeObject *heap_type); +#endif /// buffer_protocol: Fill in the view as specified by flags. extern "C" int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int flags); @@ -170,17 +192,21 @@ extern "C" int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int flags); /// buffer_protocol: Release the resources of the buffer. extern "C" void pybind11_releasebuffer(PyObject *, Py_buffer *view); +#if !defined(Py_LIMITED_API) /// Give this type a buffer interface. void enable_buffer_protocol(PyHeapTypeObject *heap_type); +#endif /** Create a brand new Python type according to the `type_record` specification. Return value: New reference. */ PyObject *make_new_python_type(const type_record &rec); +#if !defined(Py_LIMITED_API) /// The hand-filled PyHeapTypeObject path. With PYBIND11_TYPE_CREATION_VIA_SPEC it is still used /// for `py::custom_type_setup` and for metaclasses with a custom `tp_new` (which /// PyType_FromMetaclass rejects). PyObject *make_new_python_type_legacy(const type_record &rec); +#endif PYBIND11_WARNING_POP diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 8ec9e0a875..91296a4e91 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -14,9 +14,34 @@ # error "PYTHON < 3.9 IS UNSUPPORTED. pybind11 v3.0 was the last to support Python 3.8." #endif +// Python stable ABI (PEP 384) support. Requires CPython 3.12+ (PyType_FromMetaclass) and a +// matching Py_LIMITED_API value. See docs/advanced/stable_abi.rst for the feature matrix. #if defined(Py_LIMITED_API) -# error \ - "pybind11 does not support Py_LIMITED_API (the Python stable ABI) yet. Track progress at https://github.com/pybind/pybind11/discussions/6104" +# if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) +# error "Py_LIMITED_API is only supported with CPython." +# endif +# if PY_VERSION_HEX < 0x030C0000 || Py_LIMITED_API + 0 < 0x030C0000 +# error \ + "pybind11 requires Py_LIMITED_API >= 0x030C0000 and CPython >= 3.12 for the stable ABI." +# endif +# if defined(Py_GIL_DISABLED) +# error "The free-threaded build has no stable ABI; do not define Py_LIMITED_API." +# endif +# if !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# define PYBIND11_SIMPLE_GIL_MANAGEMENT // gil.h reads thread-state internals +# endif +# if !defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +# define PYBIND11_TYPE_CREATION_VIA_SPEC // the only type-creation path without struct + // access +# endif +# if !defined(PYBIND11_HAS_SUBINTERPRETER_SUPPORT) +# define PYBIND11_HAS_SUBINTERPRETER_SUPPORT 0 // reads thread/interpreter state fields +# elif PYBIND11_HAS_SUBINTERPRETER_SUPPORT +# error "Subinterpreter support is not available under Py_LIMITED_API." +# endif +# if !defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) +# define PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET // no Py_TPFLAGS_MANAGED_DICT +# endif #endif // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html @@ -260,6 +285,7 @@ #endif #include +#include #include #include #include @@ -297,9 +323,10 @@ # define PYBIND11_HAS_SPAN 1 #endif -// See description of PR #4246: +// See description of PR #4246. PyGILState_Check() is not part of the stable ABI. #if !defined(PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF) && !defined(NDEBUG) \ - && !defined(PYPY_VERSION) && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) + && !defined(PYPY_VERSION) && !defined(Py_LIMITED_API) \ + && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) # define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF #endif @@ -361,9 +388,15 @@ // behavior. /// Compatibility macros for Python 2 / Python 3 versions TODO: remove -#define PYBIND11_INSTANCE_METHOD_NEW(ptr, class_) PyInstanceMethod_New(ptr) -#define PYBIND11_INSTANCE_METHOD_CHECK PyInstanceMethod_Check -#define PYBIND11_INSTANCE_METHOD_GET_FUNCTION PyInstanceMethod_GET_FUNCTION +#if !defined(Py_LIMITED_API) +# define PYBIND11_INSTANCE_METHOD_NEW(ptr, class_) PyInstanceMethod_New(ptr) +# define PYBIND11_INSTANCE_METHOD_CHECK PyInstanceMethod_Check +# define PYBIND11_INSTANCE_METHOD_GET_FUNCTION PyInstanceMethod_GET_FUNCTION +#else // PyInstanceMethod_Type is not part of the stable ABI; see detail/class-inl.h +# define PYBIND11_INSTANCE_METHOD_NEW(ptr, class_) ::pybind11::detail::instancemethod_new(ptr) +# define PYBIND11_INSTANCE_METHOD_CHECK ::pybind11::detail::is_instancemethod +# define PYBIND11_INSTANCE_METHOD_GET_FUNCTION ::pybind11::detail::instancemethod_function +#endif #define PYBIND11_BYTES_CHECK PyBytes_Check #define PYBIND11_BYTES_FROM_STRING PyBytes_FromString #define PYBIND11_BYTES_FROM_STRING_AND_SIZE PyBytes_FromStringAndSize @@ -420,22 +453,42 @@ } while (0) #endif -#define PYBIND11_CHECK_PYTHON_VERSION \ - { \ - const char *compiled_ver \ - = PYBIND11_TOSTRING(PY_MAJOR_VERSION) "." PYBIND11_TOSTRING(PY_MINOR_VERSION); \ - const char *runtime_ver = Py_GetVersion(); \ - size_t len = std::strlen(compiled_ver); \ - if (std::strncmp(runtime_ver, compiled_ver, len) != 0 \ - || (runtime_ver[len] >= '0' && runtime_ver[len] <= '9')) { \ - PyErr_Format(PyExc_ImportError, \ - "Python version mismatch: module was compiled for Python %s, " \ - "but the interpreter version is incompatible: %s.", \ - compiled_ver, \ - runtime_ver); \ - return nullptr; \ - } \ - } +#if !defined(Py_LIMITED_API) +# define PYBIND11_CHECK_PYTHON_VERSION \ + { \ + const char *compiled_ver \ + = PYBIND11_TOSTRING(PY_MAJOR_VERSION) "." PYBIND11_TOSTRING(PY_MINOR_VERSION); \ + const char *runtime_ver = Py_GetVersion(); \ + size_t len = std::strlen(compiled_ver); \ + if (std::strncmp(runtime_ver, compiled_ver, len) != 0 \ + || (runtime_ver[len] >= '0' && runtime_ver[len] <= '9')) { \ + PyErr_Format(PyExc_ImportError, \ + "Python version mismatch: module was compiled for Python %s, " \ + "but the interpreter version is incompatible: %s.", \ + compiled_ver, \ + runtime_ver); \ + return nullptr; \ + } \ + } +#else +// A stable-ABI module runs on every CPython from the Py_LIMITED_API version on (abi3). +# define PYBIND11_CHECK_PYTHON_VERSION \ + { \ + const char *runtime_ver = Py_GetVersion(); \ + long major = std::strtol(runtime_ver, nullptr, 10); \ + const char *dot = std::strchr(runtime_ver, '.'); \ + long minor = dot ? std::strtol(dot + 1, nullptr, 10) : 0; \ + if (major != ((Py_LIMITED_API) >> 24) || minor < (((Py_LIMITED_API) >> 16) & 0xFF)) { \ + PyErr_Format(PyExc_ImportError, \ + "Python version mismatch: module was compiled for the Python %d.%d " \ + "stable ABI, but the interpreter version is incompatible: %s.", \ + (int) ((Py_LIMITED_API) >> 24), \ + (int) (((Py_LIMITED_API) >> 16) & 0xFF), \ + runtime_ver); \ + return nullptr; \ + } \ + } +#endif #define PYBIND11_CATCH_INIT_EXCEPTIONS \ catch (pybind11::error_already_set & e) { \ @@ -1445,8 +1498,10 @@ inline void silence_unused_warnings(Args &&...) {} # define PYBIND11_DETAILED_ERROR_MESSAGES #endif -// CPython 3.11+ provides Py_TPFLAGS_MANAGED_DICT, but PyPy3.11 does not, see PR #5508. -#if PY_VERSION_HEX < 0x030B0000 || defined(PYPY_VERSION) +// CPython 3.11+ provides Py_TPFLAGS_MANAGED_DICT, but PyPy3.11 does not, see PR #5508. (The +// stable ABI does not have it either; see the Py_LIMITED_API block at the top of this file.) +#if (PY_VERSION_HEX < 0x030B0000 || defined(PYPY_VERSION)) \ + && !defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) # define PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET #endif diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index cce3eb5607..af23890d57 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -312,7 +312,7 @@ struct constructor { cl.def( "__init__", [](value_and_holder &v_h, Args... args) { - if (Py_TYPE(v_h.inst) == v_h.type->type) { + if (Py_TYPE(reinterpret_cast(v_h.inst)) == v_h.type->type) { v_h.value_ptr() = construct_or_initialize>(std::forward(args)...); } else { @@ -399,8 +399,10 @@ struct factory { [func] #endif (value_and_holder &v_h, Args... args) { - construct( - v_h, func(std::forward(args)...), Py_TYPE(v_h.inst) != v_h.type->type); + construct(v_h, + func(std::forward(args)...), + Py_TYPE(reinterpret_cast(v_h.inst)) + != v_h.type->type); }, is_new_style_constructor(), extra...); @@ -447,7 +449,7 @@ struct factory { [class_func, alias_func] #endif (value_and_holder &v_h, CArgs... args) { - if (Py_TYPE(v_h.inst) == v_h.type->type) { + if (Py_TYPE(reinterpret_cast(v_h.inst)) == v_h.type->type) { // If the instance type equals the registered type we don't have inheritance, // so don't need the alias and can construct using the class function: construct(v_h, class_func(std::forward(args)...), false); @@ -535,8 +537,9 @@ struct pickle_factory { [func] #endif (value_and_holder &v_h, ArgState state) { - setstate( - v_h, func(std::forward(state)), Py_TYPE(v_h.inst) != v_h.type->type); + setstate(v_h, + func(std::forward(state)), + Py_TYPE(reinterpret_cast(v_h.inst)) != v_h.type->type); }, is_new_style_constructor(), extra...); diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 02906d0573..e01b6b2c92 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -121,29 +121,41 @@ using ExceptionTranslator = void (*)(std::exception_ptr); // The old Python Thread Local Storage (TLS) API is deprecated in Python 3.7 in favor of the new // Thread Specific Storage (TSS) API. -// Avoid unnecessary allocation of `Py_tss_t`, since we cannot use -// `Py_LIMITED_API` anyway. -#define PYBIND11_TLS_KEY_REF Py_tss_t & -#if defined(__clang__) -# define PYBIND11_TLS_KEY_INIT(var) \ - _Pragma("clang diagnostic push") /**/ \ - _Pragma("clang diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ - Py_tss_t var = Py_tss_NEEDS_INIT; \ - _Pragma("clang diagnostic pop") -#elif defined(__GNUC__) && !defined(__INTEL_COMPILER) -# define PYBIND11_TLS_KEY_INIT(var) \ - _Pragma("GCC diagnostic push") /**/ \ - _Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ - Py_tss_t var = Py_tss_NEEDS_INIT; \ - _Pragma("GCC diagnostic pop") +// `Py_tss_t` is opaque under `Py_LIMITED_API`, so the key is heap-allocated there; otherwise it +// is embedded to avoid the allocation. +#if defined(Py_LIMITED_API) +using tss_key_ptr = Py_tss_t *; // lets PYBIND11_TLS_KEY_INIT(mutable key_) parse +# define PYBIND11_TLS_KEY_REF Py_tss_t * +# define PYBIND11_TLS_KEY_INIT(var) tss_key_ptr var = nullptr; +# define PYBIND11_TLS_KEY_CREATE(var) \ + (((var) = PyThread_tss_alloc()) != nullptr && PyThread_tss_create(var) == 0) +# define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get(key) +# define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set((key), (value)) +# define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set((key), nullptr) +# define PYBIND11_TLS_FREE(key) PyThread_tss_free(key) #else -# define PYBIND11_TLS_KEY_INIT(var) Py_tss_t var = Py_tss_NEEDS_INIT; +# define PYBIND11_TLS_KEY_REF Py_tss_t & +# if defined(__clang__) +# define PYBIND11_TLS_KEY_INIT(var) \ + _Pragma("clang diagnostic push") /**/ \ + _Pragma("clang diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ + Py_tss_t var = Py_tss_NEEDS_INIT; \ + _Pragma("clang diagnostic pop") +# elif defined(__GNUC__) && !defined(__INTEL_COMPILER) +# define PYBIND11_TLS_KEY_INIT(var) \ + _Pragma("GCC diagnostic push") /**/ \ + _Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ + Py_tss_t var = Py_tss_NEEDS_INIT; \ + _Pragma("GCC diagnostic pop") +# else +# define PYBIND11_TLS_KEY_INIT(var) Py_tss_t var = Py_tss_NEEDS_INIT; +# endif +# define PYBIND11_TLS_KEY_CREATE(var) (PyThread_tss_create(&(var)) == 0) +# define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get(&(key)) +# define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set(&(key), (value)) +# define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set(&(key), nullptr) +# define PYBIND11_TLS_FREE(key) PyThread_tss_delete(&(key)) #endif -#define PYBIND11_TLS_KEY_CREATE(var) (PyThread_tss_create(&(var)) == 0) -#define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get(&(key)) -#define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set(&(key), (value)) -#define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set(&(key), nullptr) -#define PYBIND11_TLS_FREE(key) PyThread_tss_delete(&(key)) /// A smart-pointer-like wrapper around a thread-specific value. get/set of the pointer applies to /// the current thread only. @@ -217,7 +229,11 @@ void translate_exception(std::exception_ptr p); inline PyThreadState *get_thread_state_unchecked() { #if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) return PyThreadState_GET(); -#elif PY_VERSION_HEX < 0x030D0000 +#elif defined(Py_LIMITED_API) && Py_LIMITED_API < 0x030D0000 + // The stable ABI has no "current thread state or null" function before 3.13. All callers + // left under the stable ABI (the interpreter state dict lookup) hold the GIL. + return PyThreadState_Get(); +#elif PY_VERSION_HEX < 0x030D0000 && !defined(Py_LIMITED_API) return _PyThreadState_UncheckedGet(); #else return PyThreadState_GetUnchecked(); @@ -381,6 +397,11 @@ struct internals { PyTypeObject *static_property_type = nullptr; PyTypeObject *default_metaclass = nullptr; PyObject *instance_base = nullptr; +#if defined(Py_LIMITED_API) + // Replacement for PyInstanceMethod_Type, which is not part of the stable ABI. Shared so + // that every module recognizes the methods of every other module's classes. + PyTypeObject *instancemethod_type = nullptr; +#endif // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: thread_specific_storage tstate; #if PYBIND11_INTERNALS_VERSION <= 11 @@ -476,6 +497,12 @@ struct type_info { std::forward_list alias_chain; #endif +#if defined(Py_LIMITED_API) + /* Offset of the `__dict__` slot of instances for `py::dynamic_attr()` types (0 if none); + * `_PyObject_GetDictPtr` is not available under the stable ABI. */ + Py_ssize_t dictoffset = 0; +#endif + /* A simple type never occurs as a (direct or indirect) parent * of a class that makes use of multiple inheritance. * A type can be simple even if it has non-simple ancestors as long as it has no descendants. diff --git a/include/pybind11/detail/type_caster_base-inl.h b/include/pybind11/detail/type_caster_base-inl.h index cad39fb397..31532b65cb 100644 --- a/include/pybind11/detail/type_caster_base-inl.h +++ b/include/pybind11/detail/type_caster_base-inl.h @@ -286,7 +286,8 @@ find_registered_python_instance(void *src, const detail::type_info *tinfo) { return with_instance_map(src, [&](instance_map &instances) { auto it_instances = instances.equal_range(src); for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { - for (auto *instance_type : detail::all_type_info(Py_TYPE(it_i->second))) { + for (auto *instance_type : + detail::all_type_info(Py_TYPE(reinterpret_cast(it_i->second)))) { if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) { auto *wrapper = reinterpret_cast(it_i->second); if (try_incref(wrapper)) { @@ -303,7 +304,7 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE value_and_holder instance::get_value_and_holder(const type_info *find_type /*= nullptr default in common.h*/, bool throw_if_missing /*= true in common.h*/) { // Optimize common case: - if (!find_type || Py_TYPE(this) == find_type->type) { + if (!find_type || Py_TYPE(reinterpret_cast(this)) == find_type->type) { return value_and_holder(this, find_type, 0, 0); } @@ -318,10 +319,10 @@ instance::get_value_and_holder(const type_info *find_type /*= nullptr default in } #if defined(PYBIND11_DETAILED_ERROR_MESSAGES) - pybind11_fail("pybind11::detail::instance::get_value_and_holder: `" - + get_fully_qualified_tp_name(find_type->type) - + "' is not a pybind11 base of the given `" - + get_fully_qualified_tp_name(Py_TYPE(this)) + "' instance"); + pybind11_fail( + "pybind11::detail::instance::get_value_and_holder: `" + + get_fully_qualified_tp_name(find_type->type) + "' is not a pybind11 base of the given `" + + get_fully_qualified_tp_name(Py_TYPE(reinterpret_cast(this))) + "' instance"); #else pybind11_fail( "pybind11::detail::instance::get_value_and_holder: " @@ -331,7 +332,7 @@ instance::get_value_and_holder(const type_info *find_type /*= nullptr default in } PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void instance::allocate_layout() { - const auto &tinfo = all_type_info(Py_TYPE(this)); + const auto &tinfo = all_type_info(Py_TYPE(reinterpret_cast(this))); const size_t n_types = tinfo.size(); diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 7fdab68fcf..15b04604db 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -139,7 +139,7 @@ struct values_and_holders { public: explicit values_and_holders(instance *inst) - : inst{inst}, tinfo(all_type_info(Py_TYPE(inst))) {} + : inst{inst}, tinfo(all_type_info(Py_TYPE(reinterpret_cast(inst)))) {} explicit values_and_holders(PyObject *obj) : inst{nullptr}, tinfo(all_type_info(Py_TYPE(obj))) { diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index c333892ff5..9a8f4a7916 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -15,6 +15,9 @@ #if defined(PYPY_VERSION) # error Embedding the interpreter is not supported with PyPy #endif +#if defined(Py_LIMITED_API) +# error "pybind11/embed.h is not available under the stable ABI (Py_LIMITED_API)." +#endif #define PYBIND11_EMBEDDED_MODULE_IMPL(name) \ extern "C" PyObject *pybind11_init_impl_##name(); \ diff --git a/include/pybind11/eval.h b/include/pybind11/eval.h index ee24fd3584..ff0796e880 100644 --- a/include/pybind11/eval.h +++ b/include/pybind11/eval.h @@ -19,11 +19,12 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) inline void ensure_builtins_in_globals(object &global) { -#if defined(PYPY_VERSION) +#if defined(PYPY_VERSION) || defined(Py_LIMITED_API) // Running exec and eval adds `builtins` module under `__builtins__` key to // globals if not yet present. Python 3.8 made PyRun_String behave // similarly. Let's also do that for older versions, for consistency. This - // was missing from PyPy3.8 7.3.7. + // was missing from PyPy3.8 7.3.7, and PyEval_EvalCode (the stable-ABI path + // below) does not do it either. if (!global.contains("__builtins__")) global["__builtins__"] = module_::import(PYBIND11_BUILTINS_MODULE); #else @@ -31,6 +32,23 @@ inline void ensure_builtins_in_globals(object &global) { #endif } +#if defined(Py_LIMITED_API) +// PyRun_String() is not part of the stable ABI: compile and evaluate in two steps. +inline PyObject *run_string( + const char *source, const char *filename, int start, PyObject *global, PyObject *local) { + auto code = reinterpret_steal(Py_CompileString(source, filename, start)); + if (!code) { + return nullptr; + } + return PyEval_EvalCode(code.ptr(), global, local); +} +#else +inline PyObject *run_string( + const char *source, const char * /*filename*/, int start, PyObject *global, PyObject *local) { + return PyRun_String(source, start, global, local); +} +#endif + PYBIND11_NAMESPACE_END(detail) enum eval_mode { @@ -70,7 +88,8 @@ object eval(const str &expr, object global = globals(), object local = object()) pybind11_fail("invalid evaluation mode"); } - PyObject *result = PyRun_String(buffer.c_str(), start, global.ptr(), local.ptr()); + PyObject *result + = detail::run_string(buffer.c_str(), "", start, global.ptr(), local.ptr()); if (!result) { throw error_already_set(); } @@ -130,14 +149,31 @@ object eval_file(str fname, object global = globals(), object local = object()) pybind11_fail("invalid evaluation mode"); } - int closeFile = 1; std::string fname_str = (std::string) fname; +# if defined(Py_LIMITED_API) + // PyRun_FileEx() and Py_fopen() are not part of the stable ABI: read the file from Python. + object source; + try { + object io_open = module_::import("io").attr("open"); + object file = io_open(fname, "rb"); + source = file.attr("read")(); + file.attr("close")(); + } catch (error_already_set &) { + pybind11_fail("File \"" + fname_str + "\" could not be opened!"); + } + if (!global.contains("__file__")) { + global["__file__"] = std::move(fname); + } + PyObject *result = detail::run_string( + PyBytes_AsString(source.ptr()), fname_str.c_str(), start, global.ptr(), local.ptr()); +# else + int closeFile = 1; FILE *f = -# if PY_VERSION_HEX >= 0x030E0000 +# if PY_VERSION_HEX >= 0x030E0000 Py_fopen(fname.ptr(), "r"); -# else +# else _Py_fopen_obj(fname.ptr(), "r"); -# endif +# endif if (!f) { PyErr_Clear(); pybind11_fail("File \"" + fname_str + "\" could not be opened!"); @@ -149,6 +185,7 @@ object eval_file(str fname, object global = globals(), object local = object()) PyObject *result = PyRun_FileEx(f, fname_str.c_str(), start, global.ptr(), local.ptr(), closeFile); +# endif if (!result) { throw error_already_set(); diff --git a/include/pybind11/gil_simple.h b/include/pybind11/gil_simple.h index 9ff428ad84..61a0fe4269 100644 --- a/include/pybind11/gil_simple.h +++ b/include/pybind11/gil_simple.h @@ -26,7 +26,9 @@ class gil_scoped_release_simple { public: // PRECONDITION: The GIL must be held when this constructor is called. gil_scoped_release_simple() { +#if !defined(Py_LIMITED_API) // PyGILState_Check is not part of the stable ABI assert(PyGILState_Check()); +#endif state = PyEval_SaveThread(); } gil_scoped_release_simple(const gil_scoped_release_simple &) = delete; diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index ce8a286f6e..adc85795b7 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -464,7 +464,7 @@ PYBIND11_INLINE function get_type_override(const void *this_ptr, /* Don't call dispatch code if invoked from overridden function. Unfortunately this doesn't work on PyPy and GraalPy. */ -#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) +#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) && !defined(Py_LIMITED_API) PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get()); if (frame != nullptr) { PyCodeObject *f_code = PyFrame_GetCode(frame); @@ -497,6 +497,31 @@ PYBIND11_INLINE function get_type_override(const void *this_ptr, Py_DECREF(frame); } +#elif defined(Py_LIMITED_API) + // Same check through attribute access: the frame and code object layouts are not part of + // the stable ABI. + PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get()); + if (frame != nullptr) { + auto frame_obj = reinterpret_steal(reinterpret_cast(frame)); + auto f_code + = reinterpret_steal(reinterpret_cast(PyFrame_GetCode(frame))); + if (std::string(str(f_code.attr("co_name"))) == name + && f_code.attr("co_argcount").cast() > 0) { + object self_arg = f_code.attr("co_varnames")[int_(0)]; + auto locals + = reinterpret_steal(PyObject_GetAttrString(frame_obj.ptr(), "f_locals")); + auto self_caller + = locals + ? reinterpret_steal(PyObject_GetItem(locals.ptr(), self_arg.ptr())) + : object(); + if (!self_caller) { + PyErr_Clear(); + } else if (self_caller.is(self)) { + return function(); + } + } + } + #else /* PyPy currently doesn't provide a detailed cpyext emulation of frame objects, so we have to emulate this using Python. This @@ -735,13 +760,26 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u } } - auto *func = reinterpret_cast(m_ptr); // Install docstring if it's non-empty (when at least one option is enabled) auto *doc = signatures.empty() ? nullptr : PYBIND11_COMPAT_STRDUP(signatures.c_str()); +#if !defined(Py_LIMITED_API) + auto *func = reinterpret_cast(m_ptr); std::free(const_cast(PYBIND11_PYCFUNCTION_GET_DOC(func))); PYBIND11_PYCFUNCTION_SET_DOC(func, doc); +#else + // PyCFunctionObject is opaque, but the function reads its `__doc__` from the PyMethodDef, + // which pybind11 owns: the one record in the chain that allocated it has `def` set. + for (auto *it = chain_start; it != nullptr; it = it->next) { + if (it->def) { + std::free(const_cast(it->def->ml_doc)); + it->def->ml_doc = doc; + break; + } + } +#endif if (rec->is_method) { + PyObject *func = m_ptr; m_ptr = PYBIND11_INSTANCE_METHOD_NEW(m_ptr, rec->scope.ptr()); if (!m_ptr) { pybind11_fail( @@ -1293,6 +1331,9 @@ PYBIND11_INLINE void generic_type::initialize(const type_record &rec) { tinfo->simple_ancestors = true; tinfo->module_local = rec.module_local; tinfo->holder_enum_v = rec.holder_enum_v; +#if defined(Py_LIMITED_API) + tinfo->dictoffset = handle(m_ptr).attr("__dictoffset__").cast(); +#endif with_internals([&](internals &internals) { auto tindex = std::type_index(*rec.type); @@ -1384,9 +1425,10 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void enum_base::init(bool is_arithmetic, [](handle arg) -> std::string { std::string docstring; dict entries = arg.attr("__entries"); - if ((reinterpret_cast(arg.ptr()))->tp_doc) { - docstring - += std::string(reinterpret_cast(arg.ptr())->tp_doc); + // tp_doc: the class docstring; `__doc__` itself is this property. + if (const auto *tp_doc = static_cast(PyType_GetSlot( + reinterpret_cast(arg.ptr()), Py_tp_doc))) { + docstring += tp_doc; docstring += "\n\n"; } docstring += "Members:"; diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 0d48897a3e..9a1a4834d3 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -825,10 +825,10 @@ class generic_type : public object { void install_buffer_funcs(buffer_info *(*get_buffer)(PyObject *, void *), void *get_buffer_data) { - auto *type = reinterpret_cast(m_ptr); - auto *tinfo = detail::get_type_info(&type->ht_type); + auto *type = reinterpret_cast(m_ptr); + auto *tinfo = detail::get_type_info(type); - if (!type->ht_type.tp_as_buffer) { + if (PyType_GetSlot(type, Py_bf_getbuffer) == nullptr) { pybind11_fail("To be able to register buffer protocol support for the type '" + get_fully_qualified_tp_name(tinfo->type) + "' the associated class<>(..) invocation must " @@ -2054,8 +2054,10 @@ class enum_ : public class_ { def("__index__", [](Type value) { return (Scalar) value; }, pos_only()); attr("__setstate__") = cpp_function( [](detail::value_and_holder &v_h, Scalar arg) { - detail::initimpl::setstate( - v_h, static_cast(arg), Py_TYPE(v_h.inst) != v_h.type->type); + detail::initimpl::setstate(v_h, + static_cast(arg), + Py_TYPE(reinterpret_cast(v_h.inst)) + != v_h.type->type); }, detail::is_new_style_constructor(), pybind11::name("__setstate__"), diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index ab7c36aa64..8a440170e6 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -500,6 +500,7 @@ T reinterpret_steal(handle h) { PYBIND11_NAMESPACE_BEGIN(detail) +#if !defined(Py_LIMITED_API) /// The `tp_name` of a type: `module.Name` for static and pybind11 types, `Name` for Python /// heap types. inline std::string get_tp_name(PyTypeObject *type) { return type->tp_name; } @@ -508,6 +509,52 @@ inline std::string get_tp_name(PyTypeObject *type) { return type->tp_name; } inline bool tp_name_equals(PyTypeObject *type, const char *name) { return std::strcmp(type->tp_name, name) == 0; } +#else +bool type_is_managed_by_our_internals(PyTypeObject *type_obj); // detail/cpp_conduit.h + +inline std::string utf8_or_empty(const object &o) { + Py_ssize_t size = 0; + const char *data + = o && PyUnicode_Check(o.ptr()) ? PyUnicode_AsUTF8AndSize(o.ptr(), &size) : nullptr; + if (data == nullptr) { + PyErr_Clear(); + return std::string(); + } + return std::string(data, static_cast(size)); +} + +/// Reconstructs `tp_name`, which is not readable under the stable ABI: `__module__.__name__` +/// for static types outside builtins and for pybind11 types, the bare `__name__` otherwise +/// (builtins and Python heap types). +inline std::string get_tp_name(PyTypeObject *type) { + std::string name = utf8_or_empty(reinterpret_steal(PyType_GetName(type))); + if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE) && !type_is_managed_by_our_internals(type)) { + return name; + } + std::string module_ = utf8_or_empty( + reinterpret_steal(PyObject_GetAttrString((PyObject *) type, "__module__"))); + if (module_.empty() || module_ == PYBIND11_BUILTINS_MODULE) { + return name; + } + return module_ + '.' + name; +} + +inline bool tp_name_equals(PyTypeObject *type, const char *name) { + // Cheap pre-check on the unqualified name before reconstructing the full one. + const char *dot = std::strrchr(name, '.'); + auto type_name = reinterpret_steal(PyType_GetName(type)); + const char *type_name_utf8 + = type_name ? PyUnicode_AsUTF8AndSize(type_name.ptr(), nullptr) : nullptr; + if (type_name_utf8 == nullptr) { + PyErr_Clear(); + return false; + } + if (std::strcmp(type_name_utf8, dot ? dot + 1 : name) != 0) { + return false; + } + return get_tp_name(type) == name; +} +#endif // Unchecked tuple/list element access for hot paths. Function forms under the stable ABI. #if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) @@ -773,13 +820,32 @@ inline ssize_t hash(handle obj) { /// @} python_builtins PYBIND11_NAMESPACE_BEGIN(detail) +#if defined(Py_LIMITED_API) +// Replacements for the instancemethod and method APIs, which are not part of the stable ABI. +// Defined in detail/class-inl.h (the instancemethod type is created with the other internals +// types). +bool is_instancemethod(PyObject *obj); +PyObject *instancemethod_function(PyObject *obj); // borrowed reference +PyObject *instancemethod_new(PyObject *func); // new reference +bool is_bound_method(PyObject *obj); +PyObject *bound_method_function(PyObject *obj); // borrowed reference +#endif + inline handle get_function(handle value) { if (value) { +#if !defined(Py_LIMITED_API) if (PyInstanceMethod_Check(value.ptr())) { value = PyInstanceMethod_GET_FUNCTION(value.ptr()); } else if (PyMethod_Check(value.ptr())) { value = PyMethod_GET_FUNCTION(value.ptr()); } +#else + if (is_instancemethod(value.ptr())) { + value = instancemethod_function(value.ptr()); + } else if (is_bound_method(value.ptr())) { + value = bound_method_function(value.ptr()); + } +#endif } return value; } @@ -830,13 +896,20 @@ inline PyObject *dict_getitemstringref(PyObject *v, const char *key) { #endif } +// Returns a borrowed reference (the dict holds the value). inline PyObject *dict_setdefaultstring(PyObject *v, const char *key, PyObject *defaultobj) { PyObject *kv = PyUnicode_FromString(key); if (kv == nullptr) { throw error_already_set(); } +#if !defined(Py_LIMITED_API) PyObject *rv = PyDict_SetDefault(v, kv, defaultobj); +#else + // dict.setdefault() is one C-level call, so it is just as atomic as PyDict_SetDefault. + PyObject *rv = PyObject_CallMethod(v, "setdefault", "OO", kv, defaultobj); + Py_XDECREF(rv); +#endif Py_DECREF(kv); if (rv == nullptr) { throw error_already_set(); @@ -1123,6 +1196,7 @@ struct arrow_proxy { T *operator->() const { return &value; } }; +#if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) /// Lightweight iterator policy using just a simple pointer: see ``PySequence_Fast_ITEMS`` class sequence_fast_readonly { protected: @@ -1145,6 +1219,7 @@ class sequence_fast_readonly { private: PyObject **ptr; }; +#endif /// Full read and write access using the sequence protocol: see ``detail::sequence_accessor`` class sequence_slow_readwrite { @@ -1229,7 +1304,28 @@ inline bool PyUnicode_Check_Permissive(PyObject *o) { # define PYBIND11_STR_CHECK_FUN PyUnicode_Check #endif +#if !defined(Py_LIMITED_API) inline bool PyStaticMethod_Check(PyObject *o) { return Py_TYPE(o) == &PyStaticMethod_Type; } +inline PyObject *PyStaticMethod_New(PyObject *callable) { return ::PyStaticMethod_New(callable); } +#else +// PyStaticMethod_Type is not exported by the stable ABI; fetch it from builtins once. +inline PyTypeObject *get_staticmethod_type() { + static PyTypeObject *const type = [] { + PyObject *builtins = PyEval_GetBuiltins(); + PyObject *t = builtins ? PyDict_GetItemString(builtins, "staticmethod") : nullptr; + if (t == nullptr) { + pybind11_fail("pybind11::detail::get_staticmethod_type(): lookup failed"); + } + return reinterpret_cast(t); // immortal builtin + }(); + return type; +} +inline bool PyStaticMethod_Check(PyObject *o) { return Py_TYPE(o) == get_staticmethod_type(); } +inline PyObject *PyStaticMethod_New(PyObject *callable) { + return PyObject_CallFunctionObjArgs( + reinterpret_cast(get_staticmethod_type()), callable, nullptr); +} +#endif class kwargs_proxy : public handle { public: @@ -2184,7 +2280,10 @@ class function : public object { class staticmethod : public object { public: - PYBIND11_OBJECT_CVT(staticmethod, object, detail::PyStaticMethod_Check, PyStaticMethod_New) + PYBIND11_OBJECT_CVT(staticmethod, + object, + detail::PyStaticMethod_Check, + detail::PyStaticMethod_New) }; class buffer : public object { @@ -2340,7 +2439,20 @@ inline size_t len(handle h) { /// Get the length hint of a Python object. /// Returns 0 when this cannot be determined. inline size_t len_hint(handle h) { +#if !defined(Py_LIMITED_API) ssize_t result = PyObject_LengthHint(h.ptr(), 0); +#else + // PyObject_LengthHint is not part of the stable ABI: try len(), then __length_hint__(). + ssize_t result = PyObject_Length(h.ptr()); + if (result < 0) { + PyErr_Clear(); + result = -1; + if (auto hint = reinterpret_steal( + PyObject_CallMethod(h.ptr(), "__length_hint__", nullptr))) { + result = PyLong_AsSsize_t(hint.ptr()); + } + } +#endif if (result < 0) { // Sometimes a length can't be determined at all (eg generators) // In which case simply return 0 @@ -2501,6 +2613,7 @@ PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator>>=, PyNumber_InPlaceRshift) #undef PYBIND11_MATH_OPERATOR_BINARY_INPLACE // Meant to return a Python str, but this is not checked. +#if !defined(Py_LIMITED_API) /// `type.__bases__` as a tuple (empty if the type has no bases tuple yet). inline tuple get_bases(PyTypeObject *type) { if (type->tp_bases == nullptr) { @@ -2517,6 +2630,48 @@ inline tuple get_mro(PyTypeObject *type) { return reinterpret_borrow(type inline object type_lookup(PyTypeObject *type, handle name) { return reinterpret_borrow(_PyType_Lookup(type, name.ptr())); } +#else +// Generic attribute lookup on a type object: resolves the `type.__xxx__` descriptors without +// going through the metaclass's tp_getattro (pybind11's own metaclass calls back into here). +inline object type_generic_getattr(PyTypeObject *type, const char *name) { + auto name_obj = reinterpret_steal(PyUnicode_InternFromString(name)); + auto value = name_obj ? reinterpret_steal( + PyObject_GenericGetAttr((PyObject *) type, name_obj.ptr())) + : object(); + if (!value) { + PyErr_Clear(); + } + return value; +} + +inline tuple get_type_tuple_attr(PyTypeObject *type, const char *name) { + object value = type_generic_getattr(type, name); + if (!value || !PyTuple_Check(value.ptr())) { + return tuple(); + } + return reinterpret_steal(value.release()); +} + +inline tuple get_bases(PyTypeObject *type) { return get_type_tuple_attr(type, "__bases__"); } +inline tuple get_mro(PyTypeObject *type) { return get_type_tuple_attr(type, "__mro__"); } + +// No type attribute cache: walks the MRO and looks in each class's `__dict__` (a mappingproxy). +inline object type_lookup(PyTypeObject *type, handle name) { + for (handle base : get_mro(type)) { + object dict + = type_generic_getattr(reinterpret_cast(base.ptr()), "__dict__"); + if (!dict) { + continue; + } + auto value = reinterpret_steal(PyObject_GetItem(dict.ptr(), name.ptr())); + if (value) { + return value; + } + PyErr_Clear(); + } + return object(); +} +#endif inline object get_module_name_if_available(handle scope) { if (scope) { diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 7a2ac29c7c..d1a5eb652d 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -93,9 +93,14 @@ inline bool object_is_convertible_to_std_vector(const handle &src) { return !PyUnicode_Check(src.ptr()) && !PyBytes_Check(src.ptr()); } // Allow generators, set/frozenset and several common iterable types. - return (PyGen_Check(src.ptr()) != 0) || (PyAnySet_Check(src.ptr()) != 0) +#if !defined(Py_LIMITED_API) + if (PyGen_Check(src.ptr()) != 0) { + return true; + } +#endif + return (PyAnySet_Check(src.ptr()) != 0) || object_is_instance_with_one_of_tp_names( - src.ptr(), {"dict_keys", "dict_values", "dict_items", "map", "zip"}); + src.ptr(), {"generator", "dict_keys", "dict_values", "dict_items", "map", "zip"}); } inline bool object_is_convertible_to_std_set(const handle &src, bool convert) { diff --git a/tests/env.py b/tests/env.py index d205d7ef0e..bbebf53935 100644 --- a/tests/env.py +++ b/tests/env.py @@ -32,6 +32,13 @@ def _is_musl() -> bool: # Compile-time config (what the binary was built for) PY_GIL_DISABLED = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) +try: + import pybind11_tests + + # Built with Py_LIMITED_API (stable ABI); some features and messages differ. + LIMITED_API = bool(pybind11_tests.LIMITED_API) +except ImportError: # the extra_* test suites do not build pybind11_tests + LIMITED_API = False # Runtime state (what's actually happening now) sys_is_gil_enabled = getattr(sys, "_is_gil_enabled", lambda: True) diff --git a/tests/pybind11_tests.cpp b/tests/pybind11_tests.cpp index 3f342a0e9c..03873d7698 100644 --- a/tests/pybind11_tests.cpp +++ b/tests/pybind11_tests.cpp @@ -89,6 +89,12 @@ PYBIND11_MODULE(pybind11_tests, m, py::mod_gil_not_used()) { #endif m.attr("cpp_std") = cpp_std(); m.attr("PYBIND11_INTERNALS_ID") = PYBIND11_INTERNALS_ID; + m.attr("LIMITED_API") = +#if defined(Py_LIMITED_API) + true; +#else + false; +#endif // Free threaded Python uses UINT32_MAX for immortal objects. m.attr("PYBIND11_SIMPLE_GIL_MANAGEMENT") = #if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) diff --git a/tests/test_call_policies.cpp b/tests/test_call_policies.cpp index bf498be846..b5134f9855 100644 --- a/tests/test_call_policies.cpp +++ b/tests/test_call_policies.cpp @@ -254,6 +254,8 @@ TEST_SUBMODULE(call_policies, m) { #if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) // `py::call_guard()` should work in PyPy/GraalPy as well, // but it's unclear how to test it without `PyGILState_GetThisThreadState`. + // (Not under the stable ABI either: get_thread_state_unchecked() then requires the GIL.) +# if !defined(Py_LIMITED_API) auto report_gil_status = []() { auto is_gil_held = false; if (auto *tstate = py::detail::get_thread_state_unchecked()) { @@ -277,6 +279,7 @@ TEST_SUBMODULE(call_policies, m) { return report_gil_status(); }, py::call_guard()); +# endif #endif // test_keep_alive_failed_overload diff --git a/tests/test_call_policies.py b/tests/test_call_policies.py index 935160866b..04d29f5c4c 100644 --- a/tests/test_call_policies.py +++ b/tests/test_call_policies.py @@ -290,7 +290,8 @@ def test_call_guard_rejected_cast(with_hooks): @pytest.mark.skipif( - "env.PYPY or env.GRAALPY", reason="GIL state check requires CPython" + not hasattr(m, "call_guard_cast_without_gil"), + reason="GIL state check requires CPython without Py_LIMITED_API", ) def test_call_guard_cast_without_gil(): state = m.CallGuardState() diff --git a/tests/test_kwargs_and_defaults.cpp b/tests/test_kwargs_and_defaults.cpp index 831947f160..7f672bf592 100644 --- a/tests/test_kwargs_and_defaults.cpp +++ b/tests/test_kwargs_and_defaults.cpp @@ -175,7 +175,7 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { py::tuple t(a.size()); for (size_t i = 0; i < a.size(); i++) { // Use raw Python API here to avoid an extra, intermediate incref on the tuple item: - t[i] = REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast(i))); + t[i] = REFCNT(PyTuple_GetItem(a.ptr(), static_cast(i))); } return t; }); @@ -185,7 +185,7 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { t[0] = o.ref_count(); for (size_t i = 0; i < a.size(); i++) { // Use raw Python API here to avoid an extra, intermediate incref on the tuple item: - t[i + 1] = REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast(i))); + t[i + 1] = REFCNT(PyTuple_GetItem(a.ptr(), static_cast(i))); } return t; }); diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index c0943742da..2fd765af04 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -221,6 +221,9 @@ def check_self(self): instance.static_cls = check_self +@pytest.mark.skipif( + "env.LIMITED_API", reason="PyType_FromMetaclass picks the most derived metaclass" +) def test_metaclass_override(): """Overriding pybind11's default metaclass changes the behavior of `static_property`""" From 4802878738d5c95d8be795167e232035cfbbcc8f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 17:19:22 -0400 Subject: [PATCH 19/45] feat(cmake): add STABLE_ABI to pybind11_add_module() pybind11_add_module(... STABLE_ABI) builds the module against the Python stable ABI through python_add_library(USE_SABI) (CMake 3.26+, FindPython mode) and names it *.abi3.so / *.pyd. PYBIND11_STABLE_ABI makes it the default for a build tree and NO_STABLE_ABI opts a target out; PYBIND11_STABLE_ABI_VERSION (default 3.12) selects the targeted version. SHARED helper libraries get the same Py_LIMITED_API definition. The classic (FindPythonLibs) mode reports a clear error. pybind11_precompile(STABLE_ABI) compiles the precompiled library against the limited API; one build tree cannot mix stable-ABI and regular precompiled modules. The test suite gains PYBIND11_TEST_STABLE_ABI, which builds every test module as abi3 and skips the features that the stable ABI cannot offer. See https://github.com/pybind/pybind11/discussions/6104 (D2). Assisted-by: ClaudeCode:claude-fable-5-1 --- docs/compiling.rst | 14 ++++ tests/CMakeLists.txt | 27 ++++++- tests/test_cmake_build/CMakeLists.txt | 10 ++- tools/pybind11Common.cmake | 25 +++++- tools/pybind11NewTools.cmake | 107 ++++++++++++++++++++++++-- tools/pybind11Tools.cmake | 12 ++- 6 files changed, 176 insertions(+), 19 deletions(-) diff --git a/docs/compiling.rst b/docs/compiling.rst index 9e61949b02..9a8ffd7d73 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -349,6 +349,7 @@ function with the following signature: pybind11_add_module( [MODULE | SHARED] [EXCLUDE_FROM_ALL] [NO_EXTRAS] [THIN_LTO] [OPT_SIZE] [PRECOMPILE | NO_PRECOMPILE] + [STABLE_ABI | NO_STABLE_ABI] source1 [source2 ...]) This function behaves very much like CMake's builtin ``add_library`` (in fact, @@ -405,6 +406,15 @@ optimizations remain disabled. .. _ThinLTO: http://clang.llvm.org/docs/ThinLTO.html +``STABLE_ABI`` builds the module against the Python stable ABI +(``Py_LIMITED_API``), so that one ``.abi3`` module runs on every CPython from +the targeted version on (3.12 by default; set ``PYBIND11_STABLE_ABI_VERSION`` +to target a newer one). Set the CMake variable ``PYBIND11_STABLE_ABI`` to make +it the default for all ``pybind11_add_module`` calls and use ``NO_STABLE_ABI`` +to opt a target out. This needs CMake 3.26+, the FindPython mode +(``PYBIND11_FINDPYTHON=ON``) and CPython 3.12+ headers; it is not available +for free-threaded builds, PyPy, or GraalPy. + .. _precompile-mode: Pre-compiling part of pybind11 @@ -469,6 +479,10 @@ Requirements and caveats: * The library is static and per-build-tree; it is never installed or shared between projects. Each extension module links its own copy, which keeps pybind11's per-module state the same as in header-only mode. +* With ``STABLE_ABI`` the library is compiled against the limited API too. + One build tree cannot mix stable-ABI and regular precompiled modules; the + first ``PRECOMPILE`` target decides, and a later mismatch is a configure + error. * Not available with ``PYBIND11_NOPYTHON`` (the library needs Python headers). diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c23e7668e1..58e80b83f5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -72,12 +72,17 @@ option(DOWNLOAD_EIGEN "Download EIGEN" OFF) option(PYBIND11_CUDA_TESTS "Enable building CUDA tests" OFF) option(PYBIND11_TEST_SMART_HOLDER "Change the default to smart holder" OFF) option(PYBIND11_TEST_PRECOMPILE "Build the test modules against the precompiled library" OFF) +option(PYBIND11_TEST_STABLE_ABI "Build the test modules for the stable ABI (Py_LIMITED_API)" OFF) if(PYBIND11_TEST_PRECOMPILE) # Picked up by every pybind11_add_module() call below; any definition missing # from the precompiled library shows up as a link error across the test modules. set(PYBIND11_PRECOMPILE ON) endif() +if(PYBIND11_TEST_STABLE_ABI) + # Picked up by every pybind11_add_module() call below. + set(PYBIND11_STABLE_ABI ON) +endif() # These modules redefine PYBIND11_INTERNALS_VERSION inside their own TU, so they # cannot share the precompiled library. set(PYBIND11_NO_PRECOMPILE_TARGETS exo_planet_pybind11 cross_module_gil_utils) @@ -220,6 +225,19 @@ if(PYBIND11_TEST_FILTER) pybind11_filter_tests(PYBIND11_TEST_FILES ${PYBIND11_TEST_FILTER}) endif() +# Features that are not available under the stable ABI (see docs/advanced/stable_abi.rst). +if(PYBIND11_TEST_STABLE_ABI) + pybind11_filter_tests( + PYBIND11_TEST_FILES + test_chrono.cpp + test_custom_type_setup.cpp + test_multiple_interpreters.py + test_class_release_gil_before_calling_cpp_dtor.cpp # PyGILState_Check + MESSAGE + "Skipping chrono, custom_type_setup, subinterpreter and PyGILState_Check tests under the stable ABI" + ) +endif() + # Skip tests for CUDA check: # /pybind11/tests/test_constants_and_functions.cpp(125): # error: incompatible exception specifications @@ -603,7 +621,7 @@ add_custom_target( WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" USES_TERMINAL) -if(NOT PYBIND11_CUDA_TESTS) +if(NOT PYBIND11_CUDA_TESTS AND NOT PYBIND11_TEST_STABLE_ABI) set(PYBIND11_MULTIPLE_INTERPRETERS_TEST_MODULES mod_per_interpreter_gil mod_shared_interpreter_gil mod_per_interpreter_gil_with_singleton) @@ -698,7 +716,12 @@ if(NOT PYBIND11_CUDA_TESTS) add_subdirectory(pure_cpp) # Test C++ code that depends on Python, such as embedding the interpreter. Provides the `cpptest` target. - add_subdirectory(test_with_catch) + if(PYBIND11_TEST_STABLE_ABI) + message(STATUS "Skipping embed tests under the stable ABI") + add_custom_target(cpptest) # Dummy target: embed.h is not available under Py_LIMITED_API. + else() + add_subdirectory(test_with_catch) + endif() # Test CMake build using functions and targets from subdirectory or installed location add_subdirectory(test_cmake_build) diff --git a/tests/test_cmake_build/CMakeLists.txt b/tests/test_cmake_build/CMakeLists.txt index 33a78a6b1d..1d0e4af566 100644 --- a/tests/test_cmake_build/CMakeLists.txt +++ b/tests/test_cmake_build/CMakeLists.txt @@ -23,7 +23,7 @@ function(pybind11_add_build_test name) # Forward option-driven defines so the nested projects can verify they propagate # through the pybind11_headers usage requirements. foreach(opt IN ITEMS PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION - PYBIND11_SIMPLE_GIL_MANAGEMENT) + PYBIND11_SIMPLE_GIL_MANAGEMENT PYBIND11_STABLE_ABI) if(${opt}) list(APPEND build_options "-D${opt}=ON") endif() @@ -72,8 +72,9 @@ pybind11_add_build_test(subdirectory_target) pybind11_add_build_test(subdirectory_precompile) if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy" - OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy") - message(STATUS "Skipping embed test on PyPy or GraalPy") + OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy" + OR PYBIND11_STABLE_ABI) + message(STATUS "Skipping embed test on PyPy, GraalPy, or under the stable ABI") else() pybind11_add_build_test(subdirectory_embed) endif() @@ -91,7 +92,8 @@ if(PYBIND11_INSTALL) if(NOT ("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy" - OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy")) + OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy" + OR PYBIND11_STABLE_ABI)) pybind11_add_build_test(installed_embed INSTALL) endif() endif() diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index 0073d3634d..8d204afd39 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -488,9 +488,19 @@ endfunction() # built from the consumer's project with the consumer's flags; each extension # module links its own copy, preserving pybind11's per-module state. Modules # using it must compile with PYBIND11_PRECOMPILED, which the PUBLIC compile -# definition below provides automatically. +# definition below provides automatically. With STABLE_ABI the library is +# compiled against the limited API (Py_LIMITED_API); one build tree cannot mix +# stable-ABI and regular precompiled modules. function(pybind11_precompile) + cmake_parse_arguments(ARG "STABLE_ABI" "" "" ${ARGN}) if(TARGET pybind11_precompiled) + get_target_property(_existing_sabi pybind11_precompiled PYBIND11_STABLE_ABI) + if(NOT "${_existing_sabi}" STREQUAL "${ARG_STABLE_ABI}") + message( + FATAL_ERROR + "pybind11::precompiled was created for STABLE_ABI=${_existing_sabi}; one build tree " + "cannot mix stable-ABI and regular precompiled modules.") + endif() return() endif() @@ -512,6 +522,11 @@ function(pybind11_precompile) add_library(pybind11_precompiled STATIC EXCLUDE_FROM_ALL ${_pybind11_precompile_sources}) add_library(pybind11::precompiled ALIAS pybind11_precompiled) target_compile_definitions(pybind11_precompiled PUBLIC PYBIND11_PRECOMPILED) + set_target_properties(pybind11_precompiled PROPERTIES PYBIND11_STABLE_ABI "${ARG_STABLE_ABI}") + if(ARG_STABLE_ABI) + _pybind11_stable_abi_hex(_sabi_hex) + target_compile_definitions(pybind11_precompiled PRIVATE "Py_LIMITED_API=${_sabi_hex}") + endif() # pybind11::module (not just pybind11::pybind11): the library must compile with the # interpreter's ABI macros (e.g. Py_GIL_DISABLED, which FindPython attaches to # Python::Module); on free-threaded Windows they select the correct autolink library. @@ -535,9 +550,13 @@ endfunction() # Link pybind11::precompiled when the PRECOMPILE keyword or the global # PYBIND11_PRECOMPILE variable requests it, unless NO_PRECOMPILE opts out. -function(_pybind11_maybe_precompile target_name precompile no_precompile) +function(_pybind11_maybe_precompile target_name precompile no_precompile stable_abi) if((precompile OR PYBIND11_PRECOMPILE) AND NOT no_precompile) - pybind11_precompile() + if(stable_abi) + pybind11_precompile(STABLE_ABI) + else() + pybind11_precompile() + endif() target_link_libraries(${target_name} PRIVATE pybind11::precompiled) endif() endfunction() diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 175268d113..3735ff2cbb 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -35,11 +35,15 @@ if(NOT Python_FOUND AND NOT Python3_FOUND) set(_pybind11_interp_component Interpreter) endif() - # Development.Module support (required for manylinux) started in 3.18 + # Development.Module support (required for manylinux) started in 3.18; + # Development.SABIModule (stable ABI, see the STABLE_ABI option) in 3.26. if(CMAKE_VERSION VERSION_LESS 3.18) set(_pybind11_dev_component Development) - else() + elseif(CMAKE_VERSION VERSION_LESS 3.26) set(_pybind11_dev_component Development.Module OPTIONAL_COMPONENTS Development.Embed) + else() + set(_pybind11_dev_component Development.Module OPTIONAL_COMPONENTS Development.Embed + Development.SABIModule) endif() # Callers need to be able to access Python_EXECUTABLE @@ -252,12 +256,64 @@ else() PROPERTY INTERFACE_LINK_LIBRARIES pybind11::python_link_helper) endif() +# The Py_LIMITED_API value for STABLE_ABI modules (CPython 3.12 is the minimum pybind11 supports). +set(PYBIND11_STABLE_ABI_VERSION + "3.12" + CACHE STRING "Python version whose stable ABI STABLE_ABI modules target (3.12 or newer)") + +# PYBIND11_STABLE_ABI_VERSION as the Py_LIMITED_API hex value. +function(_pybind11_stable_abi_hex out_var) + string(REPLACE "." ";" _parts "${PYBIND11_STABLE_ABI_VERSION}") + list(GET _parts 0 _major) + list(GET _parts 1 _minor) + math(EXPR _hex "(${_major} << 24) | (${_minor} << 16)" OUTPUT_FORMAT HEXADECIMAL) + set(${out_var} + "${_hex}" + PARENT_SCOPE) +endfunction() + +# Fail with an explanation if a STABLE_ABI module cannot be built in this configuration. +function(_pybind11_check_stable_abi target_name lib_type) + if(CMAKE_VERSION VERSION_LESS 3.26) + message(FATAL_ERROR "${target_name}: STABLE_ABI requires CMake 3.26 or newer (USE_SABI).") + endif() + if(lib_type STREQUAL "STATIC") + message(FATAL_ERROR "${target_name}: STABLE_ABI is not supported for STATIC libraries.") + endif() + if(PYBIND11_STABLE_ABI_VERSION VERSION_LESS 3.12) + message(FATAL_ERROR "${target_name}: PYBIND11_STABLE_ABI_VERSION must be 3.12 or newer, " + "got ${PYBIND11_STABLE_ABI_VERSION}.") + endif() + if(DEFINED ${_Python}_INTERPRETER_ID AND NOT "${${_Python}_INTERPRETER_ID}" STREQUAL "Python") + message(FATAL_ERROR "${target_name}: STABLE_ABI requires CPython, found " + "${${_Python}_INTERPRETER_ID}.") + endif() + if(DEFINED ${_Python}_VERSION AND ${_Python}_VERSION VERSION_LESS PYBIND11_STABLE_ABI_VERSION) + message(FATAL_ERROR "${target_name}: STABLE_ABI ${PYBIND11_STABLE_ABI_VERSION} needs Python " + ">= ${PYBIND11_STABLE_ABI_VERSION} headers, found ${${_Python}_VERSION}.") + endif() + if("${${_Python}_SOABI}" MATCHES "^cpython-[0-9]+t" OR "${PYTHON_MODULE_EXTENSION}" MATCHES + "^\\.cpython-[0-9]+t") + message(FATAL_ERROR "${target_name}: the free-threaded build has no stable ABI.") + endif() + if(NOT TARGET ${_Python}::SABIModule) + message( + FATAL_ERROR + "${target_name}: STABLE_ABI needs the Development.SABIModule component " + "of FindPython. If your project calls find_package(Python) itself, add " + "Development.SABIModule to its components.") + endif() +endfunction() + # WITHOUT_SOABI and WITH_SOABI will disable the custom extension handling used by pybind11. # WITH_SOABI is passed on to python_add_library. function(pybind11_add_module target_name) cmake_parse_arguments( - PARSE_ARGV 1 ARG - "STATIC;SHARED;MODULE;THIN_LTO;OPT_SIZE;NO_EXTRAS;WITHOUT_SOABI;PRECOMPILE;NO_PRECOMPILE" "" + PARSE_ARGV + 1 + ARG + "STATIC;SHARED;MODULE;THIN_LTO;OPT_SIZE;NO_EXTRAS;WITHOUT_SOABI;PRECOMPILE;NO_PRECOMPILE;STABLE_ABI;NO_STABLE_ABI" + "" "") if(ARG_STATIC) @@ -268,10 +324,22 @@ function(pybind11_add_module target_name) set(lib_type MODULE) endif() + # STABLE_ABI keyword, or the PYBIND11_STABLE_ABI variable as the default; NO_STABLE_ABI opts out. + set(stable_abi OFF) + set(use_sabi "") + if((ARG_STABLE_ABI OR PYBIND11_STABLE_ABI) AND NOT ARG_NO_STABLE_ABI) + _pybind11_check_stable_abi(${target_name} ${lib_type}) + set(stable_abi ON) + if(lib_type STREQUAL "MODULE") + # Defines Py_LIMITED_API and links Python::SABIModule (python3.lib on Windows). + set(use_sabi USE_SABI ${PYBIND11_STABLE_ABI_VERSION}) + endif() + endif() + if("${_Python}" STREQUAL "Python") - python_add_library(${target_name} ${lib_type} ${ARG_UNPARSED_ARGUMENTS}) + python_add_library(${target_name} ${lib_type} ${use_sabi} ${ARG_UNPARSED_ARGUMENTS}) elseif("${_Python}" STREQUAL "Python3") - python3_add_library(${target_name} ${lib_type} ${ARG_UNPARSED_ARGUMENTS}) + python3_add_library(${target_name} ${lib_type} ${use_sabi} ${ARG_UNPARSED_ARGUMENTS}) else() message(FATAL_ERROR "Cannot detect FindPython version: ${_Python}") endif() @@ -280,17 +348,27 @@ function(pybind11_add_module target_name) if(lib_type STREQUAL "MODULE") target_link_libraries(${target_name} PRIVATE pybind11::module) + elseif(stable_abi) + # A SHARED helper library that stable-ABI modules link: same ABI, no embedding. + _pybind11_stable_abi_hex(_sabi_hex) + target_compile_definitions(${target_name} PRIVATE "Py_LIMITED_API=${_sabi_hex}") + target_link_libraries(${target_name} PRIVATE ${_Python}::SABIModule) else() target_link_libraries(${target_name} PRIVATE pybind11::embed) endif() - _pybind11_maybe_precompile(${target_name} "${ARG_PRECOMPILE}" "${ARG_NO_PRECOMPILE}") + _pybind11_maybe_precompile(${target_name} "${ARG_PRECOMPILE}" "${ARG_NO_PRECOMPILE}" + "${stable_abi}") _pybind11_default_hidden_visibility(${target_name}) # If we don't pass a WITH_SOABI or WITHOUT_SOABI, use our own default handling of extensions if(NOT ARG_WITHOUT_SOABI AND NOT "WITH_SOABI" IN_LIST ARG_UNPARSED_ARGUMENTS) - pybind11_extension(${target_name}) + if(stable_abi) + pybind11_extension_stable_abi(${target_name}) + else() + pybind11_extension(${target_name}) + endif() endif() if(ARG_NO_EXTRAS) @@ -331,3 +409,16 @@ function(pybind11_extension name) DEBUG_POSTFIX "${PYTHON_MODULE_DEBUG_POSTFIX}" SUFFIX "${PYTHON_MODULE_EXTENSION}") endfunction() + +# Stable-ABI modules carry the version-independent "abi3" tag (none on Windows). +function(pybind11_extension_stable_abi name) + if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(_ext ".pyd") + else() + set(_ext "${CMAKE_SHARED_MODULE_SUFFIX}") + endif() + if(DEFINED ${_Python}_SOSABI AND NOT "${${_Python}_SOSABI}" STREQUAL "") + set(_ext ".${${_Python}_SOSABI}${_ext}") + endif() + set_target_properties(${name} PROPERTIES PREFIX "" SUFFIX "${_ext}") +endfunction() diff --git a/tools/pybind11Tools.cmake b/tools/pybind11Tools.cmake index 9a7e5c6217..01286c77dc 100644 --- a/tools/pybind11Tools.cmake +++ b/tools/pybind11Tools.cmake @@ -140,9 +140,17 @@ endfunction() # function(pybind11_add_module target_name) set(options - "MODULE;SHARED;EXCLUDE_FROM_ALL;NO_EXTRAS;SYSTEM;THIN_LTO;OPT_SIZE;PRECOMPILE;NO_PRECOMPILE") + "MODULE;SHARED;EXCLUDE_FROM_ALL;NO_EXTRAS;SYSTEM;THIN_LTO;OPT_SIZE;PRECOMPILE;NO_PRECOMPILE;STABLE_ABI;NO_STABLE_ABI" + ) cmake_parse_arguments(ARG "${options}" "" "" ${ARGN}) + if((ARG_STABLE_ABI OR PYBIND11_STABLE_ABI) AND NOT ARG_NO_STABLE_ABI) + message( + FATAL_ERROR + "${target_name}: STABLE_ABI requires the FindPython mode (PYBIND11_FINDPYTHON=ON) " + "and CMake 3.26 or newer.") + endif() + if(ARG_MODULE AND ARG_SHARED) message(FATAL_ERROR "Can't be both MODULE and SHARED") elseif(ARG_SHARED) @@ -161,7 +169,7 @@ function(pybind11_add_module target_name) target_link_libraries(${target_name} PRIVATE pybind11::module) - _pybind11_maybe_precompile(${target_name} "${ARG_PRECOMPILE}" "${ARG_NO_PRECOMPILE}") + _pybind11_maybe_precompile(${target_name} "${ARG_PRECOMPILE}" "${ARG_NO_PRECOMPILE}" OFF) if(ARG_SYSTEM) message( From bd5a17bdf2cb7fb4a95073e1ff4878c375c920d0 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 17:19:22 -0400 Subject: [PATCH 20/45] feat(setup_helpers): Pybind11Extension(py_limited_api=True) py_limited_api=True (or a version such as "3.13") defines Py_LIMITED_API for the targeted version in addition to setuptools' abi3 file naming, so one wheel serves every CPython from that version on. See https://github.com/pybind/pybind11/discussions/6104 (D3). Assisted-by: ClaudeCode:claude-fable-5-1 --- pybind11/setup_helpers.py | 41 +++++++++++ tests/extra_setuptools/test_setuphelper.py | 80 ++++++++++++++++++++++ 2 files changed, 121 insertions(+) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 66a3eec0d4..d9aecae352 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -41,6 +41,7 @@ import contextlib import os import platform +import re import shlex import shutil import sys @@ -86,6 +87,30 @@ # directory into your path if it sits beside your setup.py. +def _limited_api_hex(value: Any) -> str: + """ + The Py_LIMITED_API value for ``py_limited_api``: True selects 3.12 (the + minimum pybind11 supports), a string such as "3.13" or "cp313" selects + that version, and an int or hex string is passed through. + """ + if value is True: + return "0x030C0000" + if isinstance(value, int): + return f"0x{value:08X}" + text = str(value) + if text.lower().startswith("0x"): + return text + match = re.fullmatch(r"(?:cp)?(\d)\.?(\d+)", text) + if not match: + msg = f"py_limited_api must be True or a Python version such as '3.12', got {value!r}" + raise ValueError(msg) + major, minor = (int(part) for part in match.groups()) + if (major, minor) < (3, 12): + msg = f"pybind11 supports the stable ABI from Python 3.12 on, got {value!r}" + raise ValueError(msg) + return f"0x{major:02X}{minor:02X}0000" + + class Pybind11Extension(_Extension): """ Build a C++11+ Extension module with pybind11. This automatically adds the @@ -108,6 +133,12 @@ class Pybind11Extension(_Extension): If you want to add pybind11 headers manually, for example for an exact git checkout, then set ``include_pybind11=False``. + + ``py_limited_api=True`` builds against the Python stable ABI (3.12, or + pass a version string such as ``"3.13"``): ``Py_LIMITED_API`` is defined + and setuptools names the module ``*.abi3.so``, so one wheel serves every + CPython from that version on. See the pybind11 documentation for the + features that are not available under the stable ABI. """ # flags are prepended, so that they can be further overridden, e.g. by @@ -128,6 +159,16 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: include_pybind11 = kwargs.pop("include_pybind11", True) + py_limited_api = kwargs.get("py_limited_api", False) + if py_limited_api: + # setuptools only uses the flag for the .abi3 file name; the + # define selects the limited API in the headers. + kwargs["py_limited_api"] = True + macros = list(kwargs.get("define_macros", []) or []) + if not any(name == "Py_LIMITED_API" for name, _ in macros): + macros.append(("Py_LIMITED_API", _limited_api_hex(py_limited_api))) + kwargs["define_macros"] = macros + super().__init__(*args, **kwargs) # Include the installed package pybind11 headers diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py index 22c0c76f5d..9404caaff4 100644 --- a/tests/extra_setuptools/test_setuphelper.py +++ b/tests/extra_setuptools/test_setuphelper.py @@ -3,6 +3,7 @@ import os import subprocess import sys +import sysconfig from textwrap import dedent import pytest @@ -110,6 +111,85 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std): ) +@pytest.mark.skipif( + sys.implementation.name != "cpython" + or sys.version_info < (3, 12) + or sysconfig.get_config_var("Py_GIL_DISABLED"), + reason="stable ABI needs CPython 3.12+ with the GIL", +) +def test_stable_abi_setup_py(monkeypatch, tmpdir): + monkeypatch.chdir(tmpdir) + monkeypatch.syspath_prepend(MAIN_DIR) + + (tmpdir / "setup.py").write_text( + dedent( + f"""\ + import sys + sys.path.append({MAIN_DIR!r}) + + from setuptools import setup + from pybind11.setup_helpers import build_ext, Pybind11Extension + + ext_modules = [ + Pybind11Extension( + "stable_abi_setup", + ["main.cpp"], + cxx_std=17, + py_limited_api=True, + ), + ] + + setup( + name="stable_abi_setup_package", + cmdclass={{"build_ext": build_ext}}, + ext_modules=ext_modules, + ) + """ + ), + encoding="ascii", + ) + + (tmpdir / "main.cpp").write_text( + dedent( + """\ + #include + + #if !defined(Py_LIMITED_API) + # error "Py_LIMITED_API should be defined" + #endif + + struct Point { int x; }; + + PYBIND11_MODULE(stable_abi_setup, m, pybind11::mod_gil_used()) { + pybind11::class_(m, "Point") + .def(pybind11::init()) + .def_readwrite("x", &Point::x); + } + """ + ), + encoding="ascii", + ) + + subprocess.check_call([sys.executable, "setup.py", "build_ext", "--inplace"]) + + built = [f for f in tmpdir.listdir() if f.basename.startswith("stable_abi_setup")] + assert len(built) == 1 + assert ".abi3" in built[0].basename or built[0].ext == ".pyd" + + (tmpdir / "test.py").write_text( + dedent( + """\ + import stable_abi_setup + assert stable_abi_setup.Point(4).x == 4 + """ + ), + encoding="ascii", + ) + subprocess.check_call( + [sys.executable, "test.py"], stdout=sys.stdout, stderr=sys.stderr + ) + + def test_intree_extensions(monkeypatch, tmpdir): monkeypatch.syspath_prepend(MAIN_DIR) From 8ac98c96027ea8dca4f6703fa63b795fe28faf8f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 17:20:36 -0400 Subject: [PATCH 21/45] ci: build and test the suite for the stable ABI Build every test module as abi3 on Linux, macOS, and Windows (one job also with the precompiled library), and run the modules built against the 3.12 headers unchanged on 3.13 and 3.14. See https://github.com/pybind/pybind11/discussions/6104 (D4). Assisted-by: ClaudeCode:claude-fable-5-1 --- .github/workflows/ci.yml | 93 ++++++++++++++++++++++++++++++++++++++++ tests/conftest.py | 10 +++-- tests/pybind11_tests.cpp | 3 ++ 3 files changed, 102 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b47e5c5120..d726a0c6d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,6 +127,99 @@ jobs: python-version: ${{ matrix.python-version }} cmake-args: ${{ matrix.cmake-args }} + # Stable ABI (Py_LIMITED_API): every test module is built as *.abi3. + stable-abi: + if: github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + include: + - runs-on: ubuntu-latest + python-version: '3.12' + cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON + # precompiled: the -inl.h files must not read any non-limited struct field + - runs-on: ubuntu-latest + python-version: '3.14' + cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON -DPYBIND11_TEST_PRECOMPILE=ON + - runs-on: macos-latest + python-version: '3.13' + cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON -DCMAKE_CXX_STANDARD=17 + - runs-on: windows-2022 + python-version: '3.12' + cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON + + name: 🧱 + uses: ./.github/workflows/reusable-standard.yml + with: + runs-on: ${{ matrix.runs-on }} + python-version: ${{ matrix.python-version }} + cmake-args: ${{ matrix.cmake-args }} + + # The point of the stable ABI: build the suite once against the 3.12 headers + # and run the same .abi3 modules on newer interpreters. + stable-abi-forward: + if: github.event.pull_request.draft == false + name: 🧱 abi3 on newer Pythons + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + PYBIND11_STABLE_ABI_SKIPPED_TESTS: >- + --ignore=tests/test_chrono.py + --ignore=tests/test_custom_type_setup.py + --ignore=tests/test_multiple_interpreters.py + --ignore=tests/test_class_release_gil_before_calling_cpp_dtor.py + + steps: + - uses: actions/checkout@v7 + + - name: Setup Python 3.12 (build) + id: py312 + uses: actions/setup-python@v7 + with: + python-version: '3.12' + + - name: Setup Python 3.13 and 3.14 (run) + uses: actions/setup-python@v7 + with: + python-version: | + 3.13 + 3.14 + allow-prereleases: true + + - name: Install uv + uses: astral-sh/setup-uv@v10.0.1 + with: + enable-cache: true + + - name: Prepare env + run: | + for py in "${{ steps.py312.outputs.python-path }}" python3.13 python3.14; do + uv pip install --python="$py" --system -r tests/requirements.txt + done + + - name: Configure (Python 3.12, stable ABI) + run: > + cmake -S. -Bbuild -Werror=dev -GNinja + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DPYBIND11_TEST_STABLE_ABI=ON + -DPYTHON_EXECUTABLE=${{ steps.py312.outputs.python-path }} + + - name: Build + run: cmake --build build + + - name: Python tests (3.12) + run: cmake --build build --target pytest + + - name: Python tests (3.13, same binaries) + working-directory: build/tests + run: python3.13 -m pytest -v ../../tests ${{ env.PYBIND11_STABLE_ABI_SKIPPED_TESTS }} + + - name: Python tests (3.14, same binaries) + working-directory: build/tests + run: python3.14 -m pytest -v ../../tests ${{ env.PYBIND11_STABLE_ABI_SKIPPED_TESTS }} + standard-large: if: github.event.pull_request.draft == false strategy: diff --git a/tests/conftest.py b/tests/conftest.py index 9d9815b88c..66403c1577 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -294,14 +294,16 @@ def pytest_report_header(): @pytest.fixture def backport_typehints() -> Callable[[SanitizedString], SanitizedString]: d = {} - if sys.version_info < (3, 13): + # The spellings are chosen at compile time (a stable-ABI module may run on a newer Python). + compiled = tuple(pybind11_tests.COMPILED_PYTHON_VERSION) + if compiled < (3, 13): d["typing_extensions.TypeIs"] = "typing.TypeIs" d["typing_extensions.CapsuleType"] = "types.CapsuleType" - if sys.version_info < (3, 12): + if compiled < (3, 12): d["typing_extensions.Buffer"] = "collections.abc.Buffer" - if sys.version_info < (3, 11): + if compiled < (3, 11): d["typing_extensions.Never"] = "typing.Never" - if sys.version_info < (3, 10): + if compiled < (3, 10): d["typing_extensions.TypeGuard"] = "typing.TypeGuard" def backport(sanitized_string: SanitizedString) -> SanitizedString: diff --git a/tests/pybind11_tests.cpp b/tests/pybind11_tests.cpp index 03873d7698..aaaacae459 100644 --- a/tests/pybind11_tests.cpp +++ b/tests/pybind11_tests.cpp @@ -89,6 +89,9 @@ PYBIND11_MODULE(pybind11_tests, m, py::mod_gil_not_used()) { #endif m.attr("cpp_std") = cpp_std(); m.attr("PYBIND11_INTERNALS_ID") = PYBIND11_INTERNALS_ID; + // The Python version the headers were compiled against; differs from the runtime version for + // stable-ABI modules. + m.attr("COMPILED_PYTHON_VERSION") = py::make_tuple(PY_MAJOR_VERSION, PY_MINOR_VERSION); m.attr("LIMITED_API") = #if defined(Py_LIMITED_API) true; From ee6d6568fdcdf10fe6ec2226dd212b08ba414e94 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 17:20:36 -0400 Subject: [PATCH 22/45] docs: describe stable ABI (abi3) support How to enable it with CMake, setuptools, or by hand; the feature matrix; ABI isolation between stable-ABI and regular modules; and the implementation differences that affect performance. See https://github.com/pybind/pybind11/discussions/6104 (D5). Assisted-by: ClaudeCode:claude-fable-5-1 --- docs/advanced/stable_abi.rst | 120 +++++++++++++++++++++++++++++++++++ docs/compiling.rst | 3 +- docs/index.rst | 1 + 3 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 docs/advanced/stable_abi.rst diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst new file mode 100644 index 0000000000..1bbfad2a50 --- /dev/null +++ b/docs/advanced/stable_abi.rst @@ -0,0 +1,120 @@ +Python stable ABI (abi3) +######################## + +CPython's `stable ABI `_ lets +one extension module run on every CPython release from the version it was +built for. pybind11 supports it from CPython 3.12 on: define +``Py_LIMITED_API`` to ``0x030C0000`` (or a newer version) and ship one +``*.abi3.so`` / ``*.pyd`` per platform instead of one module per Python +version. + +Enabling it +=========== + +With CMake, add ``STABLE_ABI`` to ``pybind11_add_module`` (CMake 3.26+ and +the FindPython mode are required), or set ``PYBIND11_STABLE_ABI`` to make it +the default for a build tree: + +.. code-block:: cmake + + pybind11_add_module(example STABLE_ABI example.cpp) + +``PYBIND11_STABLE_ABI_VERSION`` (default ``3.12``) selects the targeted +version. ``PRECOMPILE`` can be combined with ``STABLE_ABI``; the precompiled +library is then compiled against the limited API too, and one build tree +cannot mix stable-ABI and regular precompiled modules. + +With setuptools, pass ``py_limited_api=True`` (or a version such as +``"3.13"``) to ``Pybind11Extension``; the module is named ``*.abi3.so`` and +``Py_LIMITED_API`` is defined for you. + +With any other build system, define ``Py_LIMITED_API=0x030C0000`` for every +translation unit, link no version-specific Python library (``python3.lib`` on +Windows), and name the module ``.abi3.so`` (``.pyd`` on Windows). + +pybind11 rejects the combination with PyPy, GraalPy, and free-threaded +CPython at compile time: they have no stable ABI. + +What changes under the stable ABI +================================= + +The C++ API of pybind11 is the same; the differences are in what is +available and in how a few things are implemented. + +.. list-table:: + :header-rows: 1 + :widths: 30 20 50 + + * - Feature + - Stable ABI + - Notes + * - Classes, functions, casters, STL, numpy, Eigen + - supported + - + * - ``py::dynamic_attr()`` + - supported + - The ``__dict__`` slot is appended to the instance (no + ``Py_TPFLAGS_MANAGED_DICT``). + * - Buffer protocol + - supported + - Through the ``Py_bf_*`` type slots. + * - ``py::gil_scoped_acquire`` / ``release`` + - supported + - ``PYBIND11_SIMPLE_GIL_MANAGEMENT`` is forced; ``disarm()`` and + thread-state disassociation are not available. + * - ``py::metaclass(handle)`` + - restricted + - The metaclass must not define ``__new__``, and a metaclass less + derived than ``pybind11_type`` resolves to ``pybind11_type``. + * - ``py::custom_type_setup`` + - not available + - Its callback receives a ``PyHeapTypeObject *``, which is opaque. + * - ``pybind11/embed.h`` + - not available + - Embedding uses non-limited initialization functions. + * - ``pybind11/subinterpreter.h`` + - not available + - Reads thread- and interpreter-state fields. + * - ``pybind11/chrono.h`` + - not available + - The ``datetime`` C API accesses struct fields. + * - GIL-held assertions (``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``) + - not available + - ``PyGILState_Check()`` is not part of the stable ABI. + +ABI isolation +============= + +A stable-ABI module and a regular module can be loaded in one process, but +they do not share pybind11's internal state: the internals ID carries a +``_stable`` tag, so each kind of module registers its own types and +instances. Types bound in one kind of module are opaque to the other, except +through the ``pybind11_conduit_v1`` protocol +(``include/pybind11/conduit/README.txt``), which bridges the two: the C++ ABI +is unchanged, so ``PYBIND11_PLATFORM_ABI_ID`` is the same. + +Implementation notes and performance +==================================== + +Without access to CPython's struct layouts, pybind11 uses public functions +where it used to read fields directly. The costs are small and are confined +to specific operations: + +* Type objects are created with ``PyType_FromMetaclass()`` and + ``PyType_Spec`` (the ``PYBIND11_TYPE_CREATION_VIA_SPEC`` path, also + available as an opt-in for regular builds on CPython 3.12+). +* Instance allocation and deallocation look up ``tp_alloc``/``tp_free`` with + ``PyType_GetSlot()``; the ``__dict__`` of ``py::dynamic_attr()`` instances + is found through the type's cached dictionary offset. +* Attribute access on bound *classes* (not instances) walks ``__mro__`` and + the class dictionaries instead of using ``_PyType_Lookup()``. +* Methods are bound through a pybind11-provided ``instancemethod`` type and + ``types.MethodType`` instead of ``PyInstanceMethod_Type`` / + ``PyMethod_New()``. +* Tuple and list element access uses the checked function forms; type names + in error messages are reconstructed from ``__module__`` and ``__name__``. +* ``py::eval``/``py::exec`` compile with ``Py_CompileString()`` and run with + ``PyEval_EvalCode()``; ``py::eval_file`` reads the file through Python. + +Modules built for the stable ABI check the interpreter version at import: +any CPython from the targeted version on is accepted. diff --git a/docs/compiling.rst b/docs/compiling.rst index 9a8ffd7d73..6895c518cf 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -413,7 +413,8 @@ to target a newer one). Set the CMake variable ``PYBIND11_STABLE_ABI`` to make it the default for all ``pybind11_add_module`` calls and use ``NO_STABLE_ABI`` to opt a target out. This needs CMake 3.26+, the FindPython mode (``PYBIND11_FINDPYTHON=ON``) and CPython 3.12+ headers; it is not available -for free-threaded builds, PyPy, or GraalPy. +for free-threaded builds, PyPy, or GraalPy. See :doc:`advanced/stable_abi` +for the features that are not available under the stable ABI. .. _precompile-mode: diff --git a/docs/index.rst b/docs/index.rst index 77b097c574..a25684f54f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -35,6 +35,7 @@ advanced/cast/index advanced/pycpp/index advanced/embedding + advanced/stable_abi advanced/misc advanced/deprecated From a221b18a6077718e5c00335303cdc0056c8e20ec Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 18:22:27 -0400 Subject: [PATCH 23/45] fix(cmake): export SABIModule target and Python_SOSABI to callers When the consumer sets Python_ARTIFACTS_INTERACTIVE before add_subdirectory(pybind11), FindPython runs in pybind11's directory scope. Python::SABIModule was not promoted to global and Python_SOSABI was not cached, so STABLE_ABI modules either failed to configure or lost the .abi3 suffix. Add a test_cmake_build case that covers this. Assisted-by: ClaudeCode:claude-fable-5-1 --- tests/test_cmake_build/CMakeLists.txt | 5 +++ .../subdirectory_stable_abi/CMakeLists.txt | 37 +++++++++++++++++++ tests/test_cmake_build/test.py | 5 +++ tools/pybind11NewTools.cmake | 8 ++++ 4 files changed, 55 insertions(+) create mode 100644 tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt diff --git a/tests/test_cmake_build/CMakeLists.txt b/tests/test_cmake_build/CMakeLists.txt index 1d0e4af566..c145122f2e 100644 --- a/tests/test_cmake_build/CMakeLists.txt +++ b/tests/test_cmake_build/CMakeLists.txt @@ -78,6 +78,11 @@ if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" else() pybind11_add_build_test(subdirectory_embed) endif() +if(PYBIND11_STABLE_ABI + AND CMAKE_VERSION VERSION_GREATER_EQUAL 3.26 + AND NOT CMAKE_GENERATOR_PLATFORM MATCHES "Win32") + pybind11_add_build_test(subdirectory_stable_abi) +endif() if(PYBIND11_INSTALL) add_custom_target( diff --git a/tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt b/tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt new file mode 100644 index 0000000000..d835c7c2ae --- /dev/null +++ b/tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt @@ -0,0 +1,37 @@ +cmake_minimum_required(VERSION 3.26...4.2) + +project(test_subdirectory_stable_abi CXX) + +# Allow PYTHON_EXECUTABLE if in FINDPYTHON mode and building pybind11's tests +# (makes transition easier while we support both modes). +if(DEFINED PYTHON_EXECUTABLE AND NOT DEFINED Python_EXECUTABLE) + set(Python_EXECUTABLE "${PYTHON_EXECUTABLE}") +endif() + +# Keep FindPython in pybind11's directory scope; the results must still reach us. +set(Python_ARTIFACTS_INTERACTIVE TRUE) +set(PYBIND11_FINDPYTHON NEW) + +add_subdirectory("${pybind11_SOURCE_DIR}" pybind11) +pybind11_add_module(test_subdirectory_stable_abi STABLE_ABI ../main.cpp) +set_target_properties(test_subdirectory_stable_abi PROPERTIES OUTPUT_NAME test_cmake_build) + +if(DEFINED Python_EXECUTABLE) + set(_Python_EXECUTABLE "${Python_EXECUTABLE}") +elseif(DEFINED PYTHON_EXECUTABLE) + set(_Python_EXECUTABLE "${PYTHON_EXECUTABLE}") +else() + message(FATAL_ERROR "No Python executable defined (should not be possible at this stage)") +endif() + +add_custom_target( + check_subdirectory_stable_abi + ${CMAKE_COMMAND} + -E + env + PYTHONPATH=$ + PYBIND11_EXPECT_ABI3=$ + ${_Python_EXECUTABLE} + ${PROJECT_SOURCE_DIR}/../test.py + ${PROJECT_NAME} + DEPENDS test_subdirectory_stable_abi) diff --git a/tests/test_cmake_build/test.py b/tests/test_cmake_build/test.py index bb4c20e0cb..627c8530ea 100644 --- a/tests/test_cmake_build/test.py +++ b/tests/test_cmake_build/test.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import sys import test_cmake_build @@ -7,4 +8,8 @@ assert isinstance(__file__, str) # Test this is properly set assert test_cmake_build.add(1, 2) == 3 + +expect_abi3 = os.environ.get("PYBIND11_EXPECT_ABI3") +if expect_abi3: + assert ".abi3." in expect_abi3, f"expected an abi3 module name, got {expect_abi3}" print(f"{sys.argv[1]} imports, runs, and adds: 1 + 2 = 3") diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 3735ff2cbb..4f490572b3 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -73,6 +73,9 @@ if(NOT Python_FOUND AND NOT Python3_FOUND) if(TARGET Python::Module) set_property(TARGET Python::Module PROPERTY IMPORTED_GLOBAL TRUE) endif() + if(TARGET Python::SABIModule) + set_property(TARGET Python::SABIModule PROPERTY IMPORTED_GLOBAL TRUE) + endif() endif() # Explicitly export version for callers (including our own functions) @@ -89,6 +92,11 @@ if(NOT Python_FOUND AND NOT Python3_FOUND) set(Python_VERSION_PATCH "${Python_VERSION_PATCH}" CACHE INTERNAL "") + if(DEFINED Python_SOSABI) + set(Python_SOSABI + "${Python_SOSABI}" + CACHE INTERNAL "") + endif() endif() endif() From 2024bd384ca30aa6743453edaa83f7fe4f8961e2 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 21:34:24 -0400 Subject: [PATCH 24/45] feat: support the free-threaded stable ABI (abi3t, CPython 3.15+) Accept Py_LIMITED_API >= 0x030F0000 together with Py_GIL_DISABLED (PEP 803). PyObject and PyModuleDef are then incomplete types, so pybind11: - keeps `instance` and its other heap-type data as PEP 697 type data and reaches it through get_instance()/instance_object() instead of casts; - exports modules through the PEP 793 PyModExport_ hook; the PYBIND11_MODULE macro is unchanged, create_extension_module() is gone; - locks the internals with critical sections on a private object (PyMutex is not in the stable ABI) and keeps one weak reference per registered instance in place of PyUnstable_TryIncRef(); - tags the internals "_stable_ft", since abi3t modules also load on GIL-enabled interpreters. numpy.h is not available under abi3t: it mirrors NumPy's object layouts. CMake raises the stable ABI version to 3.15 on free-threaded Python and Pybind11Extension(py_limited_api=True) does the same. The test suite is built as .abi3t modules in CI for 3.15t and also run on GIL-enabled 3.15. Assisted-by: ClaudeCode:claude-fable-5-1 --- .github/workflows/ci.yml | 73 +++++++++++++ docs/advanced/stable_abi.rst | 49 +++++++-- docs/compiling.rst | 5 +- include/pybind11/cast.h | 2 +- include/pybind11/detail/class-inl.h | 79 ++++++++++---- include/pybind11/detail/class.h | 4 +- include/pybind11/detail/common.h | 102 +++++++++++++----- .../detail/function_record_pyobject.h | 20 +++- include/pybind11/detail/init.h | 13 ++- include/pybind11/detail/internals-inl.h | 2 +- include/pybind11/detail/internals.h | 44 ++++++-- .../pybind11/detail/type_caster_base-inl.h | 35 ++++-- include/pybind11/detail/type_caster_base.h | 19 ++-- include/pybind11/detail/typeid.h | 8 ++ include/pybind11/detail/value_and_holder.h | 2 +- include/pybind11/numpy.h | 4 + include/pybind11/pybind11-inl.h | 18 ++-- include/pybind11/pybind11.h | 41 ++++++- include/pybind11/pytypes.h | 9 +- .../pybind11/trampoline_self_life_support.h | 6 +- pybind11/setup_helpers.py | 20 ++-- tests/CMakeLists.txt | 13 +++ tests/cross_module_gil_utils.cpp | 52 ++++++--- ...s_module_interleaved_error_already_set.cpp | 38 +++++-- tests/exo_planet_c_api.cpp | 14 +++ tests/extra_setuptools/test_setuphelper.py | 4 +- tests/test_cmake_build/test.py | 2 +- tests/test_modules.cpp | 5 + tests/test_stl_binders.cpp | 8 +- tests/test_stl_binders.py | 9 +- tools/check_inl_headers.py | 1 + tools/pybind11NewTools.cmake | 44 ++++++-- 32 files changed, 589 insertions(+), 156 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d726a0c6d3..3afc593345 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,6 +147,13 @@ jobs: - runs-on: windows-2022 python-version: '3.12' cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON + # free-threaded stable ABI (abi3t, PEP 803): PyObject is opaque + - runs-on: ubuntu-latest + python-version: '3.15t' + cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON + - runs-on: macos-latest + python-version: '3.15t' + cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON -DPYBIND11_TEST_PRECOMPILE=ON name: 🧱 uses: ./.github/workflows/reusable-standard.yml @@ -220,6 +227,72 @@ jobs: working-directory: build/tests run: python3.14 -m pytest -v ../../tests ${{ env.PYBIND11_STABLE_ABI_SKIPPED_TESTS }} + # abi3t modules also load on GIL-enabled interpreters: build the suite with + # 3.15t and run the same .abi3t modules on 3.15. + stable-abi-forward-ft: + if: github.event.pull_request.draft == false + name: 🧱 abi3t on GIL-enabled Python + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + PYBIND11_STABLE_ABI_SKIPPED_TESTS: >- + --ignore=tests/test_chrono.py + --ignore=tests/test_custom_type_setup.py + --ignore=tests/test_multiple_interpreters.py + --ignore=tests/test_class_release_gil_before_calling_cpp_dtor.py + --ignore=tests/test_numpy_array.py + --ignore=tests/test_numpy_dtypes.py + --ignore=tests/test_numpy_scalars.py + --ignore=tests/test_numpy_vectorize.py + --ignore=tests/test_eigen_matrix.py + --ignore=tests/test_eigen_tensor.py + + steps: + - uses: actions/checkout@v7 + + - name: Setup Python 3.15t (build) + id: py315t + uses: actions/setup-python@v7 + with: + python-version: '3.15t' + allow-prereleases: true + + - name: Setup Python 3.15 (run) + uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + + - name: Install uv + uses: astral-sh/setup-uv@v10.0.1 + with: + enable-cache: true + + - name: Prepare env + run: | + for py in "${{ steps.py315t.outputs.python-path }}" python3.15; do + uv pip install --python="$py" --system -r tests/requirements.txt + done + + - name: Configure (Python 3.15t, stable ABI) + run: > + cmake -S. -Bbuild -Werror=dev -GNinja + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DPYBIND11_TEST_STABLE_ABI=ON + -DPYTHON_EXECUTABLE=${{ steps.py315t.outputs.python-path }} + + - name: Build + run: cmake --build build + + - name: Python tests (3.15t) + run: cmake --build build --target pytest + + - name: Python tests (3.15, same binaries) + working-directory: build/tests + run: python3.15 -m pytest -v ../../tests ${{ env.PYBIND11_STABLE_ABI_SKIPPED_TESTS }} + standard-large: if: github.event.pull_request.draft == false strategy: diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index 1bbfad2a50..85543baf9a 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -1,5 +1,5 @@ -Python stable ABI (abi3) -######################## +Python stable ABI (abi3, abi3t) +############################### CPython's `stable ABI `_ lets one extension module run on every CPython release from the version it was @@ -8,6 +8,12 @@ built for. pybind11 supports it from CPython 3.12 on: define ``*.abi3.so`` / ``*.pyd`` per platform instead of one module per Python version. +Free-threaded CPython has its own stable ABI from 3.15 on, ``abi3t`` +(:pep:`803`). pybind11 supports it too: build against the free-threaded +headers with ``Py_LIMITED_API`` set to ``0x030F0000`` or newer, and ship one +``*.abi3t.so`` that every free-threaded *and* GIL-enabled CPython from 3.15 +on can load. See `abi3t`_ below for what differs. + Enabling it =========== @@ -19,21 +25,23 @@ the default for a build tree: pybind11_add_module(example STABLE_ABI example.cpp) -``PYBIND11_STABLE_ABI_VERSION`` (default ``3.12``) selects the targeted -version. ``PRECOMPILE`` can be combined with ``STABLE_ABI``; the precompiled +``PYBIND11_STABLE_ABI_VERSION`` (default ``3.12``, raised to ``3.15`` on +free-threaded Python) selects the targeted version. ``PRECOMPILE`` can be +combined with ``STABLE_ABI``; the precompiled library is then compiled against the limited API too, and one build tree cannot mix stable-ABI and regular precompiled modules. With setuptools, pass ``py_limited_api=True`` (or a version such as -``"3.13"``) to ``Pybind11Extension``; the module is named ``*.abi3.so`` and -``Py_LIMITED_API`` is defined for you. +``"3.13"``) to ``Pybind11Extension``; the module is named ``*.abi3.so`` +(``*.abi3t.so`` on free-threaded Python) and ``Py_LIMITED_API`` is defined +for you. With any other build system, define ``Py_LIMITED_API=0x030C0000`` for every translation unit, link no version-specific Python library (``python3.lib`` on Windows), and name the module ``.abi3.so`` (``.pyd`` on Windows). pybind11 rejects the combination with PyPy, GraalPy, and free-threaded -CPython at compile time: they have no stable ABI. +CPython before 3.15 at compile time: they have no stable ABI. What changes under the stable ABI ================================= @@ -48,9 +56,12 @@ available and in how a few things are implemented. * - Feature - Stable ABI - Notes - * - Classes, functions, casters, STL, numpy, Eigen + * - Classes, functions, casters, STL - supported - + * - numpy, Eigen + - supported (abi3 only) + - ``numpy.h`` mirrors NumPy's object layouts, which abi3t hides. * - ``py::dynamic_attr()`` - supported - The ``__dict__`` slot is appended to the instance (no @@ -82,12 +93,32 @@ available and in how a few things are implemented. - not available - ``PyGILState_Check()`` is not part of the stable ABI. +abi3t +===== + +Under abi3t, ``PyObject`` and ``PyModuleDef`` are incomplete types and +``PyMutex`` and ``PyUnstable_TryIncRef()`` are not available. pybind11 then: + +* stores its per-instance data as :pep:`697` type data + (``PyObject_GetTypeData()``) instead of embedding ``PyObject_HEAD``; +* exports the module through the :pep:`793` ``PyModExport_`` hook + instead of ``PyInit_``, so ``PYBIND11_MODULE`` is unchanged but + ``py::module_::create_extension_module()`` is not available; +* locks its internals with critical sections on a private object, and keeps + one weak reference per registered instance so that the instance registry + can hand out strong references safely (``PyWeakref_GetRef()``). + +``pybind11/numpy.h`` (and therefore the Eigen headers) is not available. The +internals tag is ``_stable_ft``: an abi3t module and an abi3 module loaded +into the same GIL-enabled interpreter do not share internals (see below). + ABI isolation ============= A stable-ABI module and a regular module can be loaded in one process, but they do not share pybind11's internal state: the internals ID carries a -``_stable`` tag, so each kind of module registers its own types and +``_stable`` (abi3) or ``_stable_ft`` (abi3t) tag, so each kind of module +registers its own types and instances. Types bound in one kind of module are opaque to the other, except through the ``pybind11_conduit_v1`` protocol (``include/pybind11/conduit/README.txt``), which bridges the two: the C++ ABI diff --git a/docs/compiling.rst b/docs/compiling.rst index 6895c518cf..4f2143172c 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -412,8 +412,9 @@ the targeted version on (3.12 by default; set ``PYBIND11_STABLE_ABI_VERSION`` to target a newer one). Set the CMake variable ``PYBIND11_STABLE_ABI`` to make it the default for all ``pybind11_add_module`` calls and use ``NO_STABLE_ABI`` to opt a target out. This needs CMake 3.26+, the FindPython mode -(``PYBIND11_FINDPYTHON=ON``) and CPython 3.12+ headers; it is not available -for free-threaded builds, PyPy, or GraalPy. See :doc:`advanced/stable_abi` +(``PYBIND11_FINDPYTHON=ON``) and CPython 3.12+ headers (3.15+ for +free-threaded builds, which produce ``.abi3t`` modules); it is not available +for PyPy or GraalPy. See :doc:`advanced/stable_abi` for the features that are not available under the stable ABI. .. _precompile-mode: diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index a4cdcb1390..718d1bbbef 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -397,7 +397,7 @@ class type_caster : public type_caster { const auto &bases = all_type_info(reinterpret_cast(type::handle_of(h).ptr())); if (bases.size() == 1) { // Only allowing loading from a single-value type - value = values_and_holders(reinterpret_cast(h.ptr())).begin()->value_ptr(); + value = values_and_holders(get_instance(h.ptr())).begin()->value_ptr(); return true; } diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 445b986db1..f1a943ac18 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -412,7 +412,7 @@ PYBIND11_INLINE void traverse_offset_bases(void *valueptr, } } -#ifdef Py_GIL_DISABLED +#if defined(Py_GIL_DISABLED) && !defined(PYBIND11_OPAQUE_PYOBJECT) PYBIND11_INLINE void enable_try_inc_ref(PyObject *obj) { # if PY_VERSION_HEX >= 0x030E00A4 PyUnstable_EnableTryIncRef(obj); @@ -437,8 +437,15 @@ PYBIND11_INLINE void enable_try_inc_ref(PyObject *obj) { #endif PYBIND11_INLINE bool register_instance_impl(void *ptr, instance *self) { assert(ptr); -#ifdef Py_GIL_DISABLED - enable_try_inc_ref(reinterpret_cast(self)); +#if defined(PYBIND11_OPAQUE_PYOBJECT) + if (self->registry_weakref == nullptr) { + self->registry_weakref = PyWeakref_NewRef(instance_object(self), nullptr); + if (self->registry_weakref == nullptr) { + throw error_already_set(); + } + } +#elif defined(Py_GIL_DISABLED) + enable_try_inc_ref(instance_object(self)); #endif with_instance_map(ptr, [&](instance_map &instances) { instances.emplace(ptr, self); }); return true; // unused, but gives the same signature as the deregister func @@ -479,10 +486,20 @@ PYBIND11_INLINE bool deregister_instance(instance *self, void *valptr, const typ // instancemethod: `__get__` binds through types.MethodType, calls and unknown attributes go to // the wrapped function. The type is shared through the internals. struct instancemethod_object { +# if !defined(PYBIND11_OPAQUE_PYOBJECT) PyObject_HEAD +# endif PyObject *func; }; +PYBIND11_INLINE instancemethod_object *instancemethod_data(PyObject *self) { +# if defined(PYBIND11_OPAQUE_PYOBJECT) + return static_cast(PyObject_GetTypeData(self, Py_TYPE(self))); +# else + return reinterpret_cast(self); +# endif +} + PYBIND11_INLINE PyTypeObject *get_bound_method_type() { static PyTypeObject *const type = [] { auto types = reinterpret_steal(PyImport_ImportModule("types")); @@ -517,14 +534,14 @@ instancemethod_descr_get(PyObject *self, PyObject *obj, PyObject * /*type*/) { return self; } return PyObject_CallFunctionObjArgs(reinterpret_cast(get_bound_method_type()), - reinterpret_cast(self)->func, + instancemethod_data(self)->func, obj, nullptr); } extern "C" PYBIND11_INLINE PyObject * instancemethod_call(PyObject *self, PyObject *args, PyObject *kwargs) { - return PyObject_Call(reinterpret_cast(self)->func, args, kwargs); + return PyObject_Call(instancemethod_data(self)->func, args, kwargs); } extern "C" PYBIND11_INLINE PyObject *instancemethod_getattro(PyObject *self, PyObject *name) { @@ -538,11 +555,11 @@ extern "C" PYBIND11_INLINE PyObject *instancemethod_getattro(PyObject *self, PyO return get(descr.ptr(), self, reinterpret_cast(Py_TYPE(self))); } } - return PyObject_GetAttr(reinterpret_cast(self)->func, name); + return PyObject_GetAttr(instancemethod_data(self)->func, name); } extern "C" PYBIND11_INLINE PyObject *instancemethod_repr(PyObject *self) { - PyObject *func = reinterpret_cast(self)->func; + PyObject *func = instancemethod_data(self)->func; auto name = reinterpret_steal(PyObject_GetAttrString(func, "__name__")); if (!name) { PyErr_Clear(); @@ -553,20 +570,20 @@ extern "C" PYBIND11_INLINE PyObject *instancemethod_repr(PyObject *self) { extern "C" PYBIND11_INLINE int instancemethod_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(reinterpret_cast(self)->func); + Py_VISIT(instancemethod_data(self)->func); Py_VISIT(Py_TYPE(self)); return 0; } extern "C" PYBIND11_INLINE int instancemethod_clear(PyObject *self) { - Py_CLEAR(reinterpret_cast(self)->func); + Py_CLEAR(instancemethod_data(self)->func); return 0; } extern "C" PYBIND11_INLINE void instancemethod_dealloc(PyObject *self) { PyTypeObject *type = Py_TYPE(self); PyObject_GC_UnTrack(self); - Py_CLEAR(reinterpret_cast(self)->func); + Py_CLEAR(instancemethod_data(self)->func); type_free(type, self); Py_DECREF(reinterpret_cast(type)); } @@ -577,7 +594,7 @@ PYBIND11_INLINE PyTypeObject *get_instancemethod_type() { static PyMemberDef members[] = {{"__func__", Py_T_OBJECT_EX, offsetof(instancemethod_object, func), - Py_READONLY, + Py_READONLY | PYBIND11_MEMBER_OFFSET_FLAGS, nullptr}, {nullptr, 0, 0, 0, nullptr}}; static PyType_Slot slots[] @@ -592,7 +609,7 @@ PYBIND11_INLINE PyTypeObject *get_instancemethod_type() { {0, nullptr}}; static PyType_Spec spec = {PYBIND11_DUMMY_MODULE_NAME ".instancemethod", - static_cast(sizeof(instancemethod_object)), + PYBIND11_TYPE_DATA_SIZE(instancemethod_object), 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION, slots}; @@ -612,7 +629,7 @@ PYBIND11_INLINE bool is_instancemethod(PyObject *obj) { } PYBIND11_INLINE PyObject *instancemethod_function(PyObject *obj) { - return reinterpret_cast(obj)->func; + return instancemethod_data(obj)->func; } PYBIND11_INLINE PyObject *instancemethod_new(PyObject *func) { @@ -622,7 +639,7 @@ PYBIND11_INLINE PyObject *instancemethod_new(PyObject *func) { return nullptr; } Py_INCREF(func); - reinterpret_cast(self)->func = func; + instancemethod_data(self)->func = func; return self; } @@ -661,6 +678,18 @@ PYBIND11_INLINE PyObject **instance_dict_ptr(PyObject *self) { #endif } +#if defined(PYBIND11_OPAQUE_PYOBJECT) +PYBIND11_INLINE Py_ssize_t instance_data_offset() { + // The same for every pybind11 type: the type data of pybind11_object, whose base is object. + static const Py_ssize_t offset = [] { + handle base = get_internals().instance_base; + auto basicsize = base.attr("__basicsize__").cast(); + return basicsize - PyType_GetTypeDataSize(reinterpret_cast(base.ptr())); + }(); + return offset; +} +#endif + PYBIND11_INLINE PyObject *make_new_instance(PyTypeObject *type) { #if defined(PYPY_VERSION) // PyPy gets tp_basicsize wrong (issue 2482) under multiple inheritance when the first @@ -671,7 +700,7 @@ PYBIND11_INLINE PyObject *make_new_instance(PyTypeObject *type) { } #endif PyObject *self = type_alloc(type); - auto *inst = reinterpret_cast(self); + auto *inst = get_instance(self); // Allocate the value/holder internals: inst->allocate_layout(); @@ -691,7 +720,7 @@ extern "C" PYBIND11_INLINE int pybind11_object_init(PyObject *self, PyObject *, } PYBIND11_INLINE void add_patient(PyObject *nurse, PyObject *patient) { - auto *instance = reinterpret_cast(nurse); + auto *instance = get_instance(nurse); instance->has_patients = true; Py_INCREF(patient); @@ -699,7 +728,7 @@ PYBIND11_INLINE void add_patient(PyObject *nurse, PyObject *patient) { } PYBIND11_INLINE void clear_patients(PyObject *self) { - auto *instance = reinterpret_cast(self); + auto *instance = get_instance(self); std::vector patients; with_internals([&](internals &internals) { @@ -724,7 +753,7 @@ PYBIND11_INLINE void clear_patients(PyObject *self) { } PYBIND11_INLINE void clear_instance(PyObject *self) { - auto *instance = reinterpret_cast(self); + auto *instance = get_instance(self); // Deallocate any values/holders, if present: for (auto &v_h : values_and_holders(instance)) { @@ -748,6 +777,9 @@ PYBIND11_INLINE void clear_instance(PyObject *self) { // Deallocate the value/holder layout internals: instance->deallocate_layout(); +#if defined(PYBIND11_OPAQUE_PYOBJECT) + Py_CLEAR(instance->registry_weakref); +#endif if (instance->weakrefs) { PyObject_ClearWeakRefs(self); } @@ -796,9 +828,12 @@ extern "C" PYBIND11_INLINE void pybind11_object_dealloc(PyObject *self) { PYBIND11_INLINE PyObject *make_object_base_type(PyTypeObject *metaclass) { /* Support weak references (needed for the keep_alive feature) */ - static PyMemberDef members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(instance, weakrefs), Py_READONLY, nullptr}, - {nullptr, 0, 0, 0, nullptr}}; + static PyMemberDef members[] = {{"__weaklistoffset__", + Py_T_PYSSIZET, + offsetof(instance, weakrefs), + Py_READONLY | PYBIND11_MEMBER_OFFSET_FLAGS, + nullptr}, + {nullptr, 0, 0, 0, nullptr}}; static PyType_Slot slots[] = {{Py_tp_new, reinterpret_cast(pybind11_object_new)}, {Py_tp_init, reinterpret_cast(pybind11_object_init)}, @@ -806,7 +841,7 @@ PYBIND11_INLINE PyObject *make_object_base_type(PyTypeObject *metaclass) { {Py_tp_members, reinterpret_cast(members)}, {0, nullptr}}; static PyType_Spec spec = {PYBIND11_DUMMY_MODULE_NAME ".pybind11_object", - static_cast(sizeof(instance)), + PYBIND11_TYPE_DATA_SIZE(instance), 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, slots}; diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 416849d8fc..c30a36d3d0 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -107,7 +107,7 @@ void traverse_offset_bases(void *valueptr, instance *self, bool (*f)(void * /*parentptr*/, instance * /*self*/)); -#ifdef Py_GIL_DISABLED +#if defined(Py_GIL_DISABLED) && !defined(PYBIND11_OPAQUE_PYOBJECT) void enable_try_inc_ref(PyObject *obj); #endif @@ -127,6 +127,8 @@ void type_free(PyTypeObject *type, PyObject *self); /// `is_*`/`*_function` functions are also declared in pytypes.h. PyTypeObject *get_bound_method_type(); PyTypeObject *get_instancemethod_type(); +struct instancemethod_object; +instancemethod_object *instancemethod_data(PyObject *self); extern "C" PyObject *instancemethod_descr_get(PyObject *self, PyObject *obj, PyObject *type); extern "C" PyObject *instancemethod_call(PyObject *self, PyObject *args, PyObject *kwargs); extern "C" PyObject *instancemethod_getattro(PyObject *self, PyObject *name); diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 91296a4e91..ea9ca1042a 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -25,7 +25,11 @@ "pybind11 requires Py_LIMITED_API >= 0x030C0000 and CPython >= 3.12 for the stable ABI." # endif # if defined(Py_GIL_DISABLED) -# error "The free-threaded build has no stable ABI; do not define Py_LIMITED_API." +# if Py_LIMITED_API + 0 < 0x030F0000 || !defined(_Py_OPAQUE_PYOBJECT) +# error "The free-threaded stable ABI (abi3t) needs Py_LIMITED_API >= 0x030F0000." +# endif +// PEP 803: PyObject and PyModuleDef are incomplete types; see docs/advanced/stable_abi.rst. +# define PYBIND11_OPAQUE_PYOBJECT # endif # if !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) # define PYBIND11_SIMPLE_GIL_MANAGEMENT // gil.h reads thread-state internals @@ -44,6 +48,16 @@ # endif #endif +// Heap types declare their extra data as PEP 697 type data when PyObject is opaque: a negative +// PyType_Spec::basicsize and member offsets relative to that data. +#if defined(PYBIND11_OPAQUE_PYOBJECT) +# define PYBIND11_TYPE_DATA_SIZE(T) (-static_cast(sizeof(T))) +# define PYBIND11_MEMBER_OFFSET_FLAGS Py_RELATIVE_OFFSET +#else +# define PYBIND11_TYPE_DATA_SIZE(T) (static_cast(sizeof(T))) +# define PYBIND11_MEMBER_OFFSET_FLAGS 0 +#endif + // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // See also: https://github.com/python/cpython/blob/HEAD/Include/patchlevel.h /* -- start version constants -- */ @@ -539,29 +553,49 @@ Note that this is run once for each (sub-)interpreter the module is imported int possibly concurrently. The PyModuleDef is allowed to be static, but the PyObject* resulting from PyModuleDef_Init should be treated like any other PyObject (so not shared across interpreters). */ -#define PYBIND11_MODULE_PYINIT(name, ...) \ - static int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject *); \ - PYBIND11_PLUGIN_IMPL(name) { \ - PYBIND11_CHECK_PYTHON_VERSION \ - PYBIND11_PRECOMPILED_CONFIG_GUARD \ - try { \ - static ::pybind11::detail::slots_array mod_def_slots \ - = ::pybind11::detail::init_slots( \ - &PYBIND11_CONCAT(pybind11_exec_, name), ##__VA_ARGS__); \ - static PyModuleDef def{/* m_base */ PyModuleDef_HEAD_INIT, \ - /* m_name */ PYBIND11_TOSTRING(name), \ - /* m_doc */ nullptr, \ - /* m_size */ 0, \ - /* m_methods */ nullptr, \ - /* m_slots */ mod_def_slots.data(), \ - /* m_traverse */ nullptr, \ - /* m_clear */ nullptr, \ - /* m_free */ nullptr}; \ - return PyModuleDef_Init(&def); \ - } \ - PYBIND11_CATCH_INIT_EXCEPTIONS \ - return nullptr; \ - } +#if defined(PYBIND11_OPAQUE_PYOBJECT) +// PEP 793 export hook: PyModuleDef is an incomplete type, so the module is described by a static +// PySlot array instead. +# define PYBIND11_MODULE_PYINIT(name, ...) \ + static int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject *); \ + extern "C" PYBIND11_MAYBE_UNUSED PYBIND11_EXPORT PySlot *PyModExport_##name(void); \ + extern "C" PYBIND11_EXPORT PySlot *PyModExport_##name(void) { \ + PYBIND11_CHECK_PYTHON_VERSION \ + PYBIND11_PRECOMPILED_CONFIG_GUARD \ + try { \ + static ::pybind11::detail::export_slots_array mod_slots \ + = ::pybind11::detail::init_export_slots( \ + PYBIND11_TOSTRING(name), &PYBIND11_CONCAT(pybind11_exec_, name), ##__VA_ARGS__); \ + return mod_slots.data(); \ + } \ + PYBIND11_CATCH_INIT_EXCEPTIONS \ + return nullptr; \ + } +#else +# define PYBIND11_MODULE_PYINIT(name, ...) \ + static int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject *); \ + PYBIND11_PLUGIN_IMPL(name) { \ + PYBIND11_CHECK_PYTHON_VERSION \ + PYBIND11_PRECOMPILED_CONFIG_GUARD \ + try { \ + static ::pybind11::detail::slots_array mod_def_slots \ + = ::pybind11::detail::init_slots( \ + &PYBIND11_CONCAT(pybind11_exec_, name), ##__VA_ARGS__); \ + static PyModuleDef def{/* m_base */ PyModuleDef_HEAD_INIT, \ + /* m_name */ PYBIND11_TOSTRING(name), \ + /* m_doc */ nullptr, \ + /* m_size */ 0, \ + /* m_methods */ nullptr, \ + /* m_slots */ mod_def_slots.data(), \ + /* m_traverse */ nullptr, \ + /* m_clear */ nullptr, \ + /* m_free */ nullptr}; \ + return PyModuleDef_Init(&def); \ + } \ + PYBIND11_CATCH_INIT_EXCEPTIONS \ + return nullptr; \ + } +#endif #define PYBIND11_MODULE_EXEC(name, variable) \ static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &); \ @@ -720,7 +754,9 @@ struct nonsimple_values_and_holders { /// The 'instance' type which needs to be standard layout (need to be able to use 'offsetof') struct instance { +#if !defined(PYBIND11_OPAQUE_PYOBJECT) PyObject_HEAD +#endif /// Storage for pointers and holder; see simple_layout, below, for a description union { void *simple_value_holder[1 + instance_simple_holder_in_ptrs()]; @@ -770,6 +806,10 @@ struct instance { /// `old_style_placement_new` in `docs/upgrade.rst` and referenced from /// `docs/advanced/classes.rst`. bool old_style_init_active : 1; +#if defined(PYBIND11_OPAQUE_PYOBJECT) + /// Weak reference used by the instance registry: the limited API has no PyUnstable_TryIncRef. + PyObject *registry_weakref; +#endif /// Initializes all of the above type/values/holders data (but not the instance values /// themselves) @@ -792,6 +832,20 @@ struct instance { static_assert(std::is_standard_layout::value, "Internal error: `pybind11::detail::instance` is not standard layout!"); +#if defined(PYBIND11_OPAQUE_PYOBJECT) +/// Offset of the `instance` data (PEP 697 type data) from the start of the Python object. +Py_ssize_t instance_data_offset(); +inline instance *get_instance(PyObject *obj) { + return reinterpret_cast(reinterpret_cast(obj) + instance_data_offset()); +} +inline PyObject *instance_object(instance *inst) { + return reinterpret_cast(reinterpret_cast(inst) - instance_data_offset()); +} +#else +inline instance *get_instance(PyObject *obj) { return reinterpret_cast(obj); } +inline PyObject *instance_object(instance *inst) { return reinterpret_cast(inst); } +#endif + // Some older compilers (e.g. gcc 9.4.0) require // static_assert(always_false::value, "..."); // instead of diff --git a/include/pybind11/detail/function_record_pyobject.h b/include/pybind11/detail/function_record_pyobject.h index 38ee61d5f0..113054b3b6 100644 --- a/include/pybind11/detail/function_record_pyobject.h +++ b/include/pybind11/detail/function_record_pyobject.h @@ -19,10 +19,20 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) struct function_record_PyObject { +#if !defined(PYBIND11_OPAQUE_PYOBJECT) PyObject_HEAD +#endif function_record *cpp_func_rec; }; +inline function_record_PyObject *function_record_data(PyObject *obj) { +#if defined(PYBIND11_OPAQUE_PYOBJECT) + return static_cast(PyObject_GetTypeData(obj, Py_TYPE(obj))); +#else + return reinterpret_cast(obj); +#endif +} + PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods) PyObject *tp_new_impl(PyTypeObject *type, PyObject *args, PyObject *kwds); @@ -84,7 +94,7 @@ static PyType_Slot function_record_PyType_Slots[] = { static PyType_Spec function_record_PyType_Spec = {function_record_PyTypeObject_methods::tp_qualname_impl, - sizeof(function_record_PyObject), + PYBIND11_TYPE_DATA_SIZE(function_record_PyObject), 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE, function_record_PyType_Slots}; @@ -124,18 +134,18 @@ inline bool is_function_record_PyObject(PyObject *obj) { inline function_record *function_record_ptr_from_PyObject(PyObject *obj) { if (is_function_record_PyObject(obj)) { - return (reinterpret_cast(obj))->cpp_func_rec; + return function_record_data(obj)->cpp_func_rec; } return nullptr; } inline object function_record_PyObject_New() { - auto *py_func_rec = PyObject_New(function_record_PyObject, get_function_record_PyTypeObject()); + PyObject *py_func_rec = PyObject_New(PyObject, get_function_record_PyTypeObject()); if (py_func_rec == nullptr) { throw error_already_set(); } - py_func_rec->cpp_func_rec = nullptr; // For clarity/purity. Redundant in practice. - return reinterpret_steal(reinterpret_cast(py_func_rec)); + function_record_data(py_func_rec)->cpp_func_rec = nullptr; // For clarity. Redundant. + return reinterpret_steal(py_func_rec); } PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods) diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index af23890d57..9c7c6ee258 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -312,7 +312,7 @@ struct constructor { cl.def( "__init__", [](value_and_holder &v_h, Args... args) { - if (Py_TYPE(reinterpret_cast(v_h.inst)) == v_h.type->type) { + if (Py_TYPE(instance_object(v_h.inst)) == v_h.type->type) { v_h.value_ptr() = construct_or_initialize>(std::forward(args)...); } else { @@ -401,8 +401,7 @@ struct factory { (value_and_holder &v_h, Args... args) { construct(v_h, func(std::forward(args)...), - Py_TYPE(reinterpret_cast(v_h.inst)) - != v_h.type->type); + Py_TYPE(instance_object(v_h.inst)) != v_h.type->type); }, is_new_style_constructor(), extra...); @@ -449,7 +448,7 @@ struct factory { [class_func, alias_func] #endif (value_and_holder &v_h, CArgs... args) { - if (Py_TYPE(reinterpret_cast(v_h.inst)) == v_h.type->type) { + if (Py_TYPE(instance_object(v_h.inst)) == v_h.type->type) { // If the instance type equals the registered type we don't have inheritance, // so don't need the alias and can construct using the class function: construct(v_h, class_func(std::forward(args)...), false); @@ -482,9 +481,9 @@ void setstate(value_and_holder &v_h, std::pair &&result, bool need_alias) return; } // Our tests never run into an unset dict, but being careful here for now (see #5658) - auto dict = getattr(reinterpret_cast(v_h.inst), "__dict__", none()); + auto dict = getattr(instance_object(v_h.inst), "__dict__", none()); if (dict.is_none()) { - setattr(reinterpret_cast(v_h.inst), "__dict__", d); + setattr(instance_object(v_h.inst), "__dict__", d); } else { // Keep the original object dict and just update it if (PyDict_Update(dict.ptr(), d.ptr()) < 0) { @@ -539,7 +538,7 @@ struct pickle_factory { (value_and_holder &v_h, ArgState state) { setstate(v_h, func(std::forward(state)), - Py_TYPE(reinterpret_cast(v_h.inst)) != v_h.type->type); + Py_TYPE(instance_object(v_h.inst)) != v_h.type->type); }, is_new_style_constructor(), extra...); diff --git a/include/pybind11/detail/internals-inl.h b/include/pybind11/detail/internals-inl.h index a7d6450ee7..66031e534e 100644 --- a/include/pybind11/detail/internals-inl.h +++ b/include/pybind11/detail/internals-inl.h @@ -246,7 +246,7 @@ PYBIND11_INLINE size_t num_registered_instances() { size_t count = 0; for (size_t i = 0; i <= internals.instance_shards_mask; ++i) { auto &shard = internals.instance_shards[i]; - std::unique_lock lock(shard.mutex); + pycritical_section lock(shard.mutex); count += shard.registered_instances.size(); } return count; diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index e01b6b2c92..455feb53a5 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -229,10 +229,9 @@ void translate_exception(std::exception_ptr p); inline PyThreadState *get_thread_state_unchecked() { #if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) return PyThreadState_GET(); -#elif defined(Py_LIMITED_API) && Py_LIMITED_API < 0x030D0000 - // The stable ABI has no "current thread state or null" function before 3.13. All callers - // left under the stable ABI (the interpreter state dict lookup) hold the GIL. - return PyThreadState_Get(); +#elif defined(Py_LIMITED_API) + // PyThreadState_GetUnchecked() is not part of the stable ABI. + return PyGILState_GetThisThreadState(); #elif PY_VERSION_HEX < 0x030D0000 && !defined(Py_LIMITED_API) return _PyThreadState_UncheckedGet(); #else @@ -299,7 +298,34 @@ struct override_hash { using instance_map = std::unordered_multimap; -#ifdef Py_GIL_DISABLED +#if defined(PYBIND11_OPAQUE_PYOBJECT) +// PyMutex is not part of the stable ABI, but critical sections are (3.15+). The lock is the +// per-object mutex of a plain `object()`, which is kept for the lifetime of the process. +class pymutex { + friend class pycritical_section; + PyObject *lock_object; + +public: + pymutex() + : lock_object(PyObject_CallNoArgs(reinterpret_cast(&PyBaseObject_Type))) { + if (lock_object == nullptr) { + throw error_already_set(); + } + } +}; + +class pycritical_section { + PyCriticalSection cs{}; + +public: + explicit pycritical_section(pymutex &m) { PyCriticalSection_Begin(&cs, m.lock_object); } + ~pycritical_section() { PyCriticalSection_End(&cs); } + pycritical_section(const pycritical_section &) = delete; + pycritical_section &operator=(const pycritical_section &) = delete; + pycritical_section(pycritical_section &&) = delete; + pycritical_section &operator=(pycritical_section &&) = delete; +}; +#elif defined(Py_GIL_DISABLED) // Wrapper around PyMutex to provide BasicLockable semantics class pymutex { friend class pycritical_section; @@ -341,7 +367,9 @@ class pycritical_section { pycritical_section(pycritical_section &&) = delete; pycritical_section &operator=(pycritical_section &&) = delete; }; +#endif +#ifdef Py_GIL_DISABLED // Instance map shards are used to reduce mutex contention in free-threaded Python. struct instance_map_shard { instance_map registered_instances; @@ -537,7 +565,9 @@ struct native_enum_record { // regular modules in the same process. The tag keeps the two universes apart. It is deliberately // not part of PYBIND11_PLATFORM_ABI_ID: the C++ ABI is the same, so the cpp_conduit protocol // still bridges the two. -#if defined(Py_LIMITED_API) +#if defined(PYBIND11_OPAQUE_PYOBJECT) +# define PYBIND11_INTERNALS_SABI_TAG "_stable_ft" // abi3t modules also load on GIL builds +#elif defined(Py_LIMITED_API) # define PYBIND11_INTERNALS_SABI_TAG "_stable" #else # define PYBIND11_INTERNALS_SABI_TAG "" @@ -940,7 +970,7 @@ inline auto with_instance_map(const void *ptr, const F &cb) auto idx = static_cast(hash & internals.instance_shards_mask); auto &shard = internals.instance_shards[idx]; - std::unique_lock lock(shard.mutex); + pycritical_section lock(shard.mutex); return cb(shard.registered_instances); #else (void) ptr; diff --git a/include/pybind11/detail/type_caster_base-inl.h b/include/pybind11/detail/type_caster_base-inl.h index 31532b65cb..83663ee11e 100644 --- a/include/pybind11/detail/type_caster_base-inl.h +++ b/include/pybind11/detail/type_caster_base-inl.h @@ -240,7 +240,12 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle get_type_handle(const std::type_in PYBIND11_INLINE bool try_incref(PyObject *obj) { // Tries to increment the reference count of an object if it's not zero. -#if defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030E00A4 +#if defined(PYBIND11_OPAQUE_PYOBJECT) + // Not used: the registry goes through PyWeakref_GetRef() instead. + assert(Py_REFCNT(obj) > 0); + Py_INCREF(obj); + return true; +#elif defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030E00A4 return PyUnstable_TryIncRef(obj); #elif defined(Py_GIL_DISABLED) // See @@ -287,12 +292,20 @@ find_registered_python_instance(void *src, const detail::type_info *tinfo) { auto it_instances = instances.equal_range(src); for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { for (auto *instance_type : - detail::all_type_info(Py_TYPE(reinterpret_cast(it_i->second)))) { + detail::all_type_info(Py_TYPE(instance_object(it_i->second)))) { if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) { - auto *wrapper = reinterpret_cast(it_i->second); +#if defined(PYBIND11_OPAQUE_PYOBJECT) + PyObject *wrapper = nullptr; + if (it_i->second->registry_weakref != nullptr + && PyWeakref_GetRef(it_i->second->registry_weakref, &wrapper) == 1) { + return handle(wrapper); + } +#else + auto *wrapper = instance_object(it_i->second); if (try_incref(wrapper)) { return handle(wrapper); } +#endif } } } @@ -304,7 +317,7 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE value_and_holder instance::get_value_and_holder(const type_info *find_type /*= nullptr default in common.h*/, bool throw_if_missing /*= true in common.h*/) { // Optimize common case: - if (!find_type || Py_TYPE(reinterpret_cast(this)) == find_type->type) { + if (!find_type || Py_TYPE(instance_object(this)) == find_type->type) { return value_and_holder(this, find_type, 0, 0); } @@ -319,10 +332,10 @@ instance::get_value_and_holder(const type_info *find_type /*= nullptr default in } #if defined(PYBIND11_DETAILED_ERROR_MESSAGES) - pybind11_fail( - "pybind11::detail::instance::get_value_and_holder: `" - + get_fully_qualified_tp_name(find_type->type) + "' is not a pybind11 base of the given `" - + get_fully_qualified_tp_name(Py_TYPE(reinterpret_cast(this))) + "' instance"); + pybind11_fail("pybind11::detail::instance::get_value_and_holder: `" + + get_fully_qualified_tp_name(find_type->type) + + "' is not a pybind11 base of the given `" + + get_fully_qualified_tp_name(Py_TYPE(instance_object(this))) + "' instance"); #else pybind11_fail( "pybind11::detail::instance::get_value_and_holder: " @@ -332,7 +345,7 @@ instance::get_value_and_holder(const type_info *find_type /*= nullptr default in } PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void instance::allocate_layout() { - const auto &tinfo = all_type_info(Py_TYPE(reinterpret_cast(this))); + const auto &tinfo = all_type_info(Py_TYPE(instance_object(this))); const size_t n_types = tinfo.size(); @@ -403,7 +416,7 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle get_object_handle(const void *ptr, for (auto it = range.first; it != range.second; ++it) { for (const auto &vh : values_and_holders(it->second)) { if (vh.type == type) { - return handle(reinterpret_cast(it->second)); + return handle(instance_object(it->second)); } } } @@ -484,7 +497,7 @@ type_caster_generic::cast(const cast_sources &srcs, } auto inst = reinterpret_steal(make_new_instance(tinfo->type)); - auto *wrapper = reinterpret_cast(inst.ptr()); + auto *wrapper = get_instance(inst.ptr()); wrapper->owned = false; void *&valueptr = values_and_holders(wrapper).begin()->value_ptr(); diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 15b04604db..ae0c630828 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -139,12 +139,12 @@ struct values_and_holders { public: explicit values_and_holders(instance *inst) - : inst{inst}, tinfo(all_type_info(Py_TYPE(reinterpret_cast(inst)))) {} + : inst{inst}, tinfo(all_type_info(Py_TYPE(instance_object(inst)))) {} explicit values_and_holders(PyObject *obj) : inst{nullptr}, tinfo(all_type_info(Py_TYPE(obj))) { if (!tinfo.empty()) { - inst = reinterpret_cast(obj); + inst = get_instance(obj); } } @@ -430,7 +430,7 @@ handle smart_holder_from_unique_ptr(std::unique_ptr &&src, } auto inst = reinterpret_steal(make_new_instance(tinfo->type)); - auto *inst_raw_ptr = reinterpret_cast(inst.ptr()); + auto *inst_raw_ptr = get_instance(inst.ptr()); inst_raw_ptr->owned = true; void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr(); valueptr = src_raw_void_ptr; @@ -498,7 +498,7 @@ handle smart_holder_from_shared_ptr(const std::shared_ptr &src, } auto inst = reinterpret_steal(make_new_instance(tinfo->type)); - auto *inst_raw_ptr = reinterpret_cast(inst.ptr()); + auto *inst_raw_ptr = get_instance(inst.ptr()); inst_raw_ptr->owned = true; void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr(); valueptr = src_raw_void_ptr; @@ -539,7 +539,7 @@ struct shared_ptr_parent_life_support { struct shared_ptr_trampoline_self_life_support { PyObject *self; explicit shared_ptr_trampoline_self_life_support(instance *inst) - : self{reinterpret_cast(inst)} { + : self{instance_object(inst)} { gil_scoped_acquire gil; Py_INCREF(self); } @@ -578,7 +578,7 @@ struct load_helper : value_and_holder_helper { void maybe_set_python_instance_is_alias(handle src) { if (was_populated) { - python_instance_is_alias = reinterpret_cast(src.ptr())->is_alias; + python_instance_is_alias = get_instance(src.ptr())->is_alias; } } @@ -842,7 +842,7 @@ class type_caster_generic { // Case 1: If src is an exact type match for the target type then we can reinterpret_cast // the instance's value pointer to the target type: if (srctype == typeinfo->type) { - this_.load_value(reinterpret_cast(src.ptr())->get_value_and_holder()); + this_.load_value(get_instance(src.ptr())->get_value_and_holder()); return true; } // Case 2: We have a derived class @@ -857,7 +857,7 @@ class type_caster_generic { // is extremely common, we handle it specially to avoid the loop iterator and type // pointer lookup overhead) if (bases.size() == 1 && (no_cpp_mi || bases.front()->type == typeinfo->type)) { - this_.load_value(reinterpret_cast(src.ptr())->get_value_and_holder()); + this_.load_value(get_instance(src.ptr())->get_value_and_holder()); return true; } // Case 2b: the python type inherits from multiple C++ bases. Check the bases to see @@ -867,8 +867,7 @@ class type_caster_generic { for (auto *base : bases) { if (no_cpp_mi ? PyType_IsSubtype(base->type, typeinfo->type) : base->type == typeinfo->type) { - this_.load_value( - reinterpret_cast(src.ptr())->get_value_and_holder(base)); + this_.load_value(get_instance(src.ptr())->get_value_and_holder(base)); return true; } } diff --git a/include/pybind11/detail/typeid.h b/include/pybind11/detail/typeid.h index c8160ad135..bcfe32305d 100644 --- a/include/pybind11/detail/typeid.h +++ b/include/pybind11/detail/typeid.h @@ -54,6 +54,14 @@ std::string type_id() { return detail::clean_type_id(typeid(T).name()); } +#if defined(PYBIND11_OPAQUE_PYOBJECT) +// typeid() needs a complete type; PyObject is opaque under the abi3t stable ABI. +template <> +inline std::string type_id() { + return "PyObject"; +} +#endif + PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) #ifndef PYBIND11_PRECOMPILED diff --git a/include/pybind11/detail/value_and_holder.h b/include/pybind11/detail/value_and_holder.h index b24551e678..98ab73c378 100644 --- a/include/pybind11/detail/value_and_holder.h +++ b/include/pybind11/detail/value_and_holder.h @@ -84,7 +84,7 @@ struct value_and_holder { // for `tp_traverse` and `tp_clear` implementations. // WARNING: The caller is responsible for ensuring that the `reinterpret_cast` is valid. inline bool is_holder_constructed(PyObject *obj) { - auto *const instance = reinterpret_cast(obj); + auto *const instance = get_instance(obj); return instance->get_value_and_holder().holder_constructed(); } diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index a26ac2b4cd..43393f97de 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -10,6 +10,10 @@ #pragma once #include "pybind11.h" + +#if defined(PYBIND11_OPAQUE_PYOBJECT) +# error "numpy.h mirrors NumPy object layouts; it is not available with the abi3t stable ABI." +#endif #include "detail/common.h" #include "complex.h" #include "gil_safe_call_once.h" diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index adc85795b7..062d501e48 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -199,7 +199,7 @@ PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods) PYBIND11_INLINE void tp_dealloc_impl(PyObject *self) { // Save type before PyObject_Free invalidates self. auto *type = Py_TYPE(self); - auto *py_func_rec = reinterpret_cast(self); + auto *py_func_rec = function_record_data(self); cpp_function::destruct(py_func_rec->cpp_func_rec); py_func_rec->cpp_func_rec = nullptr; // PyObject_New increments the heap type refcount and allocates via @@ -665,8 +665,7 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u rec->def->ml_flags = METH_FASTCALL | METH_KEYWORDS; object py_func_rec = detail::function_record_PyObject_New(); - (reinterpret_cast(py_func_rec.ptr()))->cpp_func_rec - = unique_rec.release(); + detail::function_record_data(py_func_rec.ptr())->cpp_func_rec = unique_rec.release(); guarded_strdup.release(); object scope_module = detail::get_scope_module(rec->scope); @@ -699,8 +698,7 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u // chain. chain_start = rec; rec->next = chain; - auto *py_func_rec - = reinterpret_cast(PyCFunction_GetSelf(m_ptr)); + auto *py_func_rec = detail::function_record_data(PyCFunction_GetSelf(m_ptr)); py_func_rec->cpp_func_rec = unique_rec.release(); guarded_strdup.release(); } else { @@ -862,7 +860,7 @@ PYBIND11_INLINE PyObject *cpp_function::dispatcher(PyObject *self, auto *const tinfo = get_type_info(reinterpret_cast(overloads->scope.ptr())); - auto *const pi = reinterpret_cast(parent.ptr()); + auto *const pi = get_instance(parent.ptr()); self_value_and_holder = pi->get_value_and_holder(tinfo, true); // If this value is already registered it must mean __init__ is invoked multiple times; @@ -1293,7 +1291,7 @@ PYBIND11_INLINE PyObject *cpp_function::dispatcher(PyObject *self, return nullptr; } if (overloads->is_constructor && !self_value_and_holder.holder_constructed()) { - auto *pi = reinterpret_cast(parent.ptr()); + auto *pi = get_instance(parent.ptr()); self_value_and_holder.type->init_instance(pi, nullptr); } return result.ptr(); @@ -1606,6 +1604,7 @@ module_::add_object(const char *name, handle obj, bool overwrite) { PyModule_AddObject(ptr(), name, obj.inc_ref().ptr() /* steals a reference */); } +#if !defined(PYBIND11_OPAQUE_PYOBJECT) PYBIND11_INLINE module_ module_::create_extension_module(const char *name, const char *doc, PyModuleDef *def, @@ -1628,14 +1627,15 @@ PYBIND11_INLINE module_ module_::create_extension_module(const char *name, pybind11_fail("Internal error in module_::create_extension_module()"); } if (gil_not_used.flag()) { -#ifdef Py_GIL_DISABLED +# ifdef Py_GIL_DISABLED PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); -#endif +# endif } // TODO: Should be reinterpret_steal for Python 3, but Python also steals it again when // returned from PyInit_... // For Python 2, reinterpret_borrow was correct. return reinterpret_borrow(m); } +#endif PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 9a1a4834d3..312cecfb89 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -720,6 +720,41 @@ inline slots_array init_slots(int (*exec_fn)(PyObject *), Options &&...options) return mod_def_slots; } +#if defined(PYBIND11_OPAQUE_PYOBJECT) +/// PEP 793 slot array; large enough for every slot below plus the zero sentinel. +using export_slots_array = std::array; + +template +inline export_slots_array +init_export_slots(const char *name, int (*exec_fn)(PyObject *), Options &&...options) noexcept { + export_slots_array slots{}; + size_t next_slot = 0; + auto data_slot = [&](int id, const void *value, uint16_t flags) { + slots[next_slot].sl_id = static_cast(id); + slots[next_slot].sl_flags = flags; + slots[next_slot].sl_ptr = const_cast(value); + ++next_slot; + }; + auto func_slot = [&](int id, _Py_funcptr_t fn) { + slots[next_slot].sl_id = static_cast(id); + slots[next_slot].sl_func = fn; + ++next_slot; + }; + PyABIInfo_VAR(abi_info); + data_slot(Py_mod_name, name, PySlot_INTPTR); + data_slot(Py_mod_abi, &abi_info, PySlot_STATIC); + func_slot(Py_mod_create, reinterpret_cast<_Py_funcptr_t>(&cached_create_module)); + if (exec_fn != nullptr) { + func_slot(Py_mod_exec, reinterpret_cast<_Py_funcptr_t>(exec_fn)); + } + data_slot(Py_mod_multiple_interpreters, multi_interp_slot(options...), PySlot_INTPTR); + if (gil_not_used_option(options...)) { + data_slot(Py_mod_gil, Py_MOD_GIL_NOT_USED, PySlot_INTPTR); + } + return slots; +} +#endif + PYBIND11_NAMESPACE_END(detail) /// Wrapper for Python extension modules @@ -727,11 +762,13 @@ class module_ : public object { public: PYBIND11_OBJECT_DEFAULT(module_, object, PyModule_Check) +#if !defined(PYBIND11_OPAQUE_PYOBJECT) /// Create a new top-level Python module with the given name and docstring PYBIND11_DEPRECATED("Use PYBIND11_MODULE or module_::create_extension_module instead") explicit module_(const char *name, const char *doc = nullptr) { *this = create_extension_module(name, doc, new PyModuleDef()); } +#endif /** \rst Create Python binding for a new function within the module scope. ``Func`` @@ -782,6 +819,7 @@ class module_ : public object { // DEPRECATED (since PR #5688): Use PyModuleDef directly instead. using module_def = PyModuleDef; +#if !defined(PYBIND11_OPAQUE_PYOBJECT) /** \rst Create a new top-level module that can be used as the main module of a C extension. @@ -791,6 +829,7 @@ class module_ : public object { const char *doc, PyModuleDef *def, mod_gil_not_used gil_not_used = mod_gil_used()); +#endif }; PYBIND11_NAMESPACE_BEGIN(detail) @@ -2056,7 +2095,7 @@ class enum_ : public class_ { [](detail::value_and_holder &v_h, Scalar arg) { detail::initimpl::setstate(v_h, static_cast(arg), - Py_TYPE(reinterpret_cast(v_h.inst)) + Py_TYPE(instance_object(v_h.inst)) != v_h.type->type); }, detail::is_new_style_constructor(), diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 8a440170e6..2945d2f037 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -73,7 +73,14 @@ using tuple_accessor = accessor; /// Tag and check to identify a class which implements the Python object API class pyobject_tag {}; template -using is_pyobject = std::is_base_of>; +struct is_pyobject : std::is_base_of> {}; +#if defined(PYBIND11_OPAQUE_PYOBJECT) +// PyObject is an incomplete type under the abi3t stable ABI; type traits cannot inspect it. +template <> +struct is_pyobject : std::false_type {}; +template <> +struct is_pyobject : std::false_type {}; +#endif /** \rst A mixin class which adds common functions to `handle`, `object` and various accessors. diff --git a/include/pybind11/trampoline_self_life_support.h b/include/pybind11/trampoline_self_life_support.h index a3c14bbd86..281421440c 100644 --- a/include/pybind11/trampoline_self_life_support.h +++ b/include/pybind11/trampoline_self_life_support.h @@ -25,12 +25,12 @@ struct trampoline_self_life_support { trampoline_self_life_support() = default; void activate_life_support(const detail::value_and_holder &v_h_) { - Py_INCREF((PyObject *) v_h_.inst); + Py_INCREF(instance_object(v_h_.inst)); v_h = v_h_; } void deactivate_life_support() { - Py_DECREF((PyObject *) v_h.inst); + Py_DECREF(instance_object(v_h.inst)); v_h = detail::value_and_holder(); } @@ -42,7 +42,7 @@ struct trampoline_self_life_support { v_h.value_ptr() = nullptr; v_h.holder().release_disowned(); detail::deregister_instance(v_h.inst, value_void_ptr, v_h.type); - Py_DECREF((PyObject *) v_h.inst); // Must be after deregister. + Py_DECREF(instance_object(v_h.inst)); // Must be after deregister. PyGILState_Release(threadstate); } } diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index d9aecae352..bc0506e236 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -89,12 +89,15 @@ def _limited_api_hex(value: Any) -> str: """ - The Py_LIMITED_API value for ``py_limited_api``: True selects 3.12 (the - minimum pybind11 supports), a string such as "3.13" or "cp313" selects - that version, and an int or hex string is passed through. + The Py_LIMITED_API value for ``py_limited_api``: True selects the minimum + pybind11 supports (3.12, or 3.15 on free-threaded Python, where the stable + ABI is abi3t), a string such as "3.13" or "cp313" selects that version, and + an int or hex string is passed through. """ + free_threaded = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + minimum = (3, 15) if free_threaded else (3, 12) if value is True: - return "0x030C0000" + return f"0x{minimum[0]:02X}{minimum[1]:02X}0000" if isinstance(value, int): return f"0x{value:08X}" text = str(value) @@ -105,8 +108,13 @@ def _limited_api_hex(value: Any) -> str: msg = f"py_limited_api must be True or a Python version such as '3.12', got {value!r}" raise ValueError(msg) major, minor = (int(part) for part in match.groups()) - if (major, minor) < (3, 12): - msg = f"pybind11 supports the stable ABI from Python 3.12 on, got {value!r}" + if (major, minor) < minimum: + what = ( + "the free-threaded stable ABI (abi3t)" + if free_threaded + else "the stable ABI" + ) + msg = f"pybind11 supports {what} from Python {minimum[0]}.{minimum[1]} on, got {value!r}" raise ValueError(msg) return f"0x{major:02X}{minor:02X}0000" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 58e80b83f5..7296bec3bc 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -236,6 +236,19 @@ if(PYBIND11_TEST_STABLE_ABI) MESSAGE "Skipping chrono, custom_type_setup, subinterpreter and PyGILState_Check tests under the stable ABI" ) + # numpy.h mirrors NumPy object layouts, which the abi3t stable ABI hides (PEP 803). + if("${PYTHON_MODULE_EXTENSION}" MATCHES "^\\.cpython-[0-9]+t") + pybind11_filter_tests( + PYBIND11_TEST_FILES + test_numpy_array.cpp + test_numpy_dtypes.cpp + test_numpy_scalars.cpp + test_numpy_vectorize.cpp + test_eigen_matrix.cpp + test_eigen_tensor.cpp + MESSAGE + "Skipping numpy and eigen tests under the free-threaded stable ABI (abi3t)") + endif() endif() # Skip tests for CUDA check: diff --git a/tests/cross_module_gil_utils.cpp b/tests/cross_module_gil_utils.cpp index 39112668a4..0d0f877eb3 100644 --- a/tests/cross_module_gil_utils.cpp +++ b/tests/cross_module_gil_utils.cpp @@ -77,35 +77,59 @@ void gil_acquire_nested() { constexpr char kModuleName[] = "cross_module_gil_utils"; +#if !defined(PYBIND11_OPAQUE_PYOBJECT) struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, kModuleName, nullptr, 0, nullptr, nullptr, nullptr, nullptr, nullptr}; +#endif } // namespace #define ADD_FUNCTION(Name, ...) \ PyModule_AddObject(m, Name, PyLong_FromVoidPtr(reinterpret_cast(&__VA_ARGS__))); +static int cross_module_gil_utils_exec(PyObject *m) { + static_assert(sizeof(&gil_acquire) == sizeof(void *), + "Function pointer must have the same size as void*"); + ADD_FUNCTION("gil_acquire_funcaddr", gil_acquire) + ADD_FUNCTION("gil_multi_acquire_release_funcaddr", gil_multi_acquire_release) + ADD_FUNCTION("gil_acquire_inner_custom_funcaddr", + gil_acquire_inner) + ADD_FUNCTION("gil_acquire_nested_custom_funcaddr", + gil_acquire_nested) + ADD_FUNCTION("gil_acquire_inner_pybind11_funcaddr", + gil_acquire_inner) + ADD_FUNCTION("gil_acquire_nested_pybind11_funcaddr", + gil_acquire_nested) + return 0; +} + +#if defined(PYBIND11_OPAQUE_PYOBJECT) +// PEP 793 export hook: PyModuleDef is an incomplete type under the abi3t stable ABI. +extern "C" PYBIND11_EXPORT PySlot *PyModExport_cross_module_gil_utils() { + PyABIInfo_VAR(abi_info); + static PySlot slots[] + = {PySlot_PTR(Py_mod_name, kModuleName), + {Py_mod_exec, 0, {0}, {reinterpret_cast(&cross_module_gil_utils_exec)}}, + PySlot_PTR(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_PTR_STATIC(Py_mod_abi, &abi_info), + {0, 0, {0}, {nullptr}}}; + return slots; +} +#else extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_gil_utils() { PyObject *m = PyModule_Create(&moduledef); if (m != nullptr) { - static_assert(sizeof(&gil_acquire) == sizeof(void *), - "Function pointer must have the same size as void*"); -#ifdef Py_GIL_DISABLED +# ifdef Py_GIL_DISABLED PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); -#endif - ADD_FUNCTION("gil_acquire_funcaddr", gil_acquire) - ADD_FUNCTION("gil_multi_acquire_release_funcaddr", gil_multi_acquire_release) - ADD_FUNCTION("gil_acquire_inner_custom_funcaddr", - gil_acquire_inner) - ADD_FUNCTION("gil_acquire_nested_custom_funcaddr", - gil_acquire_nested) - ADD_FUNCTION("gil_acquire_inner_pybind11_funcaddr", - gil_acquire_inner) - ADD_FUNCTION("gil_acquire_nested_pybind11_funcaddr", - gil_acquire_nested) +# endif + if (cross_module_gil_utils_exec(m) != 0) { + Py_DECREF(m); + return nullptr; + } } return m; } +#endif diff --git a/tests/cross_module_interleaved_error_already_set.cpp b/tests/cross_module_interleaved_error_already_set.cpp index 65a4463b8c..c549df7529 100644 --- a/tests/cross_module_interleaved_error_already_set.cpp +++ b/tests/cross_module_interleaved_error_already_set.cpp @@ -32,23 +32,45 @@ void interleaved_error_already_set() { constexpr char kModuleName[] = "cross_module_interleaved_error_already_set"; +#if !defined(PYBIND11_OPAQUE_PYOBJECT) struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, kModuleName, nullptr, 0, nullptr, nullptr, nullptr, nullptr, nullptr}; +#endif + +int module_exec(PyObject *m) { + static_assert(sizeof(&interleaved_error_already_set) == sizeof(void *), + "Function pointer must have the same size as void *"); + return PyModule_AddObject( + m, + "funcaddr", + PyLong_FromVoidPtr(reinterpret_cast(&interleaved_error_already_set))); +} } // namespace +#if defined(PYBIND11_OPAQUE_PYOBJECT) +// PEP 793 export hook: PyModuleDef is an incomplete type under the abi3t stable ABI. +extern "C" PYBIND11_EXPORT PySlot *PyModExport_cross_module_interleaved_error_already_set() { + PyABIInfo_VAR(abi_info); + static PySlot slots[] = {PySlot_PTR(Py_mod_name, kModuleName), + {Py_mod_exec, 0, {0}, {reinterpret_cast(&module_exec)}}, + PySlot_PTR(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_PTR_STATIC(Py_mod_abi, &abi_info), + {0, 0, {0}, {nullptr}}}; + return slots; +} +#else extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_interleaved_error_already_set() { PyObject *m = PyModule_Create(&moduledef); if (m != nullptr) { - static_assert(sizeof(&interleaved_error_already_set) == sizeof(void *), - "Function pointer must have the same size as void *"); -#ifdef Py_GIL_DISABLED +# ifdef Py_GIL_DISABLED PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); -#endif - PyModule_AddObject( - m, - "funcaddr", - PyLong_FromVoidPtr(reinterpret_cast(&interleaved_error_already_set))); +# endif + if (module_exec(m) != 0) { + Py_DECREF(m); + return nullptr; + } } return m; } +#endif diff --git a/tests/exo_planet_c_api.cpp b/tests/exo_planet_c_api.cpp index 1510107175..75ed042982 100644 --- a/tests/exo_planet_c_api.cpp +++ b/tests/exo_planet_c_api.cpp @@ -47,6 +47,7 @@ PyMethodDef ThisMethodDef[] = {{"GetLuggage", wrapGetLuggage, METH_O, nullptr}, {"GetPoints", wrapGetPoints, METH_O, nullptr}, {nullptr, nullptr, 0, nullptr}}; +#if !defined(_Py_OPAQUE_PYOBJECT) struct PyModuleDef ThisModuleDef = { PyModuleDef_HEAD_INIT, // m_base "exo_planet_c_api", // m_name @@ -58,6 +59,7 @@ struct PyModuleDef ThisModuleDef = { nullptr, // m_clear nullptr // m_free }; +#endif } // namespace @@ -67,6 +69,17 @@ struct PyModuleDef ThisModuleDef = { # define EXO_PLANET_C_API_EXPORT __attribute__((visibility("default"))) #endif +#if defined(_Py_OPAQUE_PYOBJECT) +// PEP 793 export hook: PyModuleDef is an incomplete type under the abi3t stable ABI. +extern "C" EXO_PLANET_C_API_EXPORT PySlot *PyModExport_exo_planet_c_api() { + PyABIInfo_VAR(abi_info); + static PySlot slots[] = {PySlot_PTR(Py_mod_name, "exo_planet_c_api"), + PySlot_PTR_STATIC(Py_mod_methods, ThisMethodDef), + PySlot_PTR_STATIC(Py_mod_abi, &abi_info), + {0, 0, {0}, {nullptr}}}; + return slots; +} +#else extern "C" EXO_PLANET_C_API_EXPORT PyObject *PyInit_exo_planet_c_api() { PyObject *m = PyModule_Create(&ThisModuleDef); if (m == nullptr) { @@ -74,3 +87,4 @@ extern "C" EXO_PLANET_C_API_EXPORT PyObject *PyInit_exo_planet_c_api() { } return m; } +#endif diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py index 9404caaff4..285cf953d0 100644 --- a/tests/extra_setuptools/test_setuphelper.py +++ b/tests/extra_setuptools/test_setuphelper.py @@ -114,8 +114,8 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std): @pytest.mark.skipif( sys.implementation.name != "cpython" or sys.version_info < (3, 12) - or sysconfig.get_config_var("Py_GIL_DISABLED"), - reason="stable ABI needs CPython 3.12+ with the GIL", + or (sysconfig.get_config_var("Py_GIL_DISABLED") and sys.version_info < (3, 15)), + reason="stable ABI needs CPython 3.12+, or 3.15+ when free-threaded (abi3t)", ) def test_stable_abi_setup_py(monkeypatch, tmpdir): monkeypatch.chdir(tmpdir) diff --git a/tests/test_cmake_build/test.py b/tests/test_cmake_build/test.py index 627c8530ea..9e4af55920 100644 --- a/tests/test_cmake_build/test.py +++ b/tests/test_cmake_build/test.py @@ -11,5 +11,5 @@ expect_abi3 = os.environ.get("PYBIND11_EXPECT_ABI3") if expect_abi3: - assert ".abi3." in expect_abi3, f"expected an abi3 module name, got {expect_abi3}" + assert ".abi3" in expect_abi3, f"expected an abi3 module name, got {expect_abi3}" print(f"{sys.argv[1]} imports, runs, and adds: 1 + 2 = 3") diff --git a/tests/test_modules.cpp b/tests/test_modules.cpp index 842a3bc4b8..a70c45f086 100644 --- a/tests/test_modules.cpp +++ b/tests/test_modules.cpp @@ -77,8 +77,13 @@ TEST_SUBMODULE(modules, m) { class Dupe3 {}; class DupeException {}; +#if defined(PYBIND11_OPAQUE_PYOBJECT) + // PyModuleDef is an incomplete type under the abi3t stable ABI. + auto dm = py::reinterpret_steal(PyModule_New("dummy")); +#else // Go ahead and leak, until we have a non-leaking py::module_ constructor auto dm = py::module_::create_extension_module("dummy", nullptr, new PyModuleDef); +#endif auto failures = py::list(); py::class_(dm, "Dupe1"); diff --git a/tests/test_stl_binders.cpp b/tests/test_stl_binders.cpp index f399ec0e43..bad5fe86de 100644 --- a/tests/test_stl_binders.cpp +++ b/tests/test_stl_binders.cpp @@ -7,8 +7,10 @@ BSD-style license that can be found in the LICENSE file. */ -#include #include +#if !defined(PYBIND11_OPAQUE_PYOBJECT) +# include +#endif #include "pybind11_tests.h" @@ -238,10 +240,12 @@ TEST_SUBMODULE(stl_binders, m) { double y; bool z; }; +#if !defined(PYBIND11_OPAQUE_PYOBJECT) // needs numpy.h's generic format_descriptor m.def("create_undeclstruct", [m]() mutable { py::bind_vector>( m, "VectorUndeclStruct", py::buffer_protocol()); }); +#endif // Bind recursive container types py::bind_vector(m, "RecursiveVector"); @@ -253,6 +257,7 @@ TEST_SUBMODULE(stl_binders, m) { py::bind_vector(m, "UserVectorLike"); py::bind_map(m, "UserMapLike"); +#if !defined(PYBIND11_OPAQUE_PYOBJECT) // The rest depends on numpy: try { py::module_::import("numpy"); @@ -272,4 +277,5 @@ TEST_SUBMODULE(stl_binders, m) { py::bind_vector>(m, "VectorStruct", py::buffer_protocol()); m.def("get_vectorstruct", [] { return std::vector{{false, 5, 3.0, true}, {true, 30, -1e4, false}}; }); +#endif } diff --git a/tests/test_stl_binders.py b/tests/test_stl_binders.py index edc6d28924..523a5b991e 100644 --- a/tests/test_stl_binders.py +++ b/tests/test_stl_binders.py @@ -118,13 +118,16 @@ def test_vector_buffer(): v = m.VectorUChar(mv[::2]) assert v[1] == 3 - with pytest.raises(RuntimeError) as excinfo: - m.create_undeclstruct() # Undeclared struct contents, no buffer interface - assert "NumPy type info missing for " in str(excinfo.value) + if hasattr(m, "create_undeclstruct"): # not built with the abi3t stable ABI + with pytest.raises(RuntimeError) as excinfo: + m.create_undeclstruct() # Undeclared struct contents, no buffer interface + assert "NumPy type info missing for " in str(excinfo.value) def test_vector_buffer_numpy(): np = pytest.importorskip("numpy") + if not hasattr(m, "VectorStruct"): + pytest.skip("numpy.h is not available with the abi3t stable ABI") a = np.array([1, 2, 3, 4], dtype=np.int32) with pytest.raises(TypeError): m.VectorInt(a) diff --git a/tools/check_inl_headers.py b/tools/check_inl_headers.py index eb2ca87ad3..9063f078b9 100755 --- a/tools/check_inl_headers.py +++ b/tools/check_inl_headers.py @@ -45,6 +45,7 @@ "PYBIND11_HAS_DIRECT_STRUCT_ACCESS", "PYBIND11_HAS_SUBINTERPRETER_SUPPORT", "PYBIND11_INTERNALS_VERSION", + "PYBIND11_OPAQUE_PYOBJECT", # implied by Py_LIMITED_API with Py_GIL_DISABLED "PYBIND11_SIMPLE_GIL_MANAGEMENT", "PYBIND11_TYPE_CREATION_VIA_SPEC", "Py_GIL_DISABLED", diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 4f490572b3..5618776876 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -269,9 +269,37 @@ set(PYBIND11_STABLE_ABI_VERSION "3.12" CACHE STRING "Python version whose stable ABI STABLE_ABI modules target (3.12 or newer)") -# PYBIND11_STABLE_ABI_VERSION as the Py_LIMITED_API hex value. +# True if the interpreter is a free-threaded build (needs the abi3t variant of the stable ABI). +function(_pybind11_python_is_free_threaded out_var) + if(${_Python}_FREE_THREADED + OR "${${_Python}_SOABI}" MATCHES "^cpython-[0-9]+t" + OR "${PYTHON_MODULE_EXTENSION}" MATCHES "^\\.cpython-[0-9]+t") + set(${out_var} + ON + PARENT_SCOPE) + else() + set(${out_var} + OFF + PARENT_SCOPE) + endif() +endfunction() + +# PYBIND11_STABLE_ABI_VERSION, raised to 3.15 on free-threaded Python (abi3t starts there). +function(_pybind11_stable_abi_version out_var) + set(_version "${PYBIND11_STABLE_ABI_VERSION}") + _pybind11_python_is_free_threaded(_free_threaded) + if(_free_threaded AND _version VERSION_LESS 3.15) + set(_version "3.15") + endif() + set(${out_var} + "${_version}" + PARENT_SCOPE) +endfunction() + +# The effective stable ABI version as the Py_LIMITED_API hex value. function(_pybind11_stable_abi_hex out_var) - string(REPLACE "." ";" _parts "${PYBIND11_STABLE_ABI_VERSION}") + _pybind11_stable_abi_version(_version) + string(REPLACE "." ";" _parts "${_version}") list(GET _parts 0 _major) list(GET _parts 1 _minor) math(EXPR _hex "(${_major} << 24) | (${_minor} << 16)" OUTPUT_FORMAT HEXADECIMAL) @@ -300,9 +328,12 @@ function(_pybind11_check_stable_abi target_name lib_type) message(FATAL_ERROR "${target_name}: STABLE_ABI ${PYBIND11_STABLE_ABI_VERSION} needs Python " ">= ${PYBIND11_STABLE_ABI_VERSION} headers, found ${${_Python}_VERSION}.") endif() - if("${${_Python}_SOABI}" MATCHES "^cpython-[0-9]+t" OR "${PYTHON_MODULE_EXTENSION}" MATCHES - "^\\.cpython-[0-9]+t") - message(FATAL_ERROR "${target_name}: the free-threaded build has no stable ABI.") + _pybind11_python_is_free_threaded(_free_threaded) + if(_free_threaded + AND DEFINED ${_Python}_VERSION + AND ${_Python}_VERSION VERSION_LESS 3.15) + message(FATAL_ERROR "${target_name}: the free-threaded stable ABI (abi3t) needs Python " + ">= 3.15, found ${${_Python}_VERSION}.") endif() if(NOT TARGET ${_Python}::SABIModule) message( @@ -340,7 +371,8 @@ function(pybind11_add_module target_name) set(stable_abi ON) if(lib_type STREQUAL "MODULE") # Defines Py_LIMITED_API and links Python::SABIModule (python3.lib on Windows). - set(use_sabi USE_SABI ${PYBIND11_STABLE_ABI_VERSION}) + _pybind11_stable_abi_version(_sabi_version) + set(use_sabi USE_SABI ${_sabi_version}) endif() endif() From d7ade075561d74ba240da084c8c7df2dcb1a93c3 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 21:43:27 -0400 Subject: [PATCH 25/45] feat(numpy): support numpy.h under the abi3t stable ABI Under abi3t the size of PyObject_HEAD is not known at compile time, so the proxy structs drop it and the accessors go through the field getters NumPy 2.5 added for the same purpose (_PyArray_GET_ITEM_DATA and _PyDataType_GET_ITEM_DATA); void scalars are read through the buffer protocol. Older NumPy versions are rejected at run time. The numpy and Eigen tests run again under abi3t. Assisted-by: ClaudeCode:claude-fable-5-1 --- .github/workflows/ci.yml | 6 --- docs/advanced/stable_abi.rst | 11 +++-- include/pybind11/numpy.h | 78 +++++++++++++++++++++++++++++++----- tests/CMakeLists.txt | 13 ------ tests/test_stl_binders.cpp | 8 +--- tests/test_stl_binders.py | 9 ++--- 6 files changed, 78 insertions(+), 47 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3afc593345..dddc880786 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -240,12 +240,6 @@ jobs: --ignore=tests/test_custom_type_setup.py --ignore=tests/test_multiple_interpreters.py --ignore=tests/test_class_release_gil_before_calling_cpp_dtor.py - --ignore=tests/test_numpy_array.py - --ignore=tests/test_numpy_dtypes.py - --ignore=tests/test_numpy_scalars.py - --ignore=tests/test_numpy_vectorize.py - --ignore=tests/test_eigen_matrix.py - --ignore=tests/test_eigen_tensor.py steps: - uses: actions/checkout@v7 diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index 85543baf9a..d17f05c8c6 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -56,12 +56,10 @@ available and in how a few things are implemented. * - Feature - Stable ABI - Notes - * - Classes, functions, casters, STL + * - Classes, functions, casters, STL, numpy, Eigen - supported - - - * - numpy, Eigen - - supported (abi3 only) - - ``numpy.h`` mirrors NumPy's object layouts, which abi3t hides. + - Under abi3t, ``numpy.h`` needs NumPy 2.5+ at run time (it reads + array fields through NumPy's abi3t accessors). * - ``py::dynamic_attr()`` - supported - The ``__dict__`` slot is appended to the instance (no @@ -108,7 +106,8 @@ Under abi3t, ``PyObject`` and ``PyModuleDef`` are incomplete types and one weak reference per registered instance so that the instance registry can hand out strong references safely (``PyWeakref_GetRef()``). -``pybind11/numpy.h`` (and therefore the Eigen headers) is not available. The +``pybind11/numpy.h`` reads NumPy's object fields through the accessors NumPy +2.5 added for abi3t, so older NumPy versions are rejected at run time. The internals tag is ``_stable_ft``: an abi3t module and an abi3 module loaded into the same GIL-enabled interpreter do not share internals (see below). diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 43393f97de..aeee09ef2d 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -10,10 +10,6 @@ #pragma once #include "pybind11.h" - -#if defined(PYBIND11_OPAQUE_PYOBJECT) -# error "numpy.h mirrors NumPy object layouts; it is not available with the abi3t stable ABI." -#endif #include "detail/common.h" #include "complex.h" #include "gil_safe_call_once.h" @@ -75,9 +71,19 @@ struct handle_type_name { template struct npy_format_descriptor; +// The proxies mirror NumPy's object layouts. Under the abi3t stable ABI PyObject is opaque and +// its size differs between free-threaded and GIL-enabled interpreters; NumPy >= 2.5 then +// defines the same structs without the header and returns pointers to them from its C API +// (`_PyArray_GET_ITEM_DATA`, `_PyDataType_GET_ITEM_DATA`), which the accessors below use. +#if defined(PYBIND11_OPAQUE_PYOBJECT) +# define PYBIND11_NUMPY_PROXY_HEAD +#else +# define PYBIND11_NUMPY_PROXY_HEAD PyObject_HEAD +#endif + /* NumPy 1 proxy (always includes legacy fields) */ struct PyArrayDescr1_Proxy { - PyObject_HEAD + PYBIND11_NUMPY_PROXY_HEAD PyObject *typeobj; char kind; char type; @@ -92,7 +98,7 @@ struct PyArrayDescr1_Proxy { }; struct PyArrayDescr_Proxy { - PyObject_HEAD + PYBIND11_NUMPY_PROXY_HEAD PyObject *typeobj; char kind; char type; @@ -104,7 +110,7 @@ struct PyArrayDescr_Proxy { /* NumPy 2 proxy, including legacy fields */ struct PyArrayDescr2_Proxy { - PyObject_HEAD + PYBIND11_NUMPY_PROXY_HEAD PyObject *typeobj; char kind; char type; @@ -124,7 +130,7 @@ struct PyArrayDescr2_Proxy { }; struct PyArray_Proxy { - PyObject_HEAD + PYBIND11_NUMPY_PROXY_HEAD char *data; int nd; ssize_t *dimensions; @@ -134,12 +140,14 @@ struct PyArray_Proxy { int flags; }; +#if !defined(PYBIND11_OPAQUE_PYOBJECT) struct PyVoidScalarObject_Proxy { PyObject_VAR_HEAD char *obval; PyArrayDescr_Proxy *descr; int flags; PyObject *base; }; +#endif struct numpy_type_info { PyObject *dtype_ptr; @@ -321,6 +329,10 @@ struct npy_api { PyObject *(*PyArray_Squeeze_)(PyObject *); // Unused. Not removed because that affects ABI of the class. int (*PyArray_SetBaseObject_)(PyObject *, PyObject *); +#if defined(PYBIND11_OPAQUE_PYOBJECT) + PyArray_Proxy *(*_PyArray_GET_ITEM_DATA_)(const PyObject *); + PyArrayDescr_Proxy *(*_PyDataType_GET_ITEM_DATA_)(const PyObject *); +#endif PyObject *(*PyArray_Resize_)(PyObject *, PyArray_Dims *, int, int); PyObject *(*PyArray_Newshape_)(PyObject *, PyArray_Dims *, int); PyObject *(*PyArray_View_)(PyObject *, PyObject *, PyObject *); @@ -348,7 +360,10 @@ struct npy_api { API_PyArray_View = 137, API_PyArray_DescrConverter = 174, API_PyArray_EquivTypes = 182, - API_PyArray_SetBaseObject = 282 + API_PyArray_SetBaseObject = 282, + // NumPy >= 2.5: field access for abi3t, where PyObject_HEAD has no fixed size. + API__PyArray_GET_ITEM_DATA = 369, + API__PyDataType_GET_ITEM_DATA = 372 }; static npy_api lookup() { @@ -386,6 +401,14 @@ struct npy_api { DECL_NPY_API(PyArray_DescrConverter); DECL_NPY_API(PyArray_EquivTypes); DECL_NPY_API(PyArray_SetBaseObject); +#if defined(PYBIND11_OPAQUE_PYOBJECT) + if (api.PyArray_RUNTIME_VERSION_ < 0x16) { + pybind11_fail( + "pybind11 numpy support with the abi3t stable ABI requires numpy >= 2.5"); + } + DECL_NPY_API(_PyArray_GET_ITEM_DATA); + DECL_NPY_API(_PyDataType_GET_ITEM_DATA); +#endif #undef DECL_NPY_API return api; @@ -537,6 +560,31 @@ static constexpr int normalized_dtype_num[npy_api::NPY_VOID_ + 1] = { npy_api::NPY_VOID_, }; +#if defined(PYBIND11_OPAQUE_PYOBJECT) +inline PyArray_Proxy *array_proxy(void *ptr) { + return npy_api::get()._PyArray_GET_ITEM_DATA_(static_cast(ptr)); +} + +inline const PyArray_Proxy *array_proxy(const void *ptr) { + return npy_api::get()._PyArray_GET_ITEM_DATA_(static_cast(ptr)); +} + +inline PyArrayDescr_Proxy *array_descriptor_proxy(PyObject *ptr) { + return npy_api::get()._PyDataType_GET_ITEM_DATA_(ptr); +} + +inline const PyArrayDescr_Proxy *array_descriptor_proxy(const PyObject *ptr) { + return npy_api::get()._PyDataType_GET_ITEM_DATA_(ptr); +} + +inline const PyArrayDescr1_Proxy *array_descriptor1_proxy(const PyObject *ptr) { + return reinterpret_cast(array_descriptor_proxy(ptr)); +} + +inline const PyArrayDescr2_Proxy *array_descriptor2_proxy(const PyObject *ptr) { + return reinterpret_cast(array_descriptor_proxy(ptr)); +} +#else inline PyArray_Proxy *array_proxy(void *ptr) { return reinterpret_cast(ptr); } inline const PyArray_Proxy *array_proxy(const void *ptr) { @@ -558,6 +606,7 @@ inline const PyArrayDescr1_Proxy *array_descriptor1_proxy(const PyObject *ptr) { inline const PyArrayDescr2_Proxy *array_descriptor2_proxy(const PyObject *ptr) { return reinterpret_cast(ptr); } +#endif inline bool check_flags(const void *ptr, int flag) { return (flag == (array_proxy(ptr)->flags & flag)); @@ -1792,7 +1841,18 @@ struct npy_format_descriptor { } if (auto descr = reinterpret_steal(api.PyArray_DescrFromScalar_(obj))) { if (api.PyArray_EquivTypes_(dtype_ptr(), descr.ptr())) { +#if defined(PYBIND11_OPAQUE_PYOBJECT) + // The void scalar is opaque; its buffer is the scalar's own storage. + Py_buffer view; + if (PyObject_GetBuffer(obj, &view, PyBUF_SIMPLE) != 0) { + PyErr_Clear(); + return false; + } + value = view.buf; + PyBuffer_Release(&view); +#else value = ((PyVoidScalarObject_Proxy *) obj)->obval; +#endif return true; } } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7296bec3bc..58e80b83f5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -236,19 +236,6 @@ if(PYBIND11_TEST_STABLE_ABI) MESSAGE "Skipping chrono, custom_type_setup, subinterpreter and PyGILState_Check tests under the stable ABI" ) - # numpy.h mirrors NumPy object layouts, which the abi3t stable ABI hides (PEP 803). - if("${PYTHON_MODULE_EXTENSION}" MATCHES "^\\.cpython-[0-9]+t") - pybind11_filter_tests( - PYBIND11_TEST_FILES - test_numpy_array.cpp - test_numpy_dtypes.cpp - test_numpy_scalars.cpp - test_numpy_vectorize.cpp - test_eigen_matrix.cpp - test_eigen_tensor.cpp - MESSAGE - "Skipping numpy and eigen tests under the free-threaded stable ABI (abi3t)") - endif() endif() # Skip tests for CUDA check: diff --git a/tests/test_stl_binders.cpp b/tests/test_stl_binders.cpp index bad5fe86de..f399ec0e43 100644 --- a/tests/test_stl_binders.cpp +++ b/tests/test_stl_binders.cpp @@ -7,10 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +#include #include -#if !defined(PYBIND11_OPAQUE_PYOBJECT) -# include -#endif #include "pybind11_tests.h" @@ -240,12 +238,10 @@ TEST_SUBMODULE(stl_binders, m) { double y; bool z; }; -#if !defined(PYBIND11_OPAQUE_PYOBJECT) // needs numpy.h's generic format_descriptor m.def("create_undeclstruct", [m]() mutable { py::bind_vector>( m, "VectorUndeclStruct", py::buffer_protocol()); }); -#endif // Bind recursive container types py::bind_vector(m, "RecursiveVector"); @@ -257,7 +253,6 @@ TEST_SUBMODULE(stl_binders, m) { py::bind_vector(m, "UserVectorLike"); py::bind_map(m, "UserMapLike"); -#if !defined(PYBIND11_OPAQUE_PYOBJECT) // The rest depends on numpy: try { py::module_::import("numpy"); @@ -277,5 +272,4 @@ TEST_SUBMODULE(stl_binders, m) { py::bind_vector>(m, "VectorStruct", py::buffer_protocol()); m.def("get_vectorstruct", [] { return std::vector{{false, 5, 3.0, true}, {true, 30, -1e4, false}}; }); -#endif } diff --git a/tests/test_stl_binders.py b/tests/test_stl_binders.py index 523a5b991e..edc6d28924 100644 --- a/tests/test_stl_binders.py +++ b/tests/test_stl_binders.py @@ -118,16 +118,13 @@ def test_vector_buffer(): v = m.VectorUChar(mv[::2]) assert v[1] == 3 - if hasattr(m, "create_undeclstruct"): # not built with the abi3t stable ABI - with pytest.raises(RuntimeError) as excinfo: - m.create_undeclstruct() # Undeclared struct contents, no buffer interface - assert "NumPy type info missing for " in str(excinfo.value) + with pytest.raises(RuntimeError) as excinfo: + m.create_undeclstruct() # Undeclared struct contents, no buffer interface + assert "NumPy type info missing for " in str(excinfo.value) def test_vector_buffer_numpy(): np = pytest.importorskip("numpy") - if not hasattr(m, "VectorStruct"): - pytest.skip("numpy.h is not available with the abi3t stable ABI") a = np.array([1, 2, 3, 4], dtype=np.int32) with pytest.raises(TypeError): m.VectorInt(a) From 84cedffbc0b846fb161ac5202108c53c6b3dfe71 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 21:46:42 -0400 Subject: [PATCH 26/45] feat(chrono): support chrono.h under the stable ABI The datetime C API is not part of the limited API. Under Py_LIMITED_API, read the fields as attributes and build objects by calling the datetime types; regular builds keep the C macros. The chrono test runs under the stable ABI again. Assisted-by: ClaudeCode:claude-fable-5-1 --- .github/workflows/ci.yml | 2 - docs/advanced/stable_abi.rst | 5 +- include/pybind11/chrono.h | 189 +++++++++++++++++++++++++---------- tests/CMakeLists.txt | 4 +- 4 files changed, 142 insertions(+), 58 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dddc880786..65a568ae7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -171,7 +171,6 @@ jobs: timeout-minutes: 60 env: PYBIND11_STABLE_ABI_SKIPPED_TESTS: >- - --ignore=tests/test_chrono.py --ignore=tests/test_custom_type_setup.py --ignore=tests/test_multiple_interpreters.py --ignore=tests/test_class_release_gil_before_calling_cpp_dtor.py @@ -236,7 +235,6 @@ jobs: timeout-minutes: 60 env: PYBIND11_STABLE_ABI_SKIPPED_TESTS: >- - --ignore=tests/test_chrono.py --ignore=tests/test_custom_type_setup.py --ignore=tests/test_multiple_interpreters.py --ignore=tests/test_class_release_gil_before_calling_cpp_dtor.py diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index d17f05c8c6..0eb32c65eb 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -85,8 +85,9 @@ available and in how a few things are implemented. - not available - Reads thread- and interpreter-state fields. * - ``pybind11/chrono.h`` - - not available - - The ``datetime`` C API accesses struct fields. + - supported + - The ``datetime`` C API is not in the stable ABI; fields are read as + attributes and objects are built by calling the ``datetime`` types. * - GIL-held assertions (``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``) - not available - ``PyGILState_Check()`` is not part of the stable ABI. diff --git a/include/pybind11/chrono.h b/include/pybind11/chrono.h index 29188511ef..953217bb95 100644 --- a/include/pybind11/chrono.h +++ b/include/pybind11/chrono.h @@ -12,20 +12,120 @@ #include "pybind11.h" -#if defined(Py_LIMITED_API) -# error \ - "pybind11/chrono.h is not available under the stable ABI (Py_LIMITED_API): the datetime C API accesses struct fields." -#endif - #include #include #include -#include #include +#if !defined(Py_LIMITED_API) +# include +#endif + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) +#if defined(Py_LIMITED_API) +// The datetime C API is not part of the stable ABI: use the Python-level types and attributes. +struct datetime_types { + PyTypeObject *datetime, *date, *time, *timedelta; +}; + +inline const datetime_types &get_datetime_types() { + // The type objects are kept for the lifetime of the process. + static const datetime_types types = [] { + module_ m = module_::import("datetime"); + auto get = [&](const char *name) { + return reinterpret_cast(m.attr(name).cast().release().ptr()); + }; + return datetime_types{get("datetime"), get("date"), get("time"), get("timedelta")}; + }(); + return types; +} + +inline void datetime_import() { (void) get_datetime_types(); } +inline bool is_timedelta(handle h) { + return PyObject_TypeCheck(h.ptr(), get_datetime_types().timedelta) != 0; +} +inline bool is_datetime(handle h) { + return PyObject_TypeCheck(h.ptr(), get_datetime_types().datetime) != 0; +} +inline bool is_date(handle h) { + return PyObject_TypeCheck(h.ptr(), get_datetime_types().date) != 0; +} +inline bool is_time(handle h) { + return PyObject_TypeCheck(h.ptr(), get_datetime_types().time) != 0; +} +inline int datetime_field(handle h, const char *name) { return h.attr(name).cast(); } +# define PYBIND11_DATETIME_FIELD(fn, attr) \ + inline int fn(handle h) { return datetime_field(h, #attr); } +# define PYBIND11_DATETIME_FIELD_C(fn, c_macro) + +inline PyObject *make_timedelta(int days, int seconds, int microseconds) { + return PyObject_CallFunction(reinterpret_cast(get_datetime_types().timedelta), + "iii", + days, + seconds, + microseconds); +} +inline PyObject * +make_datetime(int year, int month, int day, int hour, int minute, int second, int microsecond) { + return PyObject_CallFunction(reinterpret_cast(get_datetime_types().datetime), + "iiiiiii", + year, + month, + day, + hour, + minute, + second, + microsecond); +} +#else +inline void datetime_import() { + if (!PyDateTimeAPI) { + PyDateTime_IMPORT; + } +} +inline bool is_timedelta(handle h) { return PyDelta_Check(h.ptr()); } +inline bool is_datetime(handle h) { return PyDateTime_Check(h.ptr()); } +inline bool is_date(handle h) { return PyDate_Check(h.ptr()); } +inline bool is_time(handle h) { return PyTime_Check(h.ptr()); } +# define PYBIND11_DATETIME_FIELD(fn, attr) +# define PYBIND11_DATETIME_FIELD_C(fn, c_macro) \ + inline int fn(handle h) { return c_macro(h.ptr()); } +inline PyObject *make_timedelta(int days, int seconds, int microseconds) { + return PyDelta_FromDSU(days, seconds, microseconds); +} +inline PyObject * +make_datetime(int year, int month, int day, int hour, int minute, int second, int microsecond) { + return PyDateTime_FromDateAndTime(year, month, day, hour, minute, second, microsecond); +} +#endif + +// One accessor per field: attribute lookup under the stable ABI, the C macro otherwise. +#define PYBIND11_DATETIME_ACCESSOR(fn, attr, c_macro) \ + PYBIND11_DATETIME_FIELD(fn, attr) \ + PYBIND11_DATETIME_FIELD_C(fn, c_macro) +PYBIND11_DATETIME_ACCESSOR(timedelta_days, days, PyDateTime_DELTA_GET_DAYS) +PYBIND11_DATETIME_ACCESSOR(timedelta_seconds, seconds, PyDateTime_DELTA_GET_SECONDS) +PYBIND11_DATETIME_ACCESSOR(timedelta_microseconds, microseconds, PyDateTime_DELTA_GET_MICROSECONDS) +PYBIND11_DATETIME_ACCESSOR(datetime_year, year, PyDateTime_GET_YEAR) +PYBIND11_DATETIME_ACCESSOR(datetime_month, month, PyDateTime_GET_MONTH) +PYBIND11_DATETIME_ACCESSOR(datetime_day, day, PyDateTime_GET_DAY) +PYBIND11_DATETIME_ACCESSOR(datetime_hour, hour, PyDateTime_DATE_GET_HOUR) +PYBIND11_DATETIME_ACCESSOR(datetime_minute, minute, PyDateTime_DATE_GET_MINUTE) +PYBIND11_DATETIME_ACCESSOR(datetime_second, second, PyDateTime_DATE_GET_SECOND) +PYBIND11_DATETIME_ACCESSOR(datetime_microsecond, microsecond, PyDateTime_DATE_GET_MICROSECOND) +PYBIND11_DATETIME_ACCESSOR(date_year, year, PyDateTime_GET_YEAR) +PYBIND11_DATETIME_ACCESSOR(date_month, month, PyDateTime_GET_MONTH) +PYBIND11_DATETIME_ACCESSOR(date_day, day, PyDateTime_GET_DAY) +PYBIND11_DATETIME_ACCESSOR(time_hour, hour, PyDateTime_TIME_GET_HOUR) +PYBIND11_DATETIME_ACCESSOR(time_minute, minute, PyDateTime_TIME_GET_MINUTE) +PYBIND11_DATETIME_ACCESSOR(time_second, second, PyDateTime_TIME_GET_SECOND) +PYBIND11_DATETIME_ACCESSOR(time_microsecond, microsecond, PyDateTime_TIME_GET_MICROSECOND) +#undef PYBIND11_DATETIME_ACCESSOR +#undef PYBIND11_DATETIME_FIELD +#undef PYBIND11_DATETIME_FIELD_C + template class duration_caster { public: @@ -38,20 +138,16 @@ class duration_caster { bool load(handle src, bool) { using namespace std::chrono; - // Lazy initialise the PyDateTime import - if (!PyDateTimeAPI) { - PyDateTime_IMPORT; - } + datetime_import(); if (!src) { return false; } // If invoked with datetime.delta object - if (PyDelta_Check(src.ptr())) { + if (is_timedelta(src)) { value = type(duration_cast>( - days(PyDateTime_DELTA_GET_DAYS(src.ptr())) - + seconds(PyDateTime_DELTA_GET_SECONDS(src.ptr())) - + microseconds(PyDateTime_DELTA_GET_MICROSECONDS(src.ptr())))); + days(timedelta_days(src)) + seconds(timedelta_seconds(src)) + + microseconds(timedelta_microseconds(src)))); return true; } // If invoked with a float we assume it is seconds and convert @@ -85,10 +181,7 @@ class duration_caster { // Works out if it is a duration or time_point and get the duration auto d = get_duration(src); - // Lazy initialise the PyDateTime import - if (!PyDateTimeAPI) { - PyDateTime_IMPORT; - } + datetime_import(); // Declare these special duration types so the conversions happen with the correct // primitive types (int) @@ -100,7 +193,7 @@ class duration_caster { auto subd = d - dd; auto ss = duration_cast(subd); auto us = duration_cast(subd - ss); - return PyDelta_FromDSU(dd.count(), ss.count(), us.count()); + return make_timedelta(dd.count(), ss.count(), us.count()); } PYBIND11_TYPE_CASTER(type, const_name("datetime.timedelta")); @@ -130,10 +223,7 @@ class type_caster> bool load(handle src, bool) { using namespace std::chrono; - // Lazy initialise the PyDateTime import - if (!PyDateTimeAPI) { - PyDateTime_IMPORT; - } + datetime_import(); if (!src) { return false; @@ -142,33 +232,33 @@ class type_caster> std::tm cal; microseconds msecs; - if (PyDateTime_Check(src.ptr())) { - cal.tm_sec = PyDateTime_DATE_GET_SECOND(src.ptr()); - cal.tm_min = PyDateTime_DATE_GET_MINUTE(src.ptr()); - cal.tm_hour = PyDateTime_DATE_GET_HOUR(src.ptr()); - cal.tm_mday = PyDateTime_GET_DAY(src.ptr()); - cal.tm_mon = PyDateTime_GET_MONTH(src.ptr()) - 1; - cal.tm_year = PyDateTime_GET_YEAR(src.ptr()) - 1900; + if (is_datetime(src)) { + cal.tm_sec = datetime_second(src); + cal.tm_min = datetime_minute(src); + cal.tm_hour = datetime_hour(src); + cal.tm_mday = datetime_day(src); + cal.tm_mon = datetime_month(src) - 1; + cal.tm_year = datetime_year(src) - 1900; cal.tm_isdst = -1; - msecs = microseconds(PyDateTime_DATE_GET_MICROSECOND(src.ptr())); - } else if (PyDate_Check(src.ptr())) { + msecs = microseconds(datetime_microsecond(src)); + } else if (is_date(src)) { cal.tm_sec = 0; cal.tm_min = 0; cal.tm_hour = 0; - cal.tm_mday = PyDateTime_GET_DAY(src.ptr()); - cal.tm_mon = PyDateTime_GET_MONTH(src.ptr()) - 1; - cal.tm_year = PyDateTime_GET_YEAR(src.ptr()) - 1900; + cal.tm_mday = date_day(src); + cal.tm_mon = date_month(src) - 1; + cal.tm_year = date_year(src) - 1900; cal.tm_isdst = -1; msecs = microseconds(0); - } else if (PyTime_Check(src.ptr())) { - cal.tm_sec = PyDateTime_TIME_GET_SECOND(src.ptr()); - cal.tm_min = PyDateTime_TIME_GET_MINUTE(src.ptr()); - cal.tm_hour = PyDateTime_TIME_GET_HOUR(src.ptr()); + } else if (is_time(src)) { + cal.tm_sec = time_second(src); + cal.tm_min = time_minute(src); + cal.tm_hour = time_hour(src); cal.tm_mday = 1; // This date (day, month, year) = (1, 0, 70) cal.tm_mon = 0; // represents 1-Jan-1970, which is the first cal.tm_year = 70; // earliest available date for Python's datetime cal.tm_isdst = -1; - msecs = microseconds(PyDateTime_TIME_GET_MICROSECOND(src.ptr())); + msecs = microseconds(time_microsecond(src)); } else { return false; } @@ -182,10 +272,7 @@ class type_caster> handle /* parent */) { using namespace std::chrono; - // Lazy initialise the PyDateTime import - if (!PyDateTimeAPI) { - PyDateTime_IMPORT; - } + datetime_import(); // Get out microseconds, and make sure they are positive, to avoid bug in eastern // hemisphere time zones (cfr. https://github.com/pybind/pybind11/issues/2417) @@ -206,13 +293,13 @@ class type_caster> if (!localtime_ptr) { throw cast_error("Unable to represent system_clock in local time"); } - return PyDateTime_FromDateAndTime(localtime.tm_year + 1900, - localtime.tm_mon + 1, - localtime.tm_mday, - localtime.tm_hour, - localtime.tm_min, - localtime.tm_sec, - us.count()); + return make_datetime(localtime.tm_year + 1900, + localtime.tm_mon + 1, + localtime.tm_mday, + localtime.tm_hour, + localtime.tm_min, + localtime.tm_sec, + us.count()); } PYBIND11_TYPE_CASTER(type, const_name("datetime.datetime")); }; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 58e80b83f5..b52324c3d7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -229,13 +229,11 @@ endif() if(PYBIND11_TEST_STABLE_ABI) pybind11_filter_tests( PYBIND11_TEST_FILES - test_chrono.cpp test_custom_type_setup.cpp test_multiple_interpreters.py test_class_release_gil_before_calling_cpp_dtor.cpp # PyGILState_Check MESSAGE - "Skipping chrono, custom_type_setup, subinterpreter and PyGILState_Check tests under the stable ABI" - ) + "Skipping custom_type_setup, subinterpreter and PyGILState_Check tests under the stable ABI") endif() # Skip tests for CUDA check: From d34635cfa35ed33833839f0f0fc9c146aeaaaea9 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 22:09:25 -0400 Subject: [PATCH 27/45] perf(stable-abi): use the default type getattro under the limited API Attribute reads on bound classes (and Python subclasses of them) went through pybind11_meta_getattro, which walks the MRO with attribute calls and raised KeyError for every miss: ~20x slower than `type`. pybind11's own instancemethod type already returns itself from `__get__` when accessed on a class, so the metaclass does not need to intercept reads. Drop the slot under Py_LIMITED_API and speed up the remaining type_lookup() users (setattro, conduit) with interned names and PyMapping_GetOptionalItem. Assisted-by: ClaudeCode:claude-fable-5-1 --- docs/advanced/stable_abi.rst | 5 ++-- include/pybind11/detail/class-inl.h | 15 ++++++----- include/pybind11/pytypes.h | 37 +++++++++++++++++++++++----- tests/test_methods_and_attributes.py | 17 +++++++++++++ 4 files changed, 60 insertions(+), 14 deletions(-) diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index 0eb32c65eb..c794219053 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -137,8 +137,9 @@ to specific operations: * Instance allocation and deallocation look up ``tp_alloc``/``tp_free`` with ``PyType_GetSlot()``; the ``__dict__`` of ``py::dynamic_attr()`` instances is found through the type's cached dictionary offset. -* Attribute access on bound *classes* (not instances) walks ``__mro__`` and - the class dictionaries instead of using ``_PyType_Lookup()``. +* Attribute *assignment* on bound classes (``Type.static_prop = value``) walks + ``__mro__`` and the class dictionaries instead of using ``_PyType_Lookup()``; + attribute reads use the metaclass default and are not affected. * Methods are bound through a pybind11-provided ``instancemethod`` type and ``types.MethodType`` instead of ``PyInstanceMethod_Type`` / ``PyMethod_New()``. diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index f1a943ac18..f836e8fae9 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -328,12 +328,15 @@ extern "C" PYBIND11_INLINE void pybind11_meta_dealloc(PyObject *obj) { #if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) PYBIND11_INLINE PyTypeObject *make_default_metaclass() { - static PyType_Slot slots[] - = {{Py_tp_call, reinterpret_cast(pybind11_meta_call)}, - {Py_tp_setattro, reinterpret_cast(pybind11_meta_setattro)}, - {Py_tp_getattro, reinterpret_cast(pybind11_meta_getattro)}, - {Py_tp_dealloc, reinterpret_cast(pybind11_meta_dealloc)}, - {0, nullptr}}; + static PyType_Slot slots[] = { + {Py_tp_call, reinterpret_cast(pybind11_meta_call)}, + {Py_tp_setattro, reinterpret_cast(pybind11_meta_setattro)}, +# if !defined(Py_LIMITED_API) + // Not needed with pybind11's own instancemethod: its `__get__` returns itself on a class. + {Py_tp_getattro, reinterpret_cast(pybind11_meta_getattro)}, +# endif + {Py_tp_dealloc, reinterpret_cast(pybind11_meta_dealloc)}, + {0, nullptr}}; static PyType_Spec spec = {PYBIND11_DUMMY_MODULE_NAME ".pybind11_type", 0, // inherit from type 0, diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 2945d2f037..5fdc7627a4 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -2660,20 +2660,45 @@ inline tuple get_type_tuple_attr(PyTypeObject *type, const char *name) { } inline tuple get_bases(PyTypeObject *type) { return get_type_tuple_attr(type, "__bases__"); } -inline tuple get_mro(PyTypeObject *type) { return get_type_tuple_attr(type, "__mro__"); } + +inline PyObject *interned_name(const char *name) { + PyObject *obj = PyUnicode_InternFromString(name); + if (obj == nullptr) { + throw error_already_set(); + } + return obj; // kept alive on purpose +} + +inline tuple get_mro(PyTypeObject *type) { + static PyObject *name = interned_name("__mro__"); + auto value = reinterpret_steal(PyObject_GenericGetAttr((PyObject *) type, name)); + if (!value || !PyTuple_Check(value.ptr())) { + PyErr_Clear(); + return tuple(); + } + return reinterpret_steal(value.release()); +} // No type attribute cache: walks the MRO and looks in each class's `__dict__` (a mappingproxy). inline object type_lookup(PyTypeObject *type, handle name) { + static PyObject *dict_name = interned_name("__dict__"); for (handle base : get_mro(type)) { - object dict - = type_generic_getattr(reinterpret_cast(base.ptr()), "__dict__"); + auto dict = reinterpret_steal(PyObject_GenericGetAttr(base.ptr(), dict_name)); if (!dict) { + PyErr_Clear(); continue; } - auto value = reinterpret_steal(PyObject_GetItem(dict.ptr(), name.ptr())); - if (value) { - return value; +# if Py_LIMITED_API >= 0x030D0000 + PyObject *value = nullptr; + if (PyMapping_GetOptionalItem(dict.ptr(), name.ptr(), &value) == 1) { + return reinterpret_steal(value); + } +# else + PyObject *value = PyObject_GetItem(dict.ptr(), name.ptr()); + if (value != nullptr) { + return reinterpret_steal(value); } +# endif PyErr_Clear(); } return object(); diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index 2fd765af04..ff5fecdb80 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -90,6 +90,23 @@ def test_methods_and_attributes(): assert cstats.move_assignments == 0 +def test_unbound_method_access(): + """`Class.method` returns the raw instancemethod, also on Python subclasses""" + + class Sub(m.ExampleMandA): + pass + + unbound = m.ExampleMandA.internal1 + assert Sub.internal1 is unbound + assert m.ExampleMandA.__dict__["internal1"] is unbound + assert unbound(m.ExampleMandA(7)) == 7 + assert Sub.__name__ == "Sub" + + # Aliasing through the class keeps the instancemethod + Sub.alias = m.ExampleMandA.internal1 + assert Sub(9).alias() == 9 + + def test_copy_method(): """Issue #443: calling copied methods fails in Python 3""" From 2dd584b80e3ff816edc2efb0174d4249607f6679 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 22:14:03 -0400 Subject: [PATCH 28/45] perf(stable-abi): bind methods with PyObject_Vectorcall Avoids the varargs collection in PyObject_CallFunctionObjArgs on every `instance.method` access under the limited API (36 -> 32 ns here; the regular build's PyMethod_New path is 22 ns). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index f836e8fae9..084d579a80 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -536,10 +536,9 @@ instancemethod_descr_get(PyObject *self, PyObject *obj, PyObject * /*type*/) { Py_INCREF(self); return self; } - return PyObject_CallFunctionObjArgs(reinterpret_cast(get_bound_method_type()), - instancemethod_data(self)->func, - obj, - nullptr); + PyObject *args[] = {instancemethod_data(self)->func, obj}; + return PyObject_Vectorcall( + reinterpret_cast(get_bound_method_type()), args, 2, nullptr); } extern "C" PYBIND11_INLINE PyObject * From 67b54565ffbd6dc78fb077106e3b945c768abd24 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 22:32:36 -0400 Subject: [PATCH 29/45] refactor(stable-abi): share type attribute, slot and spec helpers Collapse the ad hoc stable-ABI code paths into shared helpers: - type_dictoffset/type_basicsize with cached interned names replace seven __dictoffset__/__basicsize__ reads; attr.h uses the cached type_info::dictoffset - type_from_spec replaces five "PyType_FromMetaclass or fail" copies - type_data replaces the two PyObject_GetTypeData wrappers - value_and_holder::type_is_exact replaces five alias checks - tp_name_is_one_of fetches the type name once per candidate list - PYBIND11_BASE_TYPE_SLOT defines whole functions (readable layout) - chrono.h: one accessor macro per branch, interned field names - getattr(obj, name, handle()) replaces hand-rolled lookups Cheaper paths: type_lookup returns a borrowed handle with _PyType_Lookup, the bool caster reads nb_bool via PyType_GetSlot under the stable ABI, and PYBIND11_CHECK_PYTHON_VERSION compares Py_Version instead of parsing Py_GetVersion(). Remove dead code: unused list_get_item, the unreachable try_incref stub under PYBIND11_OPAQUE_PYOBJECT, the inert force-define of PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET, a redundant PyGen_Check branch and a dead preprocessor condition. Assisted-by: ClaudeCode:claude-fable-5-1 ClaudeCode:claude-fable-5-1 --- include/pybind11/attr-inl.h | 4 +- include/pybind11/cast.h | 12 +- include/pybind11/chrono.h | 37 +++-- include/pybind11/detail/class-inl.h | 135 ++++++++---------- include/pybind11/detail/class.h | 6 + include/pybind11/detail/common.h | 43 +++--- include/pybind11/detail/cpp_conduit-inl.h | 4 +- .../detail/function_record_pyobject.h | 6 +- include/pybind11/detail/init.h | 12 +- include/pybind11/detail/internals.h | 2 +- .../pybind11/detail/type_caster_base-inl.h | 23 ++- include/pybind11/detail/type_caster_base.h | 6 + include/pybind11/detail/value_and_holder.h | 4 + include/pybind11/pybind11-inl.h | 5 +- include/pybind11/pybind11.h | 6 +- include/pybind11/pytypes-inl.h | 10 +- include/pybind11/pytypes.h | 93 +++++++----- include/pybind11/stl.h | 12 +- tools/pybind11Common.cmake | 6 +- 19 files changed, 216 insertions(+), 210 deletions(-) diff --git a/include/pybind11/attr-inl.h b/include/pybind11/attr-inl.h index b3716bc3cb..1782e48e89 100644 --- a/include/pybind11/attr-inl.h +++ b/include/pybind11/attr-inl.h @@ -49,7 +49,9 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void type_record::add_base(const std::typ bases.append(reinterpret_cast(base_info->type)); -#ifdef PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET +#if defined(Py_LIMITED_API) + dynamic_attr |= base_info->dictoffset != 0; +#elif defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) dynamic_attr |= base_info->type->tp_dictoffset != 0; #else dynamic_attr |= (PyType_GetFlags(base_info->type) & Py_TPFLAGS_MANAGED_DICT) != 0; diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 718d1bbbef..c60c6256df 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -447,11 +447,18 @@ class type_caster { if (src.is_none()) { res = 0; // None is implicitly converted to False } -#if !defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) +#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) // Check that the "__bool__" attr exists else if (hasattr(src, PYBIND11_BOOL_ATTR)) { res = PyObject_IsTrue(src.ptr()); } +#elif defined(Py_LIMITED_API) + // Same as below through the stable ABI (PyType_GetSlot works on static types since + // 3.10). + else if (auto *nb_bool + = reinterpret_cast(PyType_GetSlot(Py_TYPE(src.ptr()), Py_nb_bool))) { + res = nb_bool(src.ptr()); + } #else // Alternate approach for CPython: this does the same as the above, but optimized // using the CPython API so as to avoid an unneeded attribute lookup. @@ -478,8 +485,7 @@ class type_caster { // Test if an object is a NumPy boolean (without fetching the type). static bool is_numpy_bool(handle object) { // Name changed to `numpy.bool` in NumPy 2, `numpy.bool_` is needed for 1.x support - return tp_name_equals(Py_TYPE(object.ptr()), "numpy.bool") - || tp_name_equals(Py_TYPE(object.ptr()), "numpy.bool_"); + return tp_name_is_one_of(Py_TYPE(object.ptr()), {"numpy.bool", "numpy.bool_"}); } }; diff --git a/include/pybind11/chrono.h b/include/pybind11/chrono.h index 953217bb95..d7f801a4b9 100644 --- a/include/pybind11/chrono.h +++ b/include/pybind11/chrono.h @@ -35,7 +35,7 @@ inline const datetime_types &get_datetime_types() { static const datetime_types types = [] { module_ m = module_::import("datetime"); auto get = [&](const char *name) { - return reinterpret_cast(m.attr(name).cast().release().ptr()); + return reinterpret_cast(object(m.attr(name)).release().ptr()); }; return datetime_types{get("datetime"), get("date"), get("time"), get("timedelta")}; }(); @@ -55,10 +55,19 @@ inline bool is_date(handle h) { inline bool is_time(handle h) { return PyObject_TypeCheck(h.ptr(), get_datetime_types().time) != 0; } -inline int datetime_field(handle h, const char *name) { return h.attr(name).cast(); } -# define PYBIND11_DATETIME_FIELD(fn, attr) \ - inline int fn(handle h) { return datetime_field(h, #attr); } -# define PYBIND11_DATETIME_FIELD_C(fn, c_macro) +inline int datetime_field(handle h, PyObject *name) { + auto value = reinterpret_steal(PyObject_GetAttr(h.ptr(), name)); + if (!value) { + throw error_already_set(); + } + return value.cast(); +} +// One accessor per field: interned attribute lookup here, the C macro otherwise. +# define PYBIND11_DATETIME_ACCESSOR(fn, attr, c_macro) \ + inline int fn(handle h) { \ + static PyObject *name = interned_name(#attr); \ + return datetime_field(h, name); \ + } inline PyObject *make_timedelta(int days, int seconds, int microseconds) { return PyObject_CallFunction(reinterpret_cast(get_datetime_types().timedelta), @@ -89,8 +98,7 @@ inline bool is_timedelta(handle h) { return PyDelta_Check(h.ptr()); } inline bool is_datetime(handle h) { return PyDateTime_Check(h.ptr()); } inline bool is_date(handle h) { return PyDate_Check(h.ptr()); } inline bool is_time(handle h) { return PyTime_Check(h.ptr()); } -# define PYBIND11_DATETIME_FIELD(fn, attr) -# define PYBIND11_DATETIME_FIELD_C(fn, c_macro) \ +# define PYBIND11_DATETIME_ACCESSOR(fn, attr, c_macro) \ inline int fn(handle h) { return c_macro(h.ptr()); } inline PyObject *make_timedelta(int days, int seconds, int microseconds) { return PyDelta_FromDSU(days, seconds, microseconds); @@ -101,10 +109,6 @@ make_datetime(int year, int month, int day, int hour, int minute, int second, in } #endif -// One accessor per field: attribute lookup under the stable ABI, the C macro otherwise. -#define PYBIND11_DATETIME_ACCESSOR(fn, attr, c_macro) \ - PYBIND11_DATETIME_FIELD(fn, attr) \ - PYBIND11_DATETIME_FIELD_C(fn, c_macro) PYBIND11_DATETIME_ACCESSOR(timedelta_days, days, PyDateTime_DELTA_GET_DAYS) PYBIND11_DATETIME_ACCESSOR(timedelta_seconds, seconds, PyDateTime_DELTA_GET_SECONDS) PYBIND11_DATETIME_ACCESSOR(timedelta_microseconds, microseconds, PyDateTime_DELTA_GET_MICROSECONDS) @@ -115,16 +119,11 @@ PYBIND11_DATETIME_ACCESSOR(datetime_hour, hour, PyDateTime_DATE_GET_HOUR) PYBIND11_DATETIME_ACCESSOR(datetime_minute, minute, PyDateTime_DATE_GET_MINUTE) PYBIND11_DATETIME_ACCESSOR(datetime_second, second, PyDateTime_DATE_GET_SECOND) PYBIND11_DATETIME_ACCESSOR(datetime_microsecond, microsecond, PyDateTime_DATE_GET_MICROSECOND) -PYBIND11_DATETIME_ACCESSOR(date_year, year, PyDateTime_GET_YEAR) -PYBIND11_DATETIME_ACCESSOR(date_month, month, PyDateTime_GET_MONTH) -PYBIND11_DATETIME_ACCESSOR(date_day, day, PyDateTime_GET_DAY) PYBIND11_DATETIME_ACCESSOR(time_hour, hour, PyDateTime_TIME_GET_HOUR) PYBIND11_DATETIME_ACCESSOR(time_minute, minute, PyDateTime_TIME_GET_MINUTE) PYBIND11_DATETIME_ACCESSOR(time_second, second, PyDateTime_TIME_GET_SECOND) PYBIND11_DATETIME_ACCESSOR(time_microsecond, microsecond, PyDateTime_TIME_GET_MICROSECOND) #undef PYBIND11_DATETIME_ACCESSOR -#undef PYBIND11_DATETIME_FIELD -#undef PYBIND11_DATETIME_FIELD_C template class duration_caster { @@ -245,9 +244,9 @@ class type_caster> cal.tm_sec = 0; cal.tm_min = 0; cal.tm_hour = 0; - cal.tm_mday = date_day(src); - cal.tm_mon = date_month(src) - 1; - cal.tm_year = date_year(src) - 1900; + cal.tm_mday = datetime_day(src); + cal.tm_mon = datetime_month(src) - 1; + cal.tm_year = datetime_year(src) - 1900; cal.tm_isdst = -1; msecs = microseconds(0); } else if (is_time(src)) { diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 084d579a80..235cdaf219 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -43,29 +43,40 @@ PYBIND11_INLINE PyTypeObject *type_incref(PyTypeObject *type) { // the function-local caches make each lookup happen once per module. Below 3.10 (and on PyPy, // where PyType_GetSlot on static types is not reliable) the struct fields are read directly. #if PY_VERSION_HEX >= 0x030A0000 && !defined(PYPY_VERSION) -# define PYBIND11_BASE_TYPE_SLOT(type, slot_id, field, slot_type) \ - static const auto cached = reinterpret_cast(PyType_GetSlot(&type, slot_id)); \ - return cached; +# define PYBIND11_BASE_TYPE_SLOT(fn, slot_type, type, slot_id, field) \ + PYBIND11_INLINE slot_type fn() { \ + static const auto cached \ + = reinterpret_cast(PyType_GetSlot(&type, slot_id)); \ + return cached; \ + } #else -# define PYBIND11_BASE_TYPE_SLOT(type, slot_id, field, slot_type) return type.field; +# define PYBIND11_BASE_TYPE_SLOT(fn, slot_type, type, slot_id, field) \ + PYBIND11_INLINE slot_type fn() { return type.field; } #endif -PYBIND11_INLINE ternaryfunc type_type_call(){PYBIND11_BASE_TYPE_SLOT( - PyType_Type, Py_tp_call, tp_call, ternaryfunc)} PYBIND11_INLINE setattrofunc - type_type_setattro(){PYBIND11_BASE_TYPE_SLOT( - PyType_Type, Py_tp_setattro, tp_setattro, setattrofunc)} PYBIND11_INLINE getattrofunc - type_type_getattro(){PYBIND11_BASE_TYPE_SLOT( - PyType_Type, Py_tp_getattro, tp_getattro, getattrofunc)} PYBIND11_INLINE destructor - type_type_dealloc(){PYBIND11_BASE_TYPE_SLOT( - PyType_Type, Py_tp_dealloc, tp_dealloc, destructor)} PYBIND11_INLINE descrgetfunc - property_type_descr_get(){PYBIND11_BASE_TYPE_SLOT( - PyProperty_Type, Py_tp_descr_get, tp_descr_get, descrgetfunc)} PYBIND11_INLINE descrsetfunc - property_type_descr_set() { - PYBIND11_BASE_TYPE_SLOT(PyProperty_Type, Py_tp_descr_set, tp_descr_set, descrsetfunc) -} +PYBIND11_BASE_TYPE_SLOT(type_type_call, ternaryfunc, PyType_Type, Py_tp_call, tp_call) +PYBIND11_BASE_TYPE_SLOT(type_type_setattro, setattrofunc, PyType_Type, Py_tp_setattro, tp_setattro) +PYBIND11_BASE_TYPE_SLOT(type_type_getattro, getattrofunc, PyType_Type, Py_tp_getattro, tp_getattro) +PYBIND11_BASE_TYPE_SLOT(type_type_dealloc, destructor, PyType_Type, Py_tp_dealloc, tp_dealloc) +PYBIND11_BASE_TYPE_SLOT( + property_type_descr_get, descrgetfunc, PyProperty_Type, Py_tp_descr_get, tp_descr_get) +PYBIND11_BASE_TYPE_SLOT( + property_type_descr_set, descrsetfunc, PyProperty_Type, Py_tp_descr_set, tp_descr_set) #undef PYBIND11_BASE_TYPE_SLOT +#if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +PYBIND11_INLINE PyTypeObject * +type_from_spec(const char *caller, PyTypeObject *metaclass, PyType_Spec *spec, PyObject *bases) { + PyObject *type = PyType_FromMetaclass(metaclass, nullptr, spec, bases); + if (type == nullptr) { + pybind11_fail(std::string(caller) + + ": failure in PyType_FromMetaclass(): " + error_string()); + } + return reinterpret_cast(type); +} +#endif + #if !defined(PYPY_VERSION) extern "C" PYBIND11_INLINE PyObject * pybind11_static_get(PyObject *self, PyObject * /*ob*/, PyObject *cls) { @@ -84,9 +95,7 @@ pybind11_static_set(PyObject *self, PyObject *obj, PyObject *value) { // Without Py_TPFLAGS_MANAGED_DICT the `__dict__` slot follows the property object. These two // functions find it through the type's `__dictoffset__`. PYBIND11_INLINE PyObject **static_property_dict_ptr(PyObject *self) { - static const Py_ssize_t offset = handle(reinterpret_cast(Py_TYPE(self))) - .attr("__dictoffset__") - .cast(); + static const Py_ssize_t offset = type_dictoffset(Py_TYPE(self)); return reinterpret_cast(reinterpret_cast(self) + offset); } extern "C" PYBIND11_INLINE int @@ -120,7 +129,7 @@ PYBIND11_INLINE PyTypeObject *make_static_property_type() { # else // Append the `__dict__` slot to the property layout. (PEP 697 relative offsets are not // applied to `__dictoffset__` on 3.12, so the offset is absolute.) - const auto dictoffset = type_generic_getattr(&PyProperty_Type, "__basicsize__").cast(); + const auto dictoffset = static_cast(type_basicsize(&PyProperty_Type)); PyMemberDef members[] = {{"__dictoffset__", Py_T_PYSSIZET, dictoffset, Py_READONLY, nullptr}, {nullptr, 0, 0, 0, nullptr}}; PyType_Slot slots[] @@ -137,13 +146,10 @@ PYBIND11_INLINE PyTypeObject *make_static_property_type() { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, slots}; # endif - PyObject *type = PyType_FromMetaclass( - nullptr, nullptr, &spec, reinterpret_cast(&PyProperty_Type)); - if (!type) { - pybind11_fail("make_static_property_type(): failure in PyType_FromMetaclass(): " - + error_string()); - } - return reinterpret_cast(type); + return type_from_spec("make_static_property_type()", + nullptr, + &spec, + reinterpret_cast(&PyProperty_Type)); } # else // legacy: fill in a PyHeapTypeObject by hand @@ -217,7 +223,7 @@ extern "C" PYBIND11_INLINE int pybind11_meta_setattro(PyObject *obj, PyObject *name, PyObject *value) { // Use `type_lookup()` instead of `PyObject_GetAttr()` in order to get the raw // descriptor (`property`) instead of calling `tp_descr_get` (`property.__get__()`). - object descr = type_lookup((PyTypeObject *) obj, name); + auto descr = type_lookup((PyTypeObject *) obj, name); // The following assignment combinations are possible: // 1. `Type.static_prop = value` --> descr_set: `Type.static_prop.__set__(value)` @@ -246,9 +252,9 @@ pybind11_meta_setattro(PyObject *obj, PyObject *name, PyObject *value) { } extern "C" PYBIND11_INLINE PyObject *pybind11_meta_getattro(PyObject *obj, PyObject *name) { - object descr = type_lookup((PyTypeObject *) obj, name); + auto descr = type_lookup((PyTypeObject *) obj, name); if (descr && PYBIND11_INSTANCE_METHOD_CHECK(descr.ptr())) { - return descr.release().ptr(); + return handle(descr).inc_ref().ptr(); } return type_type_getattro()(obj, name); } @@ -342,13 +348,8 @@ PYBIND11_INLINE PyTypeObject *make_default_metaclass() { 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, slots}; - PyObject *type = PyType_FromMetaclass( - nullptr, nullptr, &spec, reinterpret_cast(&PyType_Type)); - if (!type) { - pybind11_fail("make_default_metaclass(): failure in PyType_FromMetaclass(): " - + error_string()); - } - return reinterpret_cast(type); + return type_from_spec( + "make_default_metaclass()", nullptr, &spec, reinterpret_cast(&PyType_Type)); } #else // legacy: fill in a PyHeapTypeObject by hand @@ -496,11 +497,7 @@ struct instancemethod_object { }; PYBIND11_INLINE instancemethod_object *instancemethod_data(PyObject *self) { -# if defined(PYBIND11_OPAQUE_PYOBJECT) - return static_cast(PyObject_GetTypeData(self, Py_TYPE(self))); -# else - return reinterpret_cast(self); -# endif + return type_data(self); } PYBIND11_INLINE PyTypeObject *get_bound_method_type() { @@ -510,8 +507,7 @@ PYBIND11_INLINE PyTypeObject *get_bound_method_type() { throw error_already_set(); } // Static builtin type: keep the reference forever. - return reinterpret_cast( - types.attr("MethodType").cast().release().ptr()); + return reinterpret_cast(object(types.attr("MethodType")).release().ptr()); }(); return type; } @@ -521,13 +517,7 @@ PYBIND11_INLINE bool is_bound_method(PyObject *obj) { } PYBIND11_INLINE PyObject *bound_method_function(PyObject *obj) { - PyObject *func = PyObject_GetAttrString(obj, "__func__"); - if (func == nullptr) { - PyErr_Clear(); - return nullptr; - } - Py_DECREF(func); // the method object keeps the function alive - return func; + return getattr(obj, "__func__", handle()).ptr(); // the method object keeps the function alive } extern "C" PYBIND11_INLINE PyObject * @@ -562,9 +552,8 @@ extern "C" PYBIND11_INLINE PyObject *instancemethod_getattro(PyObject *self, PyO extern "C" PYBIND11_INLINE PyObject *instancemethod_repr(PyObject *self) { PyObject *func = instancemethod_data(self)->func; - auto name = reinterpret_steal(PyObject_GetAttrString(func, "__name__")); + object name = getattr(func, "__name__", handle()); if (!name) { - PyErr_Clear(); return PyUnicode_FromFormat("", self); } return PyUnicode_FromFormat("", name.ptr(), self); @@ -615,12 +604,8 @@ PYBIND11_INLINE PyTypeObject *get_instancemethod_type() { 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION, slots}; - PyObject *type = PyType_FromSpec(&spec); - if (type == nullptr) { - pybind11_fail("get_instancemethod_type(): failure in PyType_FromSpec(): " - + error_string()); - } - internals.instancemethod_type = reinterpret_cast(type); + internals.instancemethod_type + = type_from_spec("get_instancemethod_type()", nullptr, &spec, nullptr); } return internals.instancemethod_type; }); @@ -685,8 +670,8 @@ PYBIND11_INLINE Py_ssize_t instance_data_offset() { // The same for every pybind11 type: the type data of pybind11_object, whose base is object. static const Py_ssize_t offset = [] { handle base = get_internals().instance_base; - auto basicsize = base.attr("__basicsize__").cast(); - return basicsize - PyType_GetTypeDataSize(reinterpret_cast(base.ptr())); + auto *base_type = reinterpret_cast(base.ptr()); + return type_basicsize(base_type) - PyType_GetTypeDataSize(base_type); }(); return offset; } @@ -847,14 +832,12 @@ PYBIND11_INLINE PyObject *make_object_base_type(PyTypeObject *metaclass) { 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, slots}; - PyObject *type = PyType_FromMetaclass( - metaclass, nullptr, &spec, reinterpret_cast(&PyBaseObject_Type)); - if (!type) { - pybind11_fail("make_object_base_type(): failure in PyType_FromMetaclass(): " - + error_string()); - } - assert(!PyType_HasFeature(reinterpret_cast(type), Py_TPFLAGS_HAVE_GC)); - return type; + PyTypeObject *type = type_from_spec("make_object_base_type()", + metaclass, + &spec, + reinterpret_cast(&PyBaseObject_Type)); + assert(!PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC)); + return reinterpret_cast(type); } #else // legacy: fill in a PyHeapTypeObject by hand @@ -1133,10 +1116,8 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { int base_basicsize = 0; for (handle b : bases) { auto *b_type = reinterpret_cast(b.ptr()); - base_has_dict - |= type_generic_getattr(b_type, "__dictoffset__").cast() != 0; - base_basicsize = std::max(base_basicsize, - type_generic_getattr(b_type, "__basicsize__").cast()); + base_has_dict |= type_dictoffset(b_type) != 0; + base_basicsize = std::max(base_basicsize, static_cast(type_basicsize(b_type))); } if (!base_has_dict) { dict_member[0].offset = base_basicsize; @@ -1155,10 +1136,8 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { slots.push_back({0, nullptr}); PyType_Spec spec = {full_name, basicsize, 0, flags, slots.data()}; - PyObject *type = PyType_FromMetaclass(metaclass, nullptr, &spec, base.ptr()); - if (!type) { - pybind11_fail(std::string(rec.name) + ": PyType_FromMetaclass failed: " + error_string()); - } + auto *type + = reinterpret_cast(type_from_spec(rec.name, metaclass, &spec, base.ptr())); assert(!rec.dynamic_attr || PyType_HasFeature(reinterpret_cast(type), Py_TPFLAGS_HAVE_GC)); diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index c30a36d3d0..61b639e30d 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -45,6 +45,12 @@ destructor type_type_dealloc(); descrgetfunc property_type_descr_get(); descrsetfunc property_type_descr_set(); +#if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +/// PyType_FromMetaclass(), or pybind11_fail() with the Python error prefixed by `caller`. +PyTypeObject * +type_from_spec(const char *caller, PyTypeObject *metaclass, PyType_Spec *spec, PyObject *bases); +#endif + #if !defined(PYPY_VERSION) # if defined(Py_LIMITED_API) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index ea9ca1042a..e4616a4121 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -34,18 +34,15 @@ # if !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) # define PYBIND11_SIMPLE_GIL_MANAGEMENT // gil.h reads thread-state internals # endif +// The only type-creation path without struct access. # if !defined(PYBIND11_TYPE_CREATION_VIA_SPEC) -# define PYBIND11_TYPE_CREATION_VIA_SPEC // the only type-creation path without struct - // access +# define PYBIND11_TYPE_CREATION_VIA_SPEC # endif # if !defined(PYBIND11_HAS_SUBINTERPRETER_SUPPORT) # define PYBIND11_HAS_SUBINTERPRETER_SUPPORT 0 // reads thread/interpreter state fields # elif PYBIND11_HAS_SUBINTERPRETER_SUPPORT # error "Subinterpreter support is not available under Py_LIMITED_API." # endif -# if !defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) -# define PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET // no Py_TPFLAGS_MANAGED_DICT -# endif #endif // Heap types declare their extra data as PEP 697 type data when PyObject is opaque: a negative @@ -299,7 +296,6 @@ #endif #include -#include #include #include #include @@ -488,17 +484,16 @@ // A stable-ABI module runs on every CPython from the Py_LIMITED_API version on (abi3). # define PYBIND11_CHECK_PYTHON_VERSION \ { \ - const char *runtime_ver = Py_GetVersion(); \ - long major = std::strtol(runtime_ver, nullptr, 10); \ - const char *dot = std::strchr(runtime_ver, '.'); \ - long minor = dot ? std::strtol(dot + 1, nullptr, 10) : 0; \ - if (major != ((Py_LIMITED_API) >> 24) || minor < (((Py_LIMITED_API) >> 16) & 0xFF)) { \ + const unsigned long want_major = (Py_LIMITED_API) >> 24; \ + const unsigned long want_minor = ((Py_LIMITED_API) >> 16) & 0xFF; \ + if ((Py_Version >> 24) != want_major || ((Py_Version >> 16) & 0xFF) < want_minor) { \ PyErr_Format(PyExc_ImportError, \ - "Python version mismatch: module was compiled for the Python %d.%d " \ - "stable ABI, but the interpreter version is incompatible: %s.", \ - (int) ((Py_LIMITED_API) >> 24), \ - (int) (((Py_LIMITED_API) >> 16) & 0xFF), \ - runtime_ver); \ + "Python version mismatch: module was compiled for the Python " \ + "%lu.%lu stable ABI, but the interpreter version is incompatible: " \ + "%s.", \ + want_major, \ + want_minor, \ + Py_GetVersion()); \ return nullptr; \ } \ } @@ -846,6 +841,17 @@ inline instance *get_instance(PyObject *obj) { return reinterpret_cast(inst); } #endif +/// The C++ data of an object of a pybind11 type with PYBIND11_TYPE_DATA_SIZE(T) as basicsize +/// (not for `instance`: its offset is cached, see get_instance()). +template +T *type_data(PyObject *obj) { +#if defined(PYBIND11_OPAQUE_PYOBJECT) + return static_cast(PyObject_GetTypeData(obj, Py_TYPE(obj))); +#else + return reinterpret_cast(obj); +#endif +} + // Some older compilers (e.g. gcc 9.4.0) require // static_assert(always_false::value, "..."); // instead of @@ -1553,9 +1559,8 @@ inline void silence_unused_warnings(Args &&...) {} #endif // CPython 3.11+ provides Py_TPFLAGS_MANAGED_DICT, but PyPy3.11 does not, see PR #5508. (The -// stable ABI does not have it either; see the Py_LIMITED_API block at the top of this file.) -#if (PY_VERSION_HEX < 0x030B0000 || defined(PYPY_VERSION)) \ - && !defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) +// stable ABI has neither; it records the `__dict__` offset in type_info::dictoffset instead.) +#if PY_VERSION_HEX < 0x030B0000 || defined(PYPY_VERSION) # define PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET #endif diff --git a/include/pybind11/detail/cpp_conduit-inl.h b/include/pybind11/detail/cpp_conduit-inl.h index 733b180cfa..9050917baf 100644 --- a/include/pybind11/detail/cpp_conduit-inl.h +++ b/include/pybind11/detail/cpp_conduit-inl.h @@ -31,8 +31,8 @@ PYBIND11_INLINE bool type_is_managed_by_our_internals(PyTypeObject *type_obj) { } PYBIND11_INLINE bool is_instance_method_of_type(PyTypeObject *type_obj, PyObject *attr_name) { - object descr = type_lookup(type_obj, attr_name); - return descr && PyInstanceMethod_Check(descr.ptr()); + auto descr = type_lookup(type_obj, attr_name); + return descr && PYBIND11_INSTANCE_METHOD_CHECK(descr.ptr()); } PYBIND11_INLINE object try_get_cpp_conduit_method(PyObject *obj) { diff --git a/include/pybind11/detail/function_record_pyobject.h b/include/pybind11/detail/function_record_pyobject.h index 113054b3b6..de637a4c18 100644 --- a/include/pybind11/detail/function_record_pyobject.h +++ b/include/pybind11/detail/function_record_pyobject.h @@ -26,11 +26,7 @@ struct function_record_PyObject { }; inline function_record_PyObject *function_record_data(PyObject *obj) { -#if defined(PYBIND11_OPAQUE_PYOBJECT) - return static_cast(PyObject_GetTypeData(obj, Py_TYPE(obj))); -#else - return reinterpret_cast(obj); -#endif + return type_data(obj); } PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods) diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index 9c7c6ee258..4c341e7d8b 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -312,7 +312,7 @@ struct constructor { cl.def( "__init__", [](value_and_holder &v_h, Args... args) { - if (Py_TYPE(instance_object(v_h.inst)) == v_h.type->type) { + if (v_h.type_is_exact()) { v_h.value_ptr() = construct_or_initialize>(std::forward(args)...); } else { @@ -399,9 +399,7 @@ struct factory { [func] #endif (value_and_holder &v_h, Args... args) { - construct(v_h, - func(std::forward(args)...), - Py_TYPE(instance_object(v_h.inst)) != v_h.type->type); + construct(v_h, func(std::forward(args)...), !v_h.type_is_exact()); }, is_new_style_constructor(), extra...); @@ -448,7 +446,7 @@ struct factory { [class_func, alias_func] #endif (value_and_holder &v_h, CArgs... args) { - if (Py_TYPE(instance_object(v_h.inst)) == v_h.type->type) { + if (v_h.type_is_exact()) { // If the instance type equals the registered type we don't have inheritance, // so don't need the alias and can construct using the class function: construct(v_h, class_func(std::forward(args)...), false); @@ -536,9 +534,7 @@ struct pickle_factory { [func] #endif (value_and_holder &v_h, ArgState state) { - setstate(v_h, - func(std::forward(state)), - Py_TYPE(instance_object(v_h.inst)) != v_h.type->type); + setstate(v_h, func(std::forward(state)), !v_h.type_is_exact()); }, is_new_style_constructor(), extra...); diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 455feb53a5..9979c94793 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -232,7 +232,7 @@ inline PyThreadState *get_thread_state_unchecked() { #elif defined(Py_LIMITED_API) // PyThreadState_GetUnchecked() is not part of the stable ABI. return PyGILState_GetThisThreadState(); -#elif PY_VERSION_HEX < 0x030D0000 && !defined(Py_LIMITED_API) +#elif PY_VERSION_HEX < 0x030D0000 return _PyThreadState_UncheckedGet(); #else return PyThreadState_GetUnchecked(); diff --git a/include/pybind11/detail/type_caster_base-inl.h b/include/pybind11/detail/type_caster_base-inl.h index 83663ee11e..5b008be930 100644 --- a/include/pybind11/detail/type_caster_base-inl.h +++ b/include/pybind11/detail/type_caster_base-inl.h @@ -238,16 +238,12 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle get_type_handle(const std::type_in return handle(type_info ? (reinterpret_cast(type_info->type)) : nullptr); } +#if !defined(PYBIND11_OPAQUE_PYOBJECT) // the registry goes through PyWeakref_GetRef() instead PYBIND11_INLINE bool try_incref(PyObject *obj) { // Tries to increment the reference count of an object if it's not zero. -#if defined(PYBIND11_OPAQUE_PYOBJECT) - // Not used: the registry goes through PyWeakref_GetRef() instead. - assert(Py_REFCNT(obj) > 0); - Py_INCREF(obj); - return true; -#elif defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030E00A4 +# if defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030E00A4 return PyUnstable_TryIncRef(obj); -#elif defined(Py_GIL_DISABLED) +# elif defined(Py_GIL_DISABLED) // See // https://github.com/python/cpython/blob/d05140f9f77d7dfc753dd1e5ac3a5962aaa03eff/Include/internal/pycore_object.h#L761 uint32_t local = _Py_atomic_load_uint32_relaxed(&obj->ob_ref_local); @@ -258,9 +254,9 @@ PYBIND11_INLINE bool try_incref(PyObject *obj) { } if (_Py_IsOwnedByCurrentThread(obj)) { _Py_atomic_store_uint32_relaxed(&obj->ob_ref_local, local); -# ifdef Py_REF_DEBUG +# ifdef Py_REF_DEBUG _Py_INCREF_IncRefTotal(); -# endif +# endif return true; } Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&obj->ob_ref_shared); @@ -273,18 +269,19 @@ PYBIND11_INLINE bool try_incref(PyObject *obj) { if (_Py_atomic_compare_exchange_ssize( &obj->ob_ref_shared, &shared, shared + (1 << _Py_REF_SHARED_SHIFT))) { -# ifdef Py_REF_DEBUG +# ifdef Py_REF_DEBUG _Py_INCREF_IncRefTotal(); -# endif +# endif return true; } } -#else +# else assert(Py_REFCNT(obj) > 0); Py_INCREF(obj); return true; -#endif +# endif } +#endif PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle find_registered_python_instance(void *src, const detail::type_info *tinfo) { diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index ae0c630828..ab8b8e6268 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -125,11 +125,17 @@ detail::type_info *get_type_info(const std::type_info &tp, bool throw_if_missing handle get_type_handle(const std::type_info &tp, bool throw_if_missing); +#if !defined(PYBIND11_OPAQUE_PYOBJECT) bool try_incref(PyObject *obj); +#endif // Searches the inheritance graph for a registered Python instance, using all_type_info(). handle find_registered_python_instance(void *src, const detail::type_info *tinfo); +inline bool value_and_holder::type_is_exact() const { + return Py_TYPE(instance_object(inst)) == type->type; +} + // Container for accessing and iterating over an instance's values/holders struct values_and_holders { private: diff --git a/include/pybind11/detail/value_and_holder.h b/include/pybind11/detail/value_and_holder.h index 98ab73c378..df484edb32 100644 --- a/include/pybind11/detail/value_and_holder.h +++ b/include/pybind11/detail/value_and_holder.h @@ -25,6 +25,10 @@ struct value_and_holder { vh{inst->simple_layout ? inst->simple_value_holder : &inst->nonsimple.values_and_holders[vpos]} {} + /// True if the Python object is exactly the bound type, i.e. not a Python subclass that + /// may need the alias (trampoline) type. + bool type_is_exact() const; + // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) value_and_holder() = default; diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index 062d501e48..8700b0dbf8 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -508,8 +508,7 @@ PYBIND11_INLINE function get_type_override(const void *this_ptr, if (std::string(str(f_code.attr("co_name"))) == name && f_code.attr("co_argcount").cast() > 0) { object self_arg = f_code.attr("co_varnames")[int_(0)]; - auto locals - = reinterpret_steal(PyObject_GetAttrString(frame_obj.ptr(), "f_locals")); + object locals = getattr(frame_obj, "f_locals", handle()); auto self_caller = locals ? reinterpret_steal(PyObject_GetItem(locals.ptr(), self_arg.ptr())) @@ -1330,7 +1329,7 @@ PYBIND11_INLINE void generic_type::initialize(const type_record &rec) { tinfo->module_local = rec.module_local; tinfo->holder_enum_v = rec.holder_enum_v; #if defined(Py_LIMITED_API) - tinfo->dictoffset = handle(m_ptr).attr("__dictoffset__").cast(); + tinfo->dictoffset = type_dictoffset(reinterpret_cast(m_ptr)); #endif with_internals([&](internals &internals) { diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 312cecfb89..ee861b35b5 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -2093,10 +2093,8 @@ class enum_ : public class_ { def("__index__", [](Type value) { return (Scalar) value; }, pos_only()); attr("__setstate__") = cpp_function( [](detail::value_and_holder &v_h, Scalar arg) { - detail::initimpl::setstate(v_h, - static_cast(arg), - Py_TYPE(instance_object(v_h.inst)) - != v_h.type->type); + detail::initimpl::setstate( + v_h, static_cast(arg), !v_h.type_is_exact()); }, detail::is_new_style_constructor(), pybind11::name("__setstate__"), diff --git a/include/pybind11/pytypes-inl.h b/include/pybind11/pytypes-inl.h index 699fb5323d..c4ea0e5564 100644 --- a/include/pybind11/pytypes-inl.h +++ b/include/pybind11/pytypes-inl.h @@ -137,14 +137,8 @@ PYBIND11_INLINE std::string error_fetch_and_normalize::format_value_and_trace() if (m_trace) { #if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) // Attribute access instead of struct access: the traceback, frame and code object - // layouts are not part of the stable ABI. Returns a null object on failure. - auto get_attr = [](handle obj, const char *name) { - auto value = reinterpret_steal(PyObject_GetAttrString(obj.ptr(), name)); - if (!value) { - PyErr_Clear(); - } - return value; - }; + // layouts are not part of the stable ABI. A missing attribute gives a null object. + auto get_attr = [](handle obj, const char *name) { return getattr(obj, name, handle()); }; // Get the deepest trace possible. object tb = m_trace; diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 5fdc7627a4..33493f6921 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -516,6 +517,14 @@ inline std::string get_tp_name(PyTypeObject *type) { return type->tp_name; } inline bool tp_name_equals(PyTypeObject *type, const char *name) { return std::strcmp(type->tp_name, name) == 0; } +inline bool tp_name_is_one_of(PyTypeObject *type, std::initializer_list names) { + for (const char *name : names) { + if (tp_name_equals(type, name)) { + return true; + } + } + return false; +} #else bool type_is_managed_by_our_internals(PyTypeObject *type_obj); // detail/cpp_conduit.h @@ -546,9 +555,9 @@ inline std::string get_tp_name(PyTypeObject *type) { return module_ + '.' + name; } -inline bool tp_name_equals(PyTypeObject *type, const char *name) { - // Cheap pre-check on the unqualified name before reconstructing the full one. - const char *dot = std::strrchr(name, '.'); +/// True if `get_tp_name(type)` is one of `names`. The unqualified `__name__` is fetched once and +/// compared first; the full name is only reconstructed on a match. +inline bool tp_name_is_one_of(PyTypeObject *type, std::initializer_list names) { auto type_name = reinterpret_steal(PyType_GetName(type)); const char *type_name_utf8 = type_name ? PyUnicode_AsUTF8AndSize(type_name.ptr(), nullptr) : nullptr; @@ -556,10 +565,16 @@ inline bool tp_name_equals(PyTypeObject *type, const char *name) { PyErr_Clear(); return false; } - if (std::strcmp(type_name_utf8, dot ? dot + 1 : name) != 0) { - return false; + for (const char *name : names) { + const char *dot = std::strrchr(name, '.'); + if (std::strcmp(type_name_utf8, dot ? dot + 1 : name) == 0 && get_tp_name(type) == name) { + return true; + } } - return get_tp_name(type) == name; + return false; +} +inline bool tp_name_equals(PyTypeObject *type, const char *name) { + return tp_name_is_one_of(type, {name}); } #endif @@ -572,7 +587,6 @@ inline void tuple_set_item(PyObject *tup, ssize_t i, PyObject *item) { PyTuple_SET_ITEM(tup, i, item); } inline ssize_t list_size(PyObject *lst) { return PyList_GET_SIZE(lst); } -inline PyObject *list_get_item(PyObject *lst, ssize_t i) { return PyList_GET_ITEM(lst, i); } /// Steals a reference to `item`; `lst` must be a fresh list. inline void list_set_item(PyObject *lst, ssize_t i, PyObject *item) { PyList_SET_ITEM(lst, i, item); @@ -584,7 +598,6 @@ inline void tuple_set_item(PyObject *tup, ssize_t i, PyObject *item) { PyTuple_SetItem(tup, i, item); } inline ssize_t list_size(PyObject *lst) { return PyList_Size(lst); } -inline PyObject *list_get_item(PyObject *lst, ssize_t i) { return PyList_GetItem(lst, i); } inline void list_set_item(PyObject *lst, ssize_t i, PyObject *item) { PyList_SetItem(lst, i, item); } @@ -2619,7 +2632,6 @@ PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator>>=, PyNumber_InPlaceRshift) #undef PYBIND11_MATH_OPERATOR_BINARY #undef PYBIND11_MATH_OPERATOR_BINARY_INPLACE -// Meant to return a Python str, but this is not checked. #if !defined(Py_LIMITED_API) /// `type.__bases__` as a tuple (empty if the type has no bases tuple yet). inline tuple get_bases(PyTypeObject *type) { @@ -2633,25 +2645,51 @@ inline tuple get_bases(PyTypeObject *type) { inline tuple get_mro(PyTypeObject *type) { return reinterpret_borrow(type->tp_mro); } /// Look `name` up along the MRO of `type` without invoking descriptors, like `_PyType_Lookup`. -/// Returns a null object if the attribute is not found; never raises. -inline object type_lookup(PyTypeObject *type, handle name) { - return reinterpret_borrow(_PyType_Lookup(type, name.ptr())); +/// Returns a null handle if the attribute is not found; never raises. (Borrowed here; the +/// stable-ABI version below returns an owning `object`.) +inline handle type_lookup(PyTypeObject *type, handle name) { + return _PyType_Lookup(type, name.ptr()); } #else +inline PyObject *interned_name(const char *name) { + PyObject *obj = PyUnicode_InternFromString(name); + if (obj == nullptr) { + throw error_already_set(); + } + return obj; // kept alive on purpose +} + // Generic attribute lookup on a type object: resolves the `type.__xxx__` descriptors without // going through the metaclass's tp_getattro (pybind11's own metaclass calls back into here). -inline object type_generic_getattr(PyTypeObject *type, const char *name) { - auto name_obj = reinterpret_steal(PyUnicode_InternFromString(name)); - auto value = name_obj ? reinterpret_steal( - PyObject_GenericGetAttr((PyObject *) type, name_obj.ptr())) - : object(); +// Returns a null object if the attribute is not found; never raises. +inline object type_generic_getattr(PyTypeObject *type, handle name) { + auto value = reinterpret_steal(PyObject_GenericGetAttr((PyObject *) type, name.ptr())); if (!value) { PyErr_Clear(); } return value; } -inline tuple get_type_tuple_attr(PyTypeObject *type, const char *name) { +inline Py_ssize_t type_ssize_attr(PyTypeObject *type, handle name) { + object value = type_generic_getattr(type, name); + Py_ssize_t result = value ? PyLong_AsSsize_t(value.ptr()) : -1; + if (result == -1 && PyErr_Occurred()) { + throw error_already_set(); + } + return result; +} + +/// `type.__dictoffset__` / `type.__basicsize__` (tp_dictoffset and tp_basicsize). +inline Py_ssize_t type_dictoffset(PyTypeObject *type) { + static PyObject *name = interned_name("__dictoffset__"); + return type_ssize_attr(type, name); +} +inline Py_ssize_t type_basicsize(PyTypeObject *type) { + static PyObject *name = interned_name("__basicsize__"); + return type_ssize_attr(type, name); +} + +inline tuple type_tuple_attr(PyTypeObject *type, handle name) { object value = type_generic_getattr(type, name); if (!value || !PyTuple_Check(value.ptr())) { return tuple(); @@ -2659,24 +2697,14 @@ inline tuple get_type_tuple_attr(PyTypeObject *type, const char *name) { return reinterpret_steal(value.release()); } -inline tuple get_bases(PyTypeObject *type) { return get_type_tuple_attr(type, "__bases__"); } - -inline PyObject *interned_name(const char *name) { - PyObject *obj = PyUnicode_InternFromString(name); - if (obj == nullptr) { - throw error_already_set(); - } - return obj; // kept alive on purpose +inline tuple get_bases(PyTypeObject *type) { + static PyObject *name = interned_name("__bases__"); + return type_tuple_attr(type, name); } inline tuple get_mro(PyTypeObject *type) { static PyObject *name = interned_name("__mro__"); - auto value = reinterpret_steal(PyObject_GenericGetAttr((PyObject *) type, name)); - if (!value || !PyTuple_Check(value.ptr())) { - PyErr_Clear(); - return tuple(); - } - return reinterpret_steal(value.release()); + return type_tuple_attr(type, name); } // No type attribute cache: walks the MRO and looks in each class's `__dict__` (a mappingproxy). @@ -2705,6 +2733,7 @@ inline object type_lookup(PyTypeObject *type, handle name) { } #endif +// Meant to return a Python str, but this is not checked. inline object get_module_name_if_available(handle scope) { if (scope) { if (hasattr(scope, "__module__")) { diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index d1a5eb652d..a31bc36ec8 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -79,12 +79,7 @@ inline bool object_is_instance_with_one_of_tp_names(PyObject *obj, if (PyType_Check(obj)) { return false; } - for (const auto *tp_name : tp_names) { - if (tp_name_equals(Py_TYPE(obj), tp_name)) { - return true; - } - } - return false; + return tp_name_is_one_of(Py_TYPE(obj), tp_names); } inline bool object_is_convertible_to_std_vector(const handle &src) { @@ -93,11 +88,6 @@ inline bool object_is_convertible_to_std_vector(const handle &src) { return !PyUnicode_Check(src.ptr()) && !PyBytes_Check(src.ptr()); } // Allow generators, set/frozenset and several common iterable types. -#if !defined(Py_LIMITED_API) - if (PyGen_Check(src.ptr()) != 0) { - return true; - } -#endif return (PyAnySet_Check(src.ptr()) != 0) || object_is_instance_with_one_of_tp_names( src.ptr(), {"generator", "dict_keys", "dict_values", "dict_items", "map", "zip"}); diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index 8d204afd39..eeb15b83ef 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -552,11 +552,11 @@ endfunction() # PYBIND11_PRECOMPILE variable requests it, unless NO_PRECOMPILE opts out. function(_pybind11_maybe_precompile target_name precompile no_precompile stable_abi) if((precompile OR PYBIND11_PRECOMPILE) AND NOT no_precompile) + set(_kw "") if(stable_abi) - pybind11_precompile(STABLE_ABI) - else() - pybind11_precompile() + set(_kw STABLE_ABI) endif() + pybind11_precompile(${_kw}) target_link_libraries(${target_name} PRIVATE pybind11::precompiled) endif() endfunction() From 5544d87b32864b265415b0278a7a26b1dbd66e5a Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 22:38:34 -0400 Subject: [PATCH 30/45] fix(stable-abi): gate 3.13+ API use on Py_LIMITED_API, not the headers A stable-ABI build targeting 3.12 with 3.13+ headers selected PyDict_GetItemStringRef, PyDict_SetDefaultRef, PyEval_GetFrameBuiltins and PyEval_GetFrameGlobals, which the headers hide below their limited API version. Add PYBIND11_API_VERSION_HEX (Py_LIMITED_API when defined, else PY_VERSION_HEX) and use it in those guards. annotations() always takes the 3.14 path under the stable ABI, as the module can run there. Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/common.h | 4 ++++ include/pybind11/pybind11-inl.h | 4 ++-- include/pybind11/pytypes.h | 9 +++++---- tools/check_inl_headers.py | 1 + 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index e4616a4121..5712653e6f 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -43,6 +43,10 @@ # elif PYBIND11_HAS_SUBINTERPRETER_SUPPORT # error "Subinterpreter support is not available under Py_LIMITED_API." # endif +// The C API version the build may use: the limited API target, not the headers' version. +# define PYBIND11_API_VERSION_HEX Py_LIMITED_API +#else +# define PYBIND11_API_VERSION_HEX PY_VERSION_HEX #endif // Heap types declare their extra data as PEP 697 type data when PyObject is opaque: a negative diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index 8700b0dbf8..99787e92eb 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -249,7 +249,7 @@ PYBIND11_INLINE PyObject *cached_create_module(PyObject *spec, PyModuleDef *) { PYBIND11_NAMESPACE_END(detail) PYBIND11_INLINE dict globals() { -#if PY_VERSION_HEX >= 0x030d0000 +#if PYBIND11_API_VERSION_HEX >= 0x030d0000 PyObject *p = PyEval_GetFrameGlobals(); return p ? reinterpret_steal(p) : reinterpret_borrow(module_::import("__main__").attr("__dict__").ptr()); @@ -398,7 +398,7 @@ PYBIND11_INLINE void register_local_exception_translator(ExceptionTranslator &&t PYBIND11_NAMESPACE_BEGIN(detail) PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void print(const tuple &args, const dict &kwargs) { -#if PY_VERSION_HEX >= 0x030D0000 +#if PYBIND11_API_VERSION_HEX >= 0x030D0000 auto builtins = reinterpret_steal(PyEval_GetFrameBuiltins()); #else auto builtins = reinterpret_borrow(PyEval_GetBuiltins()); diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 33493f6921..0163d3369d 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -900,7 +900,7 @@ inline PyObject *dict_getitem(PyObject *v, PyObject *key) { // PyDict_GetItemStringRef was added in Python 3.13.0a1. // See also: https://github.com/python/pythoncapi-compat/blob/main/pythoncapi_compat.h inline PyObject *dict_getitemstringref(PyObject *v, const char *key) { -#if PY_VERSION_HEX >= 0x030D00A1 +#if PYBIND11_API_VERSION_HEX >= 0x030D00A1 PyObject *rv = nullptr; if (PyDict_GetItemStringRef(v, key, &rv) < 0) { throw error_already_set(); @@ -940,7 +940,7 @@ inline PyObject *dict_setdefaultstring(PyObject *v, const char *key, PyObject *d // PyDict_SetDefaultRef was added in Python 3.13.0a4. // See also: https://github.com/python/pythoncapi-compat/blob/main/pythoncapi_compat.h inline PyObject *dict_setdefaultstringref(PyObject *v, const char *key, PyObject *defaultobj) { -#if PY_VERSION_HEX >= 0x030D00A4 +#if PYBIND11_API_VERSION_HEX >= 0x030D00A4 PyObject *kv = PyUnicode_FromString(key); if (kv == nullptr) { throw error_already_set(); @@ -2554,8 +2554,9 @@ str_attr_accessor object_api::doc() const { template object object_api::annotations() const { -// This is needed again because of the lazy annotations added in 3.14+ -#if PY_VERSION_HEX < 0x030A0000 || PY_VERSION_HEX >= 0x030E0000 +// This is needed again because of the lazy annotations added in 3.14+ (a stable-ABI module +// built for an older target can still run on 3.14+) +#if PY_VERSION_HEX < 0x030A0000 || PY_VERSION_HEX >= 0x030E0000 || defined(Py_LIMITED_API) // https://docs.python.org/3/howto/annotations.html#accessing-the-annotations-dict-of-an-object-in-python-3-9-and-older if (!hasattr(derived(), "__annotations__")) { setattr(derived(), "__annotations__", dict()); diff --git a/tools/check_inl_headers.py b/tools/check_inl_headers.py index 9063f078b9..20ba1b64e0 100755 --- a/tools/check_inl_headers.py +++ b/tools/check_inl_headers.py @@ -28,6 +28,7 @@ "PY_MAJOR_VERSION", "PY_MINOR_VERSION", "PY_VERSION_HEX", + "PYBIND11_API_VERSION_HEX", # Py_LIMITED_API if defined, else PY_VERSION_HEX "Py_REF_DEBUG", "_MSC_VER", "_WIN32", From 09de9c111e48f6ba042ffd9d339ca698cdbdf625 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 22:48:08 -0400 Subject: [PATCH 31/45] perf(stable-abi): make instancemethod a vectorcall method descriptor With Py_TPFLAGS_METHOD_DESCRIPTOR and Py_TPFLAGS_HAVE_VECTORCALL (both in the 3.12 stable ABI), `obj.method(...)` calls the instancemethod with `obj` prepended instead of creating a types.MethodType first. `obj.method` without a call is unchanged. Release, CPython 3.12, `d.name()`: 77 ns regular, 45 ns abi3 (was 63). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 19 ++++++++++++++++++- include/pybind11/detail/class.h | 2 ++ tests/test_methods_and_attributes.py | 9 +++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 235cdaf219..f620e014a7 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -494,6 +494,7 @@ struct instancemethod_object { PyObject_HEAD # endif PyObject *func; + vectorcallfunc vectorcall; }; PYBIND11_INLINE instancemethod_object *instancemethod_data(PyObject *self) { @@ -536,6 +537,15 @@ instancemethod_call(PyObject *self, PyObject *args, PyObject *kwargs) { return PyObject_Call(instancemethod_data(self)->func, args, kwargs); } +// Py_TPFLAGS_METHOD_DESCRIPTOR: `obj.method(...)` calls this with `obj` prepended instead of +// creating a bound method. +extern "C" PYBIND11_INLINE PyObject *instancemethod_vectorcall(PyObject *self, + PyObject *const *args, + size_t nargsf, + PyObject *kwnames) { + return PyObject_Vectorcall(instancemethod_data(self)->func, args, nargsf, kwnames); +} + extern "C" PYBIND11_INLINE PyObject *instancemethod_getattro(PyObject *self, PyObject *name) { // Descriptors of the instancemethod type itself (`__func__`, `__class__`, ...) win; // everything else (`__name__`, `__doc__`, `__module__`, ...) comes from the function. @@ -587,6 +597,11 @@ PYBIND11_INLINE PyTypeObject *get_instancemethod_type() { offsetof(instancemethod_object, func), Py_READONLY | PYBIND11_MEMBER_OFFSET_FLAGS, nullptr}, + {"__vectorcalloffset__", + Py_T_PYSSIZET, + offsetof(instancemethod_object, vectorcall), + Py_READONLY | PYBIND11_MEMBER_OFFSET_FLAGS, + nullptr}, {nullptr, 0, 0, 0, nullptr}}; static PyType_Slot slots[] = {{Py_tp_descr_get, reinterpret_cast(instancemethod_descr_get)}, @@ -602,7 +617,8 @@ PYBIND11_INLINE PyTypeObject *get_instancemethod_type() { = {PYBIND11_DUMMY_MODULE_NAME ".instancemethod", PYBIND11_TYPE_DATA_SIZE(instancemethod_object), 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_METHOD_DESCRIPTOR | Py_TPFLAGS_HAVE_VECTORCALL, slots}; internals.instancemethod_type = type_from_spec("get_instancemethod_type()", nullptr, &spec, nullptr); @@ -627,6 +643,7 @@ PYBIND11_INLINE PyObject *instancemethod_new(PyObject *func) { } Py_INCREF(func); instancemethod_data(self)->func = func; + instancemethod_data(self)->vectorcall = instancemethod_vectorcall; return self; } diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 61b639e30d..708eace403 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -137,6 +137,8 @@ struct instancemethod_object; instancemethod_object *instancemethod_data(PyObject *self); extern "C" PyObject *instancemethod_descr_get(PyObject *self, PyObject *obj, PyObject *type); extern "C" PyObject *instancemethod_call(PyObject *self, PyObject *args, PyObject *kwargs); +extern "C" PyObject * +instancemethod_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames); extern "C" PyObject *instancemethod_getattro(PyObject *self, PyObject *name); extern "C" PyObject *instancemethod_repr(PyObject *self); extern "C" int instancemethod_traverse(PyObject *self, visitproc visit, void *arg); diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index ff5fecdb80..2ee050d574 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -106,6 +106,15 @@ class Sub(m.ExampleMandA): Sub.alias = m.ExampleMandA.internal1 assert Sub(9).alias() == 9 + # Py_TPFLAGS_METHOD_DESCRIPTOR: `obj.method(...)` skips the bound method + assert type(unbound).__flags__ & (1 << 17) + bound = m.ExampleMandA(5).overloaded + assert bound.__self__.internal1() == 5 + assert bound.__func__ is m.ExampleMandA.overloaded.__func__ + assert bound(1, 1.0) == "(int, float)" + assert m.ExampleMandA(5).overloaded(1, 1.0) == "(int, float)" + assert m.ExampleMandA.overloaded(m.ExampleMandA(5), 1, 1.0) == "(int, float)" + def test_copy_method(): """Issue #443: calling copied methods fails in Python 3""" From 691f0bd4b2735c1f2bfa23e86dcd62a7f9fa85d0 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 23:20:22 -0400 Subject: [PATCH 32/45] fix(chrono): import the datetime C API per translation unit `PyDateTimeAPI` is a static per translation unit. The non-limited helpers had external linkage, so the linker could pair one unit's `datetime_import()` with another unit's `is_datetime()`, which then read a null pointer. Give the helpers internal linkage and add a second chrono translation unit to the test module. Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/chrono.h | 18 ++++++++++-------- tests/CMakeLists.txt | 1 + tests/test_chrono.py | 8 ++++++++ tests/test_chrono_second_tu.cpp | 23 +++++++++++++++++++++++ 4 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 tests/test_chrono_second_tu.cpp diff --git a/include/pybind11/chrono.h b/include/pybind11/chrono.h index d7f801a4b9..cc8e1dd3b6 100644 --- a/include/pybind11/chrono.h +++ b/include/pybind11/chrono.h @@ -89,21 +89,23 @@ make_datetime(int year, int month, int day, int hour, int minute, int second, in microsecond); } #else -inline void datetime_import() { +// `PyDateTimeAPI` is a per-translation-unit static: these helpers get internal linkage so the +// linker cannot pair one unit's import with another unit's (null) pointer. +static inline void datetime_import() { if (!PyDateTimeAPI) { PyDateTime_IMPORT; } } -inline bool is_timedelta(handle h) { return PyDelta_Check(h.ptr()); } -inline bool is_datetime(handle h) { return PyDateTime_Check(h.ptr()); } -inline bool is_date(handle h) { return PyDate_Check(h.ptr()); } -inline bool is_time(handle h) { return PyTime_Check(h.ptr()); } +static inline bool is_timedelta(handle h) { return PyDelta_Check(h.ptr()); } +static inline bool is_datetime(handle h) { return PyDateTime_Check(h.ptr()); } +static inline bool is_date(handle h) { return PyDate_Check(h.ptr()); } +static inline bool is_time(handle h) { return PyTime_Check(h.ptr()); } # define PYBIND11_DATETIME_ACCESSOR(fn, attr, c_macro) \ - inline int fn(handle h) { return c_macro(h.ptr()); } -inline PyObject *make_timedelta(int days, int seconds, int microseconds) { + static inline int fn(handle h) { return c_macro(h.ptr()); } +static inline PyObject *make_timedelta(int days, int seconds, int microseconds) { return PyDelta_FromDSU(days, seconds, microseconds); } -inline PyObject * +static inline PyObject * make_datetime(int year, int month, int day, int hour, int minute, int second, int microsecond) { return PyDateTime_FromDateAndTime(year, month, day, hour, minute, second, microsecond); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b52324c3d7..0d8cf4ed79 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -130,6 +130,7 @@ set(PYBIND11_TEST_FILES test_builtin_casters test_call_policies test_callbacks + test_chrono_second_tu.cpp test_chrono test_class test_class_cross_module_use_after_one_module_dealloc diff --git a/tests/test_chrono.py b/tests/test_chrono.py index ed889fbd6b..66d9453bd2 100644 --- a/tests/test_chrono.py +++ b/tests/test_chrono.py @@ -6,6 +6,7 @@ import env # noqa: F401 from pybind11_tests import chrono as m +from pybind11_tests import chrono_second_tu as m2 def test_chrono_system_clock(): @@ -29,6 +30,13 @@ def test_chrono_system_clock(): assert diff.seconds <= diff_python.seconds +def test_chrono_second_translation_unit(): + """The datetime C API import must work per translation unit (two-TU crash)""" + delta = datetime.timedelta(seconds=1, microseconds=5) + assert m2.duration_roundtrip(delta) == delta + assert m.test_chrono2(m.test_chrono1()) is not None + + def test_chrono_system_clock_roundtrip(): date1 = datetime.datetime.today() diff --git a/tests/test_chrono_second_tu.cpp b/tests/test_chrono_second_tu.cpp new file mode 100644 index 0000000000..14f18e16df --- /dev/null +++ b/tests/test_chrono_second_tu.cpp @@ -0,0 +1,23 @@ +/* + tests/test_chrono_second_tu.cpp -- chrono casters from a second translation unit + + Only durations are used here. If the datetime helpers have external linkage, the linker can + pair this TU's `datetime_import()` with test_chrono.cpp's `is_datetime()`, which then reads + a null `PyDateTimeAPI`. + + Copyright (c) 2026 Henry Schreiner + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include + +#include "pybind11_tests.h" + +#include + +TEST_SUBMODULE(chrono_second_tu, m) { + m.def("duration_roundtrip", + [](const std::chrono::system_clock::duration &d) { return d; }); +} From 6c99d851884bd0678df3cf45c52fe649de4e27f5 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 23:20:22 -0400 Subject: [PATCH 33/45] fix(stable-abi): keep instance lookup safe across critical-section suspension Regular free-threaded builds lock the instance-map shards with PyMutex again, which is never suspended. Under abi3t only critical sections are available; `find_registered_python_instance()` now takes strong references under the shard lock and checks the types after releasing it, since `all_type_info()` can suspend the section while waiting for the internals lock. Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/internals.h | 6 ++++ .../pybind11/detail/type_caster_base-inl.h | 33 ++++++++++++++----- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 9979c94793..8c807a59cf 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -970,7 +970,13 @@ inline auto with_instance_map(const void *ptr, const F &cb) auto idx = static_cast(hash & internals.instance_shards_mask); auto &shard = internals.instance_shards[idx]; +# if defined(PYBIND11_OPAQUE_PYOBJECT) + // A critical section is suspended while `cb` waits for another one (e.g. the internals + // lock): `cb` must not keep iterators or instance pointers across such calls. pycritical_section lock(shard.mutex); +# else + std::unique_lock lock(shard.mutex); +# endif return cb(shard.registered_instances); #else (void) ptr; diff --git a/include/pybind11/detail/type_caster_base-inl.h b/include/pybind11/detail/type_caster_base-inl.h index 5b008be930..d7f86d5cc8 100644 --- a/include/pybind11/detail/type_caster_base-inl.h +++ b/include/pybind11/detail/type_caster_base-inl.h @@ -285,29 +285,46 @@ PYBIND11_INLINE bool try_incref(PyObject *obj) { PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle find_registered_python_instance(void *src, const detail::type_info *tinfo) { +#if defined(PYBIND11_OPAQUE_PYOBJECT) + // all_type_info() takes the internals lock, which can suspend the instance map's critical + // section: take strong references first, then check the types without the map lock. + std::vector candidates = with_instance_map(src, [&](instance_map &instances) { + std::vector found; + auto it_instances = instances.equal_range(src); + for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { + PyObject *wrapper = nullptr; + if (it_i->second->registry_weakref != nullptr + && PyWeakref_GetRef(it_i->second->registry_weakref, &wrapper) == 1) { + found.emplace_back(reinterpret_steal(wrapper)); + } + } + return found; + }); + for (auto &wrapper : candidates) { + for (auto *instance_type : detail::all_type_info(Py_TYPE(wrapper.ptr()))) { + if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) { + return wrapper.release(); + } + } + } + return handle(); +#else return with_instance_map(src, [&](instance_map &instances) { auto it_instances = instances.equal_range(src); for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { for (auto *instance_type : detail::all_type_info(Py_TYPE(instance_object(it_i->second)))) { if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) { -#if defined(PYBIND11_OPAQUE_PYOBJECT) - PyObject *wrapper = nullptr; - if (it_i->second->registry_weakref != nullptr - && PyWeakref_GetRef(it_i->second->registry_weakref, &wrapper) == 1) { - return handle(wrapper); - } -#else auto *wrapper = instance_object(it_i->second); if (try_incref(wrapper)) { return handle(wrapper); } -#endif } } } return handle(); }); +#endif } PYBIND11_NOINLINE_ATTR PYBIND11_INLINE value_and_holder From 0667f6abca6e9447884a51bb08d25f9a55abee7e Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 23:20:22 -0400 Subject: [PATCH 34/45] test(stable-abi): check the method-descriptor flag only under the limited API CPython's `PyInstanceMethod_Type`, used in regular builds, does not set `Py_TPFLAGS_METHOD_DESCRIPTOR`. Assisted-by: ClaudeCode:claude-fable-5-1 --- tests/test_methods_and_attributes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index 2ee050d574..ded30b54f5 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -106,8 +106,10 @@ class Sub(m.ExampleMandA): Sub.alias = m.ExampleMandA.internal1 assert Sub(9).alias() == 9 - # Py_TPFLAGS_METHOD_DESCRIPTOR: `obj.method(...)` skips the bound method - assert type(unbound).__flags__ & (1 << 17) + # Py_TPFLAGS_METHOD_DESCRIPTOR: `obj.method(...)` skips the bound method. Only pybind11's + # own instancemethod (limited API) has it; CPython's PyInstanceMethod_Type does not. + if env.LIMITED_API: + assert type(unbound).__flags__ & (1 << 17) bound = m.ExampleMandA(5).overloaded assert bound.__self__.internal1() == 5 assert bound.__func__ is m.ExampleMandA.overloaded.__func__ From 7084836319ced087f50845524e25e750df67ae9f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 23:20:22 -0400 Subject: [PATCH 35/45] fix(cmake): do not link libpython into SHARED STABLE_ABI libraries `python_add_library(SHARED)` always links `Python::Python`. Create the target with `add_library()` instead so the helper only depends on the stable ABI. Assisted-by: ClaudeCode:claude-fable-5-1 --- tools/pybind11NewTools.cmake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 5618776876..e48fbaa283 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -376,7 +376,10 @@ function(pybind11_add_module target_name) endif() endif() - if("${_Python}" STREQUAL "Python") + if(stable_abi AND lib_type STREQUAL "SHARED") + # python_add_library(SHARED) links the embedding library; a stable-ABI helper must not. + add_library(${target_name} SHARED ${ARG_UNPARSED_ARGUMENTS}) + elseif("${_Python}" STREQUAL "Python") python_add_library(${target_name} ${lib_type} ${use_sabi} ${ARG_UNPARSED_ARGUMENTS}) elseif("${_Python}" STREQUAL "Python3") python3_add_library(${target_name} ${lib_type} ${use_sabi} ${ARG_UNPARSED_ARGUMENTS}) From 1efa690fed820c261bcb01a3ba227bbf0e14c8f9 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 23:20:22 -0400 Subject: [PATCH 36/45] ci(stable-abi): resolve forward-test ignores relative to build/tests Assisted-by: ClaudeCode:claude-fable-5-1 --- .github/workflows/ci.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65a568ae7f..0479ab4b50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,10 +170,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 env: + # Relative to the working directory of the forward steps, build/tests PYBIND11_STABLE_ABI_SKIPPED_TESTS: >- - --ignore=tests/test_custom_type_setup.py - --ignore=tests/test_multiple_interpreters.py - --ignore=tests/test_class_release_gil_before_calling_cpp_dtor.py + --ignore=../../tests/test_custom_type_setup.py + --ignore=../../tests/test_multiple_interpreters.py + --ignore=../../tests/test_class_release_gil_before_calling_cpp_dtor.py steps: - uses: actions/checkout@v7 @@ -234,10 +235,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 env: + # Relative to the working directory of the forward steps, build/tests PYBIND11_STABLE_ABI_SKIPPED_TESTS: >- - --ignore=tests/test_custom_type_setup.py - --ignore=tests/test_multiple_interpreters.py - --ignore=tests/test_class_release_gil_before_calling_cpp_dtor.py + --ignore=../../tests/test_custom_type_setup.py + --ignore=../../tests/test_multiple_interpreters.py + --ignore=../../tests/test_class_release_gil_before_calling_cpp_dtor.py steps: - uses: actions/checkout@v7 From 2fac1f5ec64e49f61e9e0f0eff5b26dde252d9cc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 03:23:37 +0000 Subject: [PATCH 37/45] style: pre-commit fixes --- tests/test_chrono_second_tu.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_chrono_second_tu.cpp b/tests/test_chrono_second_tu.cpp index 14f18e16df..6f8a2fd4cb 100644 --- a/tests/test_chrono_second_tu.cpp +++ b/tests/test_chrono_second_tu.cpp @@ -18,6 +18,5 @@ #include TEST_SUBMODULE(chrono_second_tu, m) { - m.def("duration_roundtrip", - [](const std::chrono::system_clock::duration &d) { return d; }); + m.def("duration_roundtrip", [](const std::chrono::system_clock::duration &d) { return d; }); } From 569c3b17cd9b781e40cbec7621123eab27152e01 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 23:53:03 -0400 Subject: [PATCH 38/45] fix(pypy): use the accessors PyPy provides PyPy has `PyCFunction_GET_SELF` only as a macro and has no `PyThreadState_GetInterpreter()`. Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/common.h | 7 +++++++ include/pybind11/detail/internals.h | 16 ++++++++++++---- include/pybind11/functional.h | 2 +- include/pybind11/pybind11-inl.h | 4 ++-- include/pybind11/pybind11.h | 2 +- 5 files changed, 23 insertions(+), 8 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 5712653e6f..176fe26313 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -453,6 +453,13 @@ pybind11::detail::ensure_internals(); \ } +// PyPy has only the macro form. +#if defined(PYPY_VERSION) +# define PYBIND11_PYCFUNCTION_GET_SELF(func) PyCFunction_GET_SELF(func) +#else +# define PYBIND11_PYCFUNCTION_GET_SELF(func) PyCFunction_GetSelf(func) +#endif + #if !defined(GRAALVM_PYTHON) # define PYBIND11_PYCFUNCTION_GET_DOC(func) ((func)->m_ml->ml_doc) # define PYBIND11_PYCFUNCTION_SET_DOC(func, doc) \ diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 8c807a59cf..b9140442df 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -239,9 +239,17 @@ inline PyThreadState *get_thread_state_unchecked() { #endif } +inline PyInterpreterState *get_interpreter(PyThreadState *tstate) { +#if defined(PYPY_VERSION) // No PyThreadState_GetInterpreter() + return tstate->interp; +#else + return PyThreadState_GetInterpreter(tstate); +#endif +} + inline PyInterpreterState *get_interpreter_state_unchecked() { auto *tstate = get_thread_state_unchecked(); - return tstate ? PyThreadState_GetInterpreter(tstate) : nullptr; + return tstate ? get_interpreter(tstate) : nullptr; } object get_python_state_dict(); @@ -727,7 +735,7 @@ class internals_pp_manager { // internals_pp so that it can be pulled from the interpreter's state dict. That is // slow, so we use the current PyThreadState to check if it is necessary. auto *tstate = get_thread_state_unchecked(); - if (!tstate || PyThreadState_GetInterpreter(tstate) != last_istate_tls()) { + if (!tstate || get_interpreter(tstate) != last_istate_tls()) { gil_scoped_acquire_simple gil; if (!tstate) { tstate = get_thread_state_unchecked(); @@ -735,7 +743,7 @@ class internals_pp_manager { // Update the cache only on success; a stale interp with a null pp would make // later calls return nullptr. auto *pp = get_or_create_pp_in_state_dict(); - last_istate_tls() = PyThreadState_GetInterpreter(tstate); + last_istate_tls() = get_interpreter(tstate); internals_p_tls() = pp; } return internals_p_tls(); @@ -765,7 +773,7 @@ class internals_pp_manager { if (has_seen_non_main_interpreter()) { auto *tstate = get_thread_state_unchecked(); // this could be called without an active interpreter, just use what was cached - if (!tstate || PyThreadState_GetInterpreter(tstate) == last_istate_tls()) { + if (!tstate || get_interpreter(tstate) == last_istate_tls()) { auto tpp = internals_p_tls(); { std::lock_guard lock(pp_set_mutex_); diff --git a/include/pybind11/functional.h b/include/pybind11/functional.h index cfa0a88a08..3c169cb842 100644 --- a/include/pybind11/functional.h +++ b/include/pybind11/functional.h @@ -82,7 +82,7 @@ struct type_caster> { captured variables), in which case the roundtrip can be avoided. */ if (auto cfunc = func.cpp_function()) { - auto *cfunc_self = PyCFunction_GetSelf(cfunc.ptr()); + auto *cfunc_self = PYBIND11_PYCFUNCTION_GET_SELF(cfunc.ptr()); if (cfunc_self == nullptr) { PyErr_Clear(); } else { diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index 99787e92eb..2229717555 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -635,7 +635,7 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u detail::function_record *chain = nullptr, *chain_start = rec; if (rec->sibling) { if (PyCFunction_Check(rec->sibling.ptr())) { - auto *self = PyCFunction_GetSelf(rec->sibling.ptr()); + auto *self = PYBIND11_PYCFUNCTION_GET_SELF(rec->sibling.ptr()); if (self == nullptr) { pybind11_fail("initialize_generic: Unexpected nullptr from PyCFunction_GetSelf"); } @@ -697,7 +697,7 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u // chain. chain_start = rec; rec->next = chain; - auto *py_func_rec = detail::function_record_data(PyCFunction_GetSelf(m_ptr)); + auto *py_func_rec = detail::function_record_data(PYBIND11_PYCFUNCTION_GET_SELF(m_ptr)); py_func_rec->cpp_func_rec = unique_rec.release(); guarded_strdup.release(); } else { diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index ee861b35b5..2317a9a894 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1890,7 +1890,7 @@ class class_ : public detail::generic_type { return nullptr; } - handle func_self = PyCFunction_GetSelf(h.ptr()); + handle func_self = PYBIND11_PYCFUNCTION_GET_SELF(h.ptr()); if (!func_self) { throw error_already_set(); } From d86ba7d224f818b959fe6b35fce3220d9fb2e4ef Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 23:53:03 -0400 Subject: [PATCH 39/45] fix: MSVC C4456 shadowing and clang-tidy macro parentheses Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 4 ++-- include/pybind11/pybind11-inl.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index f620e014a7..672c65391f 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -46,12 +46,12 @@ PYBIND11_INLINE PyTypeObject *type_incref(PyTypeObject *type) { # define PYBIND11_BASE_TYPE_SLOT(fn, slot_type, type, slot_id, field) \ PYBIND11_INLINE slot_type fn() { \ static const auto cached \ - = reinterpret_cast(PyType_GetSlot(&type, slot_id)); \ + = reinterpret_cast(PyType_GetSlot(&(type), slot_id)); \ return cached; \ } #else # define PYBIND11_BASE_TYPE_SLOT(fn, slot_type, type, slot_id, field) \ - PYBIND11_INLINE slot_type fn() { return type.field; } + PYBIND11_INLINE slot_type fn() { return (type).field; } #endif PYBIND11_BASE_TYPE_SLOT(type_type_call, ternaryfunc, PyType_Type, Py_tp_call, tp_call) diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index 2229717555..8da7234198 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -776,13 +776,13 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u #endif if (rec->is_method) { - PyObject *func = m_ptr; + PyObject *cfunc = m_ptr; m_ptr = PYBIND11_INSTANCE_METHOD_NEW(m_ptr, rec->scope.ptr()); if (!m_ptr) { pybind11_fail( "cpp_function::cpp_function(): Could not allocate instance method object"); } - Py_DECREF(func); + Py_DECREF(cfunc); } } From 22ae75be4856c4f41b942012e8c9cf4b1eb47c51 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 8 Oct 2026 00:50:17 -0400 Subject: [PATCH 40/45] feat: build abi3t from GIL-enabled Python; fix stable ABI CI failures Add PYBIND11_ABI3T (CMake) and py_limited_api="3.15t" (setuptools) to target abi3t with Py_TARGET_ABI3T from GIL-enabled CPython 3.15+, with the .abi3t suffix and python3t.lib on Windows. Document Py_TARGET_ABI3T. Use `auto it` in the tuple/dict iterator tests under Py_LIMITED_API (as on PyPy), and build the cross-module RTTI bindings without the stable ABI so they share internals with the embedding executable. Assisted-by: ClaudeCode:claude-opus-5-5 --- .github/workflows/ci.yml | 71 +++++++++++++++++ docs/advanced/stable_abi.rst | 37 ++++++--- docs/compiling.rst | 7 +- pybind11/setup_helpers.py | 87 ++++++++++++++------- tests/extra_setuptools/test_setuphelper.py | 19 ++++- tests/test_cross_module_rtti/CMakeLists.txt | 5 +- tests/test_pytypes.cpp | 4 +- tools/pybind11Common.cmake | 7 +- tools/pybind11NewTools.cmake | 87 +++++++++++++++++---- 9 files changed, 260 insertions(+), 64 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0479ab4b50..1d3865ca35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -287,6 +287,77 @@ jobs: working-directory: build/tests run: python3.15 -m pytest -v ../../tests ${{ env.PYBIND11_STABLE_ABI_SKIPPED_TESTS }} + # PYBIND11_ABI3T builds abi3t from GIL-enabled Python; run the same modules on 3.15t. + stable-abi-abi3t-from-gil: + if: github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + runs-on: [ubuntu-latest, windows-latest] + name: 🧱 abi3t from GIL-enabled Python • ${{ matrix.runs-on }} + runs-on: ${{ matrix.runs-on }} + timeout-minutes: 60 + defaults: + run: + shell: bash + env: + # Relative to the working directory of the forward steps, build/tests + PYBIND11_STABLE_ABI_SKIPPED_TESTS: >- + --ignore=../../tests/test_custom_type_setup.py + --ignore=../../tests/test_multiple_interpreters.py + --ignore=../../tests/test_class_release_gil_before_calling_cpp_dtor.py + + steps: + - uses: actions/checkout@v7 + + - name: Setup Python 3.15t (run) + id: py315t + uses: actions/setup-python@v7 + with: + python-version: '3.15t' + allow-prereleases: true + + - name: Setup Python 3.15 (build) + id: py315 + uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + + - name: Install uv + uses: astral-sh/setup-uv@v10.0.1 + with: + enable-cache: true + + - name: Prepare env + run: | + for py in "${{ steps.py315t.outputs.python-path }}" "${{ steps.py315.outputs.python-path }}"; do + uv pip install --python="$py" --system -r tests/requirements.txt + done + + - name: Configure (Python 3.15, abi3t) + run: > + cmake -S. -Bbuild -Werror=dev + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DPYBIND11_TEST_STABLE_ABI=ON + -DPYBIND11_ABI3T=ON + -DCMAKE_BUILD_TYPE=Release + -DPYTHON_EXECUTABLE="${{ steps.py315.outputs.python-path }}" + + - name: Build + run: cmake --build build --config Release -j 4 + + - name: Python tests (3.15) + run: cmake --build build --config Release --target pytest + + - name: Python tests (3.15t, same binaries) + working-directory: build/tests + run: > + "${{ steps.py315t.outputs.python-path }}" -m pytest -v ../../tests + ${{ env.PYBIND11_STABLE_ABI_SKIPPED_TESTS }} + standard-large: if: github.event.pull_request.draft == false strategy: diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index c794219053..5b179b3b9e 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -8,11 +8,13 @@ built for. pybind11 supports it from CPython 3.12 on: define ``*.abi3.so`` / ``*.pyd`` per platform instead of one module per Python version. -Free-threaded CPython has its own stable ABI from 3.15 on, ``abi3t`` -(:pep:`803`). pybind11 supports it too: build against the free-threaded -headers with ``Py_LIMITED_API`` set to ``0x030F0000`` or newer, and ship one -``*.abi3t.so`` that every free-threaded *and* GIL-enabled CPython from 3.15 -on can load. See `abi3t`_ below for what differs. +CPython 3.15 adds a second stable ABI, ``abi3t`` (:pep:`803`). An ``abi3t`` +module loads on every free-threaded *and* GIL-enabled CPython from 3.15 on, so +one ``*.abi3t.so`` per platform covers both builds. pybind11 supports it too: +define ``Py_TARGET_ABI3T`` to ``0x030F0000`` (or a newer version). Headers of +either build work. With free-threaded headers, ``Py_LIMITED_API`` alone also +selects ``abi3t``, because free-threaded CPython has no ``abi3``. See `abi3t`_ +below for what differs. Enabling it =========== @@ -25,20 +27,31 @@ the default for a build tree: pybind11_add_module(example STABLE_ABI example.cpp) -``PYBIND11_STABLE_ABI_VERSION`` (default ``3.12``, raised to ``3.15`` on -free-threaded Python) selects the targeted version. ``PRECOMPILE`` can be -combined with ``STABLE_ABI``; the precompiled -library is then compiled against the limited API too, and one build tree -cannot mix stable-ABI and regular precompiled modules. +``PYBIND11_STABLE_ABI_VERSION`` (default ``3.12``, raised to ``3.15`` for +``abi3t``) selects the targeted version. On free-threaded Python the modules +are always ``abi3t``. Set ``PYBIND11_ABI3T`` to ``ON`` to also build ``abi3t`` +modules with GIL-enabled Python: + +.. code-block:: bash + + cmake -S . -B build -DPYBIND11_STABLE_ABI=ON -DPYBIND11_ABI3T=ON + +``PRECOMPILE`` can be combined with ``STABLE_ABI``; the precompiled library +is then compiled against the limited API too, and one build tree cannot mix +stable-ABI and regular precompiled modules. With setuptools, pass ``py_limited_api=True`` (or a version such as ``"3.13"``) to ``Pybind11Extension``; the module is named ``*.abi3.so`` -(``*.abi3t.so`` on free-threaded Python) and ``Py_LIMITED_API`` is defined -for you. +and ``Py_LIMITED_API`` is defined for you. Add a ``t`` (``"3.15t"``) for +``abi3t``, which is always used on free-threaded Python; ``Py_TARGET_ABI3T`` +is then defined and the module is named ``*.abi3t.so``. With GIL-enabled +Python, use ``build_ext`` from ``pybind11.setup_helpers`` to get that name. With any other build system, define ``Py_LIMITED_API=0x030C0000`` for every translation unit, link no version-specific Python library (``python3.lib`` on Windows), and name the module ``.abi3.so`` (``.pyd`` on Windows). +For ``abi3t``, define ``Py_TARGET_ABI3T=0x030F0000`` instead, link +``python3t.lib`` on Windows, and name the module ``.abi3t.so``. pybind11 rejects the combination with PyPy, GraalPy, and free-threaded CPython before 3.15 at compile time: they have no stable ABI. diff --git a/docs/compiling.rst b/docs/compiling.rst index 4f2143172c..ba338dcadf 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -412,9 +412,10 @@ the targeted version on (3.12 by default; set ``PYBIND11_STABLE_ABI_VERSION`` to target a newer one). Set the CMake variable ``PYBIND11_STABLE_ABI`` to make it the default for all ``pybind11_add_module`` calls and use ``NO_STABLE_ABI`` to opt a target out. This needs CMake 3.26+, the FindPython mode -(``PYBIND11_FINDPYTHON=ON``) and CPython 3.12+ headers (3.15+ for -free-threaded builds, which produce ``.abi3t`` modules); it is not available -for PyPy or GraalPy. See :doc:`advanced/stable_abi` +(``PYBIND11_FINDPYTHON=ON``) and CPython 3.12+ headers. Free-threaded +builds produce ``.abi3t`` modules and need 3.15+; set ``PYBIND11_ABI3T`` to +also build ``.abi3t`` modules with GIL-enabled CPython 3.15+. It is not +available for PyPy or GraalPy. See :doc:`advanced/stable_abi` for the features that are not available under the stable ABI. .. _precompile-mode: diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index bc0506e236..afd79ff1c8 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -71,6 +71,7 @@ import distutils.ccompiler import distutils.errors +from importlib.machinery import EXTENSION_SUFFIXES if TYPE_CHECKING: from typing_extensions import Self @@ -87,36 +88,40 @@ # directory into your path if it sits beside your setup.py. -def _limited_api_hex(value: Any) -> str: +def _stable_abi_macro(value: Any) -> tuple[str, str]: """ - The Py_LIMITED_API value for ``py_limited_api``: True selects the minimum - pybind11 supports (3.12, or 3.15 on free-threaded Python, where the stable - ABI is abi3t), a string such as "3.13" or "cp313" selects that version, and - an int or hex string is passed through. + The define for ``py_limited_api``, as ``(name, hex value)``. True selects + the minimum pybind11 supports (3.12 for abi3, 3.15 for abi3t), a string + such as "3.13" or "cp313" selects that version, and an int or hex string is + passed through. A "t" suffix ("3.15t", "cp315t") selects abi3t, which is + always used on free-threaded Python. """ - free_threaded = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) - minimum = (3, 15) if free_threaded else (3, 12) - if value is True: - return f"0x{minimum[0]:02X}{minimum[1]:02X}0000" - if isinstance(value, int): - return f"0x{value:08X}" + abi3t = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) text = str(value) - if text.lower().startswith("0x"): - return text - match = re.fullmatch(r"(?:cp)?(\d)\.?(\d+)", text) - if not match: - msg = f"py_limited_api must be True or a Python version such as '3.12', got {value!r}" - raise ValueError(msg) - major, minor = (int(part) for part in match.groups()) - if (major, minor) < minimum: + if isinstance(value, str) and text.lower().endswith("t"): + abi3t = True + text = text[:-1] + minimum = (3, 15) if abi3t else (3, 12) + if value is True: + number = (minimum[0] << 24) | (minimum[1] << 16) + elif isinstance(value, int): + number = value + elif text.lower().startswith("0x"): + number = int(text, 16) + else: + match = re.fullmatch(r"(?:cp)?(\d)\.?(\d+)", text) + if not match: + msg = f"py_limited_api must be True or a Python version such as '3.12', got {value!r}" + raise ValueError(msg) + major, minor = (int(part) for part in match.groups()) + number = (major << 24) | (minor << 16) + if (number >> 24, (number >> 16) & 0xFF) < minimum: what = ( - "the free-threaded stable ABI (abi3t)" - if free_threaded - else "the stable ABI" + "the stable ABI for free-threading (abi3t)" if abi3t else "the stable ABI" ) msg = f"pybind11 supports {what} from Python {minimum[0]}.{minimum[1]} on, got {value!r}" raise ValueError(msg) - return f"0x{major:02X}{minor:02X}0000" + return ("Py_TARGET_ABI3T" if abi3t else "Py_LIMITED_API", f"0x{number:08X}") class Pybind11Extension(_Extension): @@ -145,8 +150,12 @@ class Pybind11Extension(_Extension): ``py_limited_api=True`` builds against the Python stable ABI (3.12, or pass a version string such as ``"3.13"``): ``Py_LIMITED_API`` is defined and setuptools names the module ``*.abi3.so``, so one wheel serves every - CPython from that version on. See the pybind11 documentation for the - features that are not available under the stable ABI. + CPython from that version on. A ``"t"`` suffix (``"3.15t"``) selects abi3t + (``Py_TARGET_ABI3T``, ``*.abi3t.so``), which also loads on free-threaded + CPython; it is always used on free-threaded Python. Use the ``build_ext`` + from this module to get the ``*.abi3t.so`` name on GIL-enabled Python. See + the pybind11 documentation for the features that are not available under + the stable ABI. """ # flags are prepended, so that they can be further overridden, e.g. by @@ -172,9 +181,13 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: # setuptools only uses the flag for the .abi3 file name; the # define selects the limited API in the headers. kwargs["py_limited_api"] = True + macro = _stable_abi_macro(py_limited_api) + self._abi3t = macro[0] == "Py_TARGET_ABI3T" macros = list(kwargs.get("define_macros", []) or []) - if not any(name == "Py_LIMITED_API" for name, _ in macros): - macros.append(("Py_LIMITED_API", _limited_api_hex(py_limited_api))) + if not any( + name in {"Py_LIMITED_API", "Py_TARGET_ABI3T"} for name, _ in macros + ): + macros.append(macro) kwargs["define_macros"] = macros super().__init__(*args, **kwargs) @@ -330,6 +343,26 @@ def build_extensions(self) -> None: super().build_extensions() + def get_ext_filename(self, fullname: str) -> str: + """ + setuptools picks the first ``.abi3*`` suffix, which is ``.abi3`` on + GIL-enabled Python; abi3t modules need ``.abi3t``. + """ + filename: str = super().get_ext_filename(fullname) # type: ignore[no-untyped-call] + ext = getattr(self, "ext_map", {}).get(fullname) + if getattr(ext, "_abi3t", False) and not WIN: + for suffix in EXTENSION_SUFFIXES: + abi3t_suffix = suffix.replace(".abi3", ".abi3t", 1) + if ( + suffix.startswith(".abi3") + and not suffix.startswith(".abi3t") + and filename.endswith(suffix) + and abi3t_suffix in EXTENSION_SUFFIXES + ): + filename = filename[: -len(suffix)] + abi3t_suffix + break + return filename + def intree_extensions( paths: Iterable[str], package_dir: dict[str, str] | None = None diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py index 285cf953d0..09bcf9038f 100644 --- a/tests/extra_setuptools/test_setuphelper.py +++ b/tests/extra_setuptools/test_setuphelper.py @@ -117,7 +117,19 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std): or (sysconfig.get_config_var("Py_GIL_DISABLED") and sys.version_info < (3, 15)), reason="stable ABI needs CPython 3.12+, or 3.15+ when free-threaded (abi3t)", ) -def test_stable_abi_setup_py(monkeypatch, tmpdir): +@pytest.mark.parametrize( + "py_limited_api", + [ + True, + pytest.param( + "3.15t", + marks=pytest.mark.skipif( + sys.version_info < (3, 15), reason="abi3t needs CPython 3.15+" + ), + ), + ], +) +def test_stable_abi_setup_py(monkeypatch, tmpdir, py_limited_api): monkeypatch.chdir(tmpdir) monkeypatch.syspath_prepend(MAIN_DIR) @@ -135,7 +147,7 @@ def test_stable_abi_setup_py(monkeypatch, tmpdir): "stable_abi_setup", ["main.cpp"], cxx_std=17, - py_limited_api=True, + py_limited_api={py_limited_api!r}, ), ] @@ -175,6 +187,9 @@ def test_stable_abi_setup_py(monkeypatch, tmpdir): built = [f for f in tmpdir.listdir() if f.basename.startswith("stable_abi_setup")] assert len(built) == 1 assert ".abi3" in built[0].basename or built[0].ext == ".pyd" + abi3t = py_limited_api == "3.15t" or sysconfig.get_config_var("Py_GIL_DISABLED") + if not WIN: + assert (".abi3t" in built[0].basename) == bool(abi3t) (tmpdir / "test.py").write_text( dedent( diff --git a/tests/test_cross_module_rtti/CMakeLists.txt b/tests/test_cross_module_rtti/CMakeLists.txt index c9b95bfba1..11f6ed3765 100644 --- a/tests/test_cross_module_rtti/CMakeLists.txt +++ b/tests/test_cross_module_rtti/CMakeLists.txt @@ -40,7 +40,10 @@ target_compile_features(test_cross_module_rtti_lib PUBLIC cxx_std_11) generate_export_header(test_cross_module_rtti_lib) -pybind11_add_module(test_cross_module_rtti_bindings SHARED bindings.cpp) +# The embedding executable cannot use the stable ABI, and must share internals with the bindings; +# the precompiled library would be built for the stable ABI too. +pybind11_add_module(test_cross_module_rtti_bindings SHARED NO_STABLE_ABI NO_PRECOMPILE + bindings.cpp) target_link_libraries(test_cross_module_rtti_bindings PUBLIC test_cross_module_rtti_lib::test_cross_module_rtti_lib) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index b500eb7972..e73af5aaf0 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -734,8 +734,8 @@ TEST_SUBMODULE(pytypes, m) { // pytypes.h could be changed to enforce the "most correct" user code below, by removing // `const` from iterator `reference` using type aliases, but that will break existing // user code. -#if (defined(__APPLE__) && defined(__clang__)) || defined(PYPY_VERSION) -// This is "most correct" and enforced on these platforms. +#if (defined(__APPLE__) && defined(__clang__)) || defined(PYPY_VERSION) || defined(Py_LIMITED_API) +// This is "most correct" and enforced on these platforms (and the stable ABI, as on PyPy). # define PYBIND11_AUTO_IT auto it #else // This works on many platforms and is (unfortunately) reflective of existing user code. diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index eeb15b83ef..f8906a8fef 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -525,7 +525,12 @@ function(pybind11_precompile) set_target_properties(pybind11_precompiled PROPERTIES PYBIND11_STABLE_ABI "${ARG_STABLE_ABI}") if(ARG_STABLE_ABI) _pybind11_stable_abi_hex(_sabi_hex) - target_compile_definitions(pybind11_precompiled PRIVATE "Py_LIMITED_API=${_sabi_hex}") + _pybind11_stable_abi_is_abi3t(_abi3t) + if(_abi3t) + target_compile_definitions(pybind11_precompiled PRIVATE "Py_TARGET_ABI3T=${_sabi_hex}") + else() + target_compile_definitions(pybind11_precompiled PRIVATE "Py_LIMITED_API=${_sabi_hex}") + endif() endif() # pybind11::module (not just pybind11::pybind11): the library must compile with the # interpreter's ABI macros (e.g. Py_GIL_DISABLED, which FindPython attaches to diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index e48fbaa283..d7b4aed999 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -269,6 +269,10 @@ set(PYBIND11_STABLE_ABI_VERSION "3.12" CACHE STRING "Python version whose stable ABI STABLE_ABI modules target (3.12 or newer)") +set(PYBIND11_ABI3T + OFF + CACHE BOOL "STABLE_ABI modules target abi3t (PEP 803), also from GIL-enabled Python 3.15+") + # True if the interpreter is a free-threaded build (needs the abi3t variant of the stable ABI). function(_pybind11_python_is_free_threaded out_var) if(${_Python}_FREE_THREADED @@ -284,11 +288,22 @@ function(_pybind11_python_is_free_threaded out_var) endif() endfunction() -# PYBIND11_STABLE_ABI_VERSION, raised to 3.15 on free-threaded Python (abi3t starts there). +# True if STABLE_ABI modules target abi3t: always on free-threaded Python, else PYBIND11_ABI3T. +function(_pybind11_stable_abi_is_abi3t out_var) + _pybind11_python_is_free_threaded(_abi3t) + if(PYBIND11_ABI3T) + set(_abi3t ON) + endif() + set(${out_var} + ${_abi3t} + PARENT_SCOPE) +endfunction() + +# PYBIND11_STABLE_ABI_VERSION, raised to 3.15 for abi3t (abi3t starts there). function(_pybind11_stable_abi_version out_var) set(_version "${PYBIND11_STABLE_ABI_VERSION}") - _pybind11_python_is_free_threaded(_free_threaded) - if(_free_threaded AND _version VERSION_LESS 3.15) + _pybind11_stable_abi_is_abi3t(_abi3t) + if(_abi3t AND _version VERSION_LESS 3.15) set(_version "3.15") endif() set(${out_var} @@ -308,6 +323,31 @@ function(_pybind11_stable_abi_hex out_var) PARENT_SCOPE) endfunction() +# Add the stable ABI define and import library to a target that FindPython's USE_SABI did not set up. +function(_pybind11_stable_abi_setup target_name) + _pybind11_stable_abi_hex(_hex) + _pybind11_stable_abi_is_abi3t(_abi3t) + _pybind11_python_is_free_threaded(_free_threaded) + if(NOT _abi3t) + target_compile_definitions(${target_name} PRIVATE "Py_LIMITED_API=${_hex}") + else() + target_compile_definitions(${target_name} PRIVATE "Py_TARGET_ABI3T=${_hex}") + endif() + if(_abi3t + AND NOT _free_threaded + AND WIN32) + # Python::SABIModule is python3.lib here, but abi3t needs python3t.lib from the same directory. + target_include_directories(${target_name} SYSTEM PRIVATE ${${_Python}_INCLUDE_DIRS}) + find_library( + _pybind11_python3t_lib python3t + PATHS ${${_Python}_SABI_LIBRARY_DIRS} + NO_DEFAULT_PATH REQUIRED) + target_link_libraries(${target_name} PRIVATE "${_pybind11_python3t_lib}") + else() + target_link_libraries(${target_name} PRIVATE ${_Python}::SABIModule) + endif() +endfunction() + # Fail with an explanation if a STABLE_ABI module cannot be built in this configuration. function(_pybind11_check_stable_abi target_name lib_type) if(CMAKE_VERSION VERSION_LESS 3.26) @@ -328,11 +368,11 @@ function(_pybind11_check_stable_abi target_name lib_type) message(FATAL_ERROR "${target_name}: STABLE_ABI ${PYBIND11_STABLE_ABI_VERSION} needs Python " ">= ${PYBIND11_STABLE_ABI_VERSION} headers, found ${${_Python}_VERSION}.") endif() - _pybind11_python_is_free_threaded(_free_threaded) - if(_free_threaded + _pybind11_stable_abi_is_abi3t(_abi3t) + if(_abi3t AND DEFINED ${_Python}_VERSION AND ${_Python}_VERSION VERSION_LESS 3.15) - message(FATAL_ERROR "${target_name}: the free-threaded stable ABI (abi3t) needs Python " + message(FATAL_ERROR "${target_name}: the stable ABI for free-threading (abi3t) needs Python " ">= 3.15, found ${${_Python}_VERSION}.") endif() if(NOT TARGET ${_Python}::SABIModule) @@ -366,19 +406,26 @@ function(pybind11_add_module target_name) # STABLE_ABI keyword, or the PYBIND11_STABLE_ABI variable as the default; NO_STABLE_ABI opts out. set(stable_abi OFF) set(use_sabi "") + set(own_sabi OFF) if((ARG_STABLE_ABI OR PYBIND11_STABLE_ABI) AND NOT ARG_NO_STABLE_ABI) _pybind11_check_stable_abi(${target_name} ${lib_type}) set(stable_abi ON) - if(lib_type STREQUAL "MODULE") + _pybind11_stable_abi_is_abi3t(_abi3t) + _pybind11_python_is_free_threaded(_free_threaded) + if(lib_type STREQUAL "SHARED" OR (_abi3t AND NOT _free_threaded)) + # python_add_library(SHARED) links the embedding library, and USE_SABI only selects abi3t + # on free-threaded Python. + set(own_sabi ON) + elseif(lib_type STREQUAL "MODULE") # Defines Py_LIMITED_API and links Python::SABIModule (python3.lib on Windows). _pybind11_stable_abi_version(_sabi_version) set(use_sabi USE_SABI ${_sabi_version}) endif() endif() - if(stable_abi AND lib_type STREQUAL "SHARED") - # python_add_library(SHARED) links the embedding library; a stable-ABI helper must not. - add_library(${target_name} SHARED ${ARG_UNPARSED_ARGUMENTS}) + if(own_sabi) + add_library(${target_name} ${lib_type} ${ARG_UNPARSED_ARGUMENTS}) + _pybind11_stable_abi_setup(${target_name}) elseif("${_Python}" STREQUAL "Python") python_add_library(${target_name} ${lib_type} ${use_sabi} ${ARG_UNPARSED_ARGUMENTS}) elseif("${_Python}" STREQUAL "Python3") @@ -390,12 +437,14 @@ function(pybind11_add_module target_name) target_link_libraries(${target_name} PRIVATE pybind11::headers) if(lib_type STREQUAL "MODULE") - target_link_libraries(${target_name} PRIVATE pybind11::module) + if(NOT own_sabi) + target_link_libraries(${target_name} PRIVATE pybind11::module) + else() + # Python::Module would link the version-specific library; take only the module link flags. + target_link_libraries(${target_name} PRIVATE pybind11::python_link_helper) + endif() elseif(stable_abi) # A SHARED helper library that stable-ABI modules link: same ABI, no embedding. - _pybind11_stable_abi_hex(_sabi_hex) - target_compile_definitions(${target_name} PRIVATE "Py_LIMITED_API=${_sabi_hex}") - target_link_libraries(${target_name} PRIVATE ${_Python}::SABIModule) else() target_link_libraries(${target_name} PRIVATE pybind11::embed) endif() @@ -453,7 +502,7 @@ function(pybind11_extension name) SUFFIX "${PYTHON_MODULE_EXTENSION}") endfunction() -# Stable-ABI modules carry the version-independent "abi3" tag (none on Windows). +# Stable-ABI modules carry the version-independent "abi3" or "abi3t" tag (none on Windows). function(pybind11_extension_stable_abi name) if(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(_ext ".pyd") @@ -461,7 +510,13 @@ function(pybind11_extension_stable_abi name) set(_ext "${CMAKE_SHARED_MODULE_SUFFIX}") endif() if(DEFINED ${_Python}_SOSABI AND NOT "${${_Python}_SOSABI}" STREQUAL "") - set(_ext ".${${_Python}_SOSABI}${_ext}") + set(_sosabi "${${_Python}_SOSABI}") + _pybind11_stable_abi_is_abi3t(_abi3t) + if(_abi3t AND _sosabi MATCHES "^abi3(-|$)") + # SOSABI is abi3 on GIL-enabled Python. + string(REGEX REPLACE "^abi3" "abi3t" _sosabi "${_sosabi}") + endif() + set(_ext ".${_sosabi}${_ext}") endif() set_target_properties(${name} PROPERTIES PREFIX "" SUFFIX "${_ext}") endfunction() From 78233abe27512ad1987d9f61406fbca15e2f826e Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 8 Oct 2026 09:06:24 -0400 Subject: [PATCH 41/45] fix(stable-abi): Windows abi3t export symbol, MSVC C4702, .pyd name check Assisted-by: ClaudeCode:claude-opus-5-5 --- include/pybind11/cast.h | 4 ++++ pybind11/setup_helpers.py | 12 ++++++++++++ tests/test_cmake_build/test.py | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index c60c6256df..8925c1e01d 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -927,9 +927,13 @@ struct copyable_holder_caster : public type_caster_base { } } + // The fallback overload always throws; MSVC LTCG then flags this return as unreachable. + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_MSVC(4702) bool set_foreign_holder(handle src) { return holder_caster_foreign_helpers::set_foreign_holder(src, (type *) value, &holder); } + PYBIND11_WARNING_POP void load_value(value_and_holder &&v_h) { if (v_h.holder_constructed()) { diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index afd79ff1c8..9d3de40b22 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -343,6 +343,18 @@ def build_extensions(self) -> None: super().build_extensions() + def get_export_symbols(self, ext: _Extension) -> list[str]: # type: ignore[override] + """ + abi3t modules export the PEP 793 ``PyModExport_*`` hook, not ``PyInit_*``. + """ + symbols: list[str] = super().get_export_symbols(ext) # type: ignore[no-untyped-call] + if getattr(ext, "_abi3t", False): + name = ext.name.split(".")[-1] + symbols = [ + f"PyModExport_{name}" if s == f"PyInit_{name}" else s for s in symbols + ] + return symbols + def get_ext_filename(self, fullname: str) -> str: """ setuptools picks the first ``.abi3*`` suffix, which is ``.abi3`` on diff --git a/tests/test_cmake_build/test.py b/tests/test_cmake_build/test.py index 9e4af55920..3ea3f0d107 100644 --- a/tests/test_cmake_build/test.py +++ b/tests/test_cmake_build/test.py @@ -10,6 +10,6 @@ assert test_cmake_build.add(1, 2) == 3 expect_abi3 = os.environ.get("PYBIND11_EXPECT_ABI3") -if expect_abi3: +if expect_abi3 and sys.platform != "win32": assert ".abi3" in expect_abi3, f"expected an abi3 module name, got {expect_abi3}" print(f"{sys.argv[1]} imports, runs, and adds: 1 + 2 = 3") From f53e9752e9d8d69ada5948aa4d4b371b8196cb2c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 8 Oct 2026 09:41:21 -0400 Subject: [PATCH 42/45] feat: make PyType_FromMetaclass the default type-creation path on CPython 3.12+ PYBIND11_TYPE_CREATION_VIA_SPEC is now a 0/1 value that defaults to 1 on CPython 3.12+ (and is forced on under Py_LIMITED_API). Define it as 0, or configure with -DPYBIND11_TYPE_CREATION_VIA_SPEC=OFF, to keep the legacy PyHeapTypeObject path. The CI job that used to opt in now opts out. The spec path now honors PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET (appending a `__dictoffset__` member instead of Py_TPFLAGS_MANAGED_DICT), and pybind11_traverse/pybind11_clear no longer call the managed-dict functions on 3.13+ when that macro is defined. Assisted-by: ClaudeCode:claude-fable-5-1 ClaudeCode:claude-fable-5-1 --- .github/workflows/ci.yml | 10 +++++----- CMakeLists.txt | 12 +++++++++--- docs/advanced/stable_abi.rst | 6 ++++-- include/pybind11/detail/class-inl.h | 28 +++++++++++++++------------- include/pybind11/detail/class.h | 8 ++++---- include/pybind11/detail/common.h | 21 +++++++++++++++------ include/pybind11/detail/internals.h | 2 +- 7 files changed, 53 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d3865ca35..925f72bbe1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,7 +101,7 @@ jobs: # Opt-in PyType_FromMetaclass type creation (the future default on CPython 3.12+ # and the only path under the stable ABI). - type-creation-via-spec: + type-creation-legacy: if: github.event.pull_request.draft == false strategy: fail-fast: false @@ -109,16 +109,16 @@ jobs: include: - runs-on: ubuntu-latest python-version: '3.12' - cmake-args: -DPYBIND11_TYPE_CREATION_VIA_SPEC=ON + cmake-args: -DPYBIND11_TYPE_CREATION_VIA_SPEC=OFF - runs-on: ubuntu-latest python-version: '3.14' - cmake-args: -DPYBIND11_TYPE_CREATION_VIA_SPEC=ON -DPYBIND11_TEST_PRECOMPILE=ON + cmake-args: -DPYBIND11_TYPE_CREATION_VIA_SPEC=OFF -DPYBIND11_TEST_PRECOMPILE=ON - runs-on: macos-latest python-version: '3.13' - cmake-args: -DPYBIND11_TYPE_CREATION_VIA_SPEC=ON + cmake-args: -DPYBIND11_TYPE_CREATION_VIA_SPEC=OFF - runs-on: windows-2022 python-version: '3.13' - cmake-args: -DPYBIND11_TYPE_CREATION_VIA_SPEC=ON + cmake-args: -DPYBIND11_TYPE_CREATION_VIA_SPEC=OFF name: 📐 uses: ./.github/workflows/reusable-standard.yml diff --git a/CMakeLists.txt b/CMakeLists.txt index 024626e5d8..77a19add9d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,8 +99,12 @@ option(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION "To enforce that a handle_type_name<> specialization exists" OFF) option(PYBIND11_SIMPLE_GIL_MANAGEMENT "Use simpler GIL management logic that does not support disassociation" OFF) -option(PYBIND11_TYPE_CREATION_VIA_SPEC - "Create type objects with PyType_FromMetaclass (CPython 3.12+ only, opt-in preview)" OFF) +set(PYBIND11_TYPE_CREATION_VIA_SPEC + "" + CACHE + STRING + "Create type objects with PyType_FromMetaclass: ON, OFF, or empty for the default (ON on CPython 3.12+)" +) set(PYBIND11_INTERNALS_VERSION "" CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.") @@ -326,7 +330,9 @@ if(NOT TARGET pybind11_headers) target_compile_definitions(pybind11_headers INTERFACE PYBIND11_SIMPLE_GIL_MANAGEMENT) endif() if(PYBIND11_TYPE_CREATION_VIA_SPEC) - target_compile_definitions(pybind11_headers INTERFACE PYBIND11_TYPE_CREATION_VIA_SPEC) + target_compile_definitions(pybind11_headers INTERFACE PYBIND11_TYPE_CREATION_VIA_SPEC=1) + elseif(NOT "${PYBIND11_TYPE_CREATION_VIA_SPEC}" STREQUAL "") + target_compile_definitions(pybind11_headers INTERFACE PYBIND11_TYPE_CREATION_VIA_SPEC=0) endif() else() # It is invalid to install a target twice, too. diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index 5b179b3b9e..ee49128a47 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -145,8 +145,10 @@ where it used to read fields directly. The costs are small and are confined to specific operations: * Type objects are created with ``PyType_FromMetaclass()`` and - ``PyType_Spec`` (the ``PYBIND11_TYPE_CREATION_VIA_SPEC`` path, also - available as an opt-in for regular builds on CPython 3.12+). + ``PyType_Spec``. This is also the default for regular builds on CPython + 3.12+; define ``PYBIND11_TYPE_CREATION_VIA_SPEC=0`` (CMake: + ``-DPYBIND11_TYPE_CREATION_VIA_SPEC=OFF``) to use the previous path, which + fills in the ``PyHeapTypeObject`` fields directly. * Instance allocation and deallocation look up ``tp_alloc``/``tp_free`` with ``PyType_GetSlot()``; the ``__dict__`` of ``py::dynamic_attr()`` instances is found through the type's cached dictionary offset. diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 672c65391f..4928054130 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -65,7 +65,7 @@ PYBIND11_BASE_TYPE_SLOT( #undef PYBIND11_BASE_TYPE_SLOT -#if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +#if PYBIND11_TYPE_CREATION_VIA_SPEC PYBIND11_INLINE PyTypeObject * type_from_spec(const char *caller, PyTypeObject *metaclass, PyType_Spec *spec, PyObject *bases) { PyObject *type = PyType_FromMetaclass(metaclass, nullptr, spec, bases); @@ -89,9 +89,9 @@ pybind11_static_set(PyObject *self, PyObject *obj, PyObject *value) { return property_type_descr_set()(self, cls, value); } -# if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +# if PYBIND11_TYPE_CREATION_VIA_SPEC -# if defined(Py_LIMITED_API) +# if defined(Py_LIMITED_API) || defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) // Without Py_TPFLAGS_MANAGED_DICT the `__dict__` slot follows the property object. These two // functions find it through the type's `__dictoffset__`. PYBIND11_INLINE PyObject **static_property_dict_ptr(PyObject *self) { @@ -113,7 +113,7 @@ extern "C" PYBIND11_INLINE int pybind11_static_property_clear(PyObject *self) { PYBIND11_INLINE PyTypeObject *make_static_property_type() { // Since Python-3.12 property-derived types are required to have dynamic attributes (to set // `__doc__`), hence the GC and dict slots. -# if !defined(Py_LIMITED_API) +# if !defined(Py_LIMITED_API) && !defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) static PyType_Slot slots[] = {{Py_tp_descr_get, reinterpret_cast(pybind11_static_get)}, {Py_tp_descr_set, reinterpret_cast(pybind11_static_set)}, {Py_tp_traverse, reinterpret_cast(pybind11_traverse)}, @@ -331,7 +331,7 @@ extern "C" PYBIND11_INLINE void pybind11_meta_dealloc(PyObject *obj) { type_type_dealloc()(obj); } -#if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +#if PYBIND11_TYPE_CREATION_VIA_SPEC PYBIND11_INLINE PyTypeObject *make_default_metaclass() { static PyType_Slot slots[] = { @@ -828,7 +828,7 @@ extern "C" PYBIND11_INLINE void pybind11_object_dealloc(PyObject *self) { Py_DECREF(reinterpret_cast(type)); } -#if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +#if PYBIND11_TYPE_CREATION_VIA_SPEC PYBIND11_INLINE PyObject *make_object_base_type(PyTypeObject *metaclass) { /* Support weak references (needed for the keep_alive feature) */ @@ -904,7 +904,8 @@ PYBIND11_INLINE PyObject *make_object_base_type(PyTypeObject *metaclass) { #endif // PYBIND11_TYPE_CREATION_VIA_SPEC extern "C" PYBIND11_INLINE int pybind11_traverse(PyObject *self, visitproc visit, void *arg) { -#if PY_VERSION_HEX >= 0x030D0000 && !defined(Py_LIMITED_API) +#if PY_VERSION_HEX >= 0x030D0000 && !defined(Py_LIMITED_API) \ + && !defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) int ret = PyObject_VisitManagedDict(self, visit, arg); if (ret) { return ret; @@ -920,7 +921,8 @@ extern "C" PYBIND11_INLINE int pybind11_traverse(PyObject *self, visitproc visit } extern "C" PYBIND11_INLINE int pybind11_clear(PyObject *self) { -#if PY_VERSION_HEX >= 0x030D0000 && !defined(Py_LIMITED_API) +#if PY_VERSION_HEX >= 0x030D0000 && !defined(Py_LIMITED_API) \ + && !defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) PyObject_ClearManagedDict(self); #else if (PyObject **dict = instance_dict_ptr(self)) { @@ -1066,7 +1068,7 @@ PYBIND11_INLINE void enable_buffer_protocol(PyHeapTypeObject *heap_type) { } #endif // !Py_LIMITED_API -#if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +#if PYBIND11_TYPE_CREATION_VIA_SPEC PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { auto &internals = get_internals(); @@ -1117,16 +1119,16 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { flags |= Py_TPFLAGS_BASETYPE; } int basicsize = 0; // inherit the instance layout from the base -# if defined(Py_LIMITED_API) - // No Py_TPFLAGS_MANAGED_DICT in the stable ABI: the `__dict__` slot is appended to the base - // layout unless a base has one already. (PEP 697 relative offsets are not applied to +# if defined(Py_LIMITED_API) || defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) + // Without Py_TPFLAGS_MANAGED_DICT (not in the stable ABI) the `__dict__` slot is appended to + // the base layout unless a base has one already. (PEP 697 relative offsets are not applied to // `__dictoffset__` on 3.12, so the offset is absolute.) PyMemberDef dict_member[] = {{"__dictoffset__", Py_T_PYSSIZET, 0, Py_READONLY, nullptr}, {nullptr, 0, 0, 0, nullptr}}; # endif if (rec.dynamic_attr) { flags |= Py_TPFLAGS_HAVE_GC; -# if !defined(Py_LIMITED_API) +# if !defined(Py_LIMITED_API) && !defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) flags |= Py_TPFLAGS_MANAGED_DICT; # else bool base_has_dict = false; diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 708eace403..42687313e5 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -45,7 +45,7 @@ destructor type_type_dealloc(); descrgetfunc property_type_descr_get(); descrsetfunc property_type_descr_set(); -#if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +#if PYBIND11_TYPE_CREATION_VIA_SPEC /// PyType_FromMetaclass(), or pybind11_fail() with the Python error prefixed by `caller`. PyTypeObject * type_from_spec(const char *caller, PyTypeObject *metaclass, PyType_Spec *spec, PyObject *bases); @@ -212,9 +212,9 @@ void enable_buffer_protocol(PyHeapTypeObject *heap_type); PyObject *make_new_python_type(const type_record &rec); #if !defined(Py_LIMITED_API) -/// The hand-filled PyHeapTypeObject path. With PYBIND11_TYPE_CREATION_VIA_SPEC it is still used -/// for `py::custom_type_setup` and for metaclasses with a custom `tp_new` (which -/// PyType_FromMetaclass rejects). +/// The hand-filled PyHeapTypeObject path: the only one before CPython 3.12 or with +/// PYBIND11_TYPE_CREATION_VIA_SPEC=0. Otherwise still used for `py::custom_type_setup` and for +/// metaclasses with a custom `tp_new` (which PyType_FromMetaclass rejects). PyObject *make_new_python_type_legacy(const type_record &rec); #endif diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 176fe26313..1f0b179355 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -36,7 +36,9 @@ # endif // The only type-creation path without struct access. # if !defined(PYBIND11_TYPE_CREATION_VIA_SPEC) -# define PYBIND11_TYPE_CREATION_VIA_SPEC +# define PYBIND11_TYPE_CREATION_VIA_SPEC 1 +# elif !PYBIND11_TYPE_CREATION_VIA_SPEC +# error "PYBIND11_TYPE_CREATION_VIA_SPEC=0 is not possible under Py_LIMITED_API." # endif # if !defined(PYBIND11_HAS_SUBINTERPRETER_SUPPORT) # define PYBIND11_HAS_SUBINTERPRETER_SUPPORT 0 // reads thread/interpreter state fields @@ -282,11 +284,18 @@ # define PYBIND11_SIMPLE_GIL_MANAGEMENT #endif -// Opt-in: create the pybind11 type objects (metaclass, static property, instance base and all -// bound classes) through PyType_FromMetaclass() with PyType_Spec instead of filling in raw -// PyHeapTypeObject fields. Requires CPython 3.12+. This is the only type-creation path under the -// stable ABI and is planned to become the default on CPython 3.12+. -#if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +// Create the pybind11 type objects (metaclass, static property, instance base and all bound +// classes) through PyType_FromMetaclass() with PyType_Spec instead of filling in raw +// PyHeapTypeObject fields. The default on CPython 3.12+ and the only path under the stable ABI; +// define PYBIND11_TYPE_CREATION_VIA_SPEC=0 to opt out. `py::custom_type_setup()` and +// metaclasses with a custom `tp_new` still use the legacy path. +#if !defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +# if PY_VERSION_HEX >= 0x030C0000 && !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) +# define PYBIND11_TYPE_CREATION_VIA_SPEC 1 +# else +# define PYBIND11_TYPE_CREATION_VIA_SPEC 0 +# endif +#elif PYBIND11_TYPE_CREATION_VIA_SPEC # if PY_VERSION_HEX < 0x030C0000 || defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) # error "PYBIND11_TYPE_CREATION_VIA_SPEC requires CPython 3.12 or newer." # endif diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index b9140442df..4683a7cb3a 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -84,7 +84,7 @@ # else # define PYBIND11_PRECOMPILED_CFG_LA 0 # endif -# if defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +# if PYBIND11_TYPE_CREATION_VIA_SPEC # define PYBIND11_PRECOMPILED_CFG_TS 1 # else # define PYBIND11_PRECOMPILED_CFG_TS 0 From 2c1c6130db6d9853b35d1b260aa6d6d92e3bbb51 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 8 Oct 2026 09:56:07 -0400 Subject: [PATCH 43/45] docs: describe the scikit-build-core stable ABI setup wheel.py-api only selects the wheel tag; the CMake side must enable STABLE_ABI from SKBUILD_SABI_COMPONENT and SKBUILD_SABI_VERSION. Assisted-by: ClaudeCode:claude-fable-5-1 ClaudeCode:claude-fable-5-1 --- docs/advanced/stable_abi.rst | 14 ++++++++++++++ docs/compiling.rst | 7 +++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index ee49128a47..fc636dd54b 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -40,6 +40,20 @@ modules with GIL-enabled Python: is then compiled against the limited API too, and one build tree cannot mix stable-ABI and regular precompiled modules. +With scikit-build-core, set ``wheel.py-api = "cp312"`` in ``pyproject.toml`` +(``"cp312.cp315t"`` to also get ``abi3t`` wheels from free-threaded Python; +the pybind11 defaults produce the matching module in both cases). This only +selects the wheel tag, and scikit-build-core ignores it when the build Python +is too old or is not CPython, so enable ``STABLE_ABI`` from the variables it +sets, before ``find_package(pybind11)``: + +.. code-block:: cmake + + if(SKBUILD_SABI_COMPONENT) + set(PYBIND11_STABLE_ABI ON) + set(PYBIND11_STABLE_ABI_VERSION "${SKBUILD_SABI_VERSION}" CACHE STRING "") + endif() + With setuptools, pass ``py_limited_api=True`` (or a version such as ``"3.13"``) to ``Pybind11Extension``; the module is named ``*.abi3.so`` and ``Py_LIMITED_API`` is defined for you. Add a ``t`` (``"3.15t"``) for diff --git a/docs/compiling.rst b/docs/compiling.rst index ba338dcadf..094ad88073 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -48,6 +48,8 @@ add a ``pyproject.toml`` file like this: You don't need setuptools files like ``MANIFEST.in``, ``setup.py``, or ``setup.cfg``, as this is not setuptools. See `scikit-build-core`_ for details. +To ship one wheel per platform instead of one per Python version, see +:doc:`advanced/stable_abi`. For projects you plan to upload to PyPI, be sure to fill out the ``[project]`` table with other important metadata as well (see `Writing pyproject.toml`_). @@ -415,8 +417,9 @@ to opt a target out. This needs CMake 3.26+, the FindPython mode (``PYBIND11_FINDPYTHON=ON``) and CPython 3.12+ headers. Free-threaded builds produce ``.abi3t`` modules and need 3.15+; set ``PYBIND11_ABI3T`` to also build ``.abi3t`` modules with GIL-enabled CPython 3.15+. It is not -available for PyPy or GraalPy. See :doc:`advanced/stable_abi` -for the features that are not available under the stable ABI. +available for PyPy or GraalPy. See :doc:`advanced/stable_abi` for the +features that are not available under the stable ABI and for the +scikit-build-core setup (``wheel.py-api`` plus ``SKBUILD_SABI_COMPONENT``). .. _precompile-mode: From 75be228cf9fcfe826ff51111f3f77ab6dbd4a72a Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 8 Oct 2026 10:49:40 -0400 Subject: [PATCH 44/45] fix(stable-abi): gate PyDict_*Ref helpers on the limited API version PyDict_SetDefaultRef is only in the limited API from 3.15, so a Py_LIMITED_API=0x030E0000 build failed to compile. Alpha-suffixed gates also never matched a limited API target, so 3.13 targets did not use PyDict_GetItemStringRef. Add a 3.14-target CI build. Assisted-by: ClaudeCode:claude-fable-5-1 ClaudeCode:claude-fable-5-1 --- .github/workflows/ci.yml | 4 ++++ include/pybind11/pytypes.h | 10 ++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 925f72bbe1..81466bb097 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,6 +141,10 @@ jobs: - runs-on: ubuntu-latest python-version: '3.14' cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON -DPYBIND11_TEST_PRECOMPILE=ON + # A newer limited API target: gates must follow the limited API, not PY_VERSION_HEX + - runs-on: ubuntu-latest + python-version: '3.14' + cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON -DPYBIND11_STABLE_ABI_VERSION=3.14 - runs-on: macos-latest python-version: '3.13' cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON -DCMAKE_CXX_STANDARD=17 diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 0163d3369d..c3a4f6843c 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -897,10 +897,11 @@ inline PyObject *dict_getitem(PyObject *v, PyObject *key) { return rv; } -// PyDict_GetItemStringRef was added in Python 3.13.0a1. +// PyDict_GetItemStringRef was added in Python 3.13.0a1 (limited API: 3.13). // See also: https://github.com/python/pythoncapi-compat/blob/main/pythoncapi_compat.h inline PyObject *dict_getitemstringref(PyObject *v, const char *key) { -#if PYBIND11_API_VERSION_HEX >= 0x030D00A1 +#if (defined(Py_LIMITED_API) && Py_LIMITED_API + 0 >= 0x030D0000) \ + || (!defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030D00A1) PyObject *rv = nullptr; if (PyDict_GetItemStringRef(v, key, &rv) < 0) { throw error_already_set(); @@ -937,10 +938,11 @@ inline PyObject *dict_setdefaultstring(PyObject *v, const char *key, PyObject *d return rv; } -// PyDict_SetDefaultRef was added in Python 3.13.0a4. +// PyDict_SetDefaultRef was added in Python 3.13.0a4 (limited API: 3.15). // See also: https://github.com/python/pythoncapi-compat/blob/main/pythoncapi_compat.h inline PyObject *dict_setdefaultstringref(PyObject *v, const char *key, PyObject *defaultobj) { -#if PYBIND11_API_VERSION_HEX >= 0x030D00A4 +#if (defined(Py_LIMITED_API) && Py_LIMITED_API + 0 >= 0x030F0000) \ + || (!defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030D00A4) PyObject *kv = PyUnicode_FromString(key); if (kv == nullptr) { throw error_already_set(); From f7103aa1f140fa50cfabfd5e85367d8808ef8b9b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 10 Oct 2026 19:53:32 -0400 Subject: [PATCH 45/45] fix(stable-abi): address review findings - Pin registry candidates before values_and_holders() takes the internals lock in get_object_handle(); it now returns an owning object. - Provide type_dictoffset()/type_basicsize() for non-limited builds so PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET compiles on 3.12+. - Fall back to the legacy type-creation path when a base's metaclass is not compatible with the requested one. - Detect Windows free-threaded SOABI (cp315t-win_amd64). - Read the python3t.lib directory from the Python::SABIModule target. - Include test_chrono_second_tu when PYBIND11_TEST_OVERRIDE selects test_chrono. Assisted-by: ClaudeCode:claude-fable-5-1 ClaudeCode:claude-fable-5-1 --- docs/advanced/stable_abi.rst | 4 ++- include/pybind11/detail/class-inl.h | 10 ++++-- .../pybind11/detail/type_caster_base-inl.h | 34 +++++++++++++++++-- include/pybind11/detail/type_caster_base.h | 3 +- include/pybind11/pybind11-inl.h | 2 +- include/pybind11/pytypes.h | 4 +++ tests/CMakeLists.txt | 4 +++ tests/test_methods_and_attributes.cpp | 13 +++++++ tests/test_methods_and_attributes.py | 11 ++++++ tools/pybind11NewTools.cmake | 24 ++++++++++--- 10 files changed, 95 insertions(+), 14 deletions(-) diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index fc636dd54b..18cace424b 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -101,7 +101,9 @@ available and in how a few things are implemented. * - ``py::metaclass(handle)`` - restricted - The metaclass must not define ``__new__``, and a metaclass less - derived than ``pybind11_type`` resolves to ``pybind11_type``. + derived than ``pybind11_type`` resolves to ``pybind11_type``. The + metaclass of every base must be ``pybind11_type`` or a base of the + requested metaclass (``PyType_FromMetaclass()`` rejects a conflict). * - ``py::custom_type_setup`` - not available - Its callback receives a ``PyHeapTypeObject *``, which is opaque. diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 4928054130..89e4e39e8d 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -1076,10 +1076,14 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { : internals.default_metaclass; // PyType_FromMetaclass() cannot run a pre-PyType_Ready callback, rejects metaclasses with a - // custom tp_new, and (correctly) refuses a metaclass that is less derived than the one of - // the base; the legacy path accepts all three. + // custom tp_new, and (correctly) refuses a metaclass that is not at least as derived as the + // metaclass of each base; the legacy path accepts all three. # if !defined(Py_LIMITED_API) - if (rec.custom_type_setup_callback + bool base_metaclass_mismatch = false; + for (handle b : rec.bases) { + base_metaclass_mismatch |= !PyType_IsSubtype(metaclass, Py_TYPE(b.ptr())); + } + if (rec.custom_type_setup_callback || base_metaclass_mismatch || PyType_GetSlot(metaclass, Py_tp_new) != PyType_GetSlot(&PyType_Type, Py_tp_new) || (metaclass != internals.default_metaclass && !PyType_IsSubtype(metaclass, internals.default_metaclass))) { diff --git a/include/pybind11/detail/type_caster_base-inl.h b/include/pybind11/detail/type_caster_base-inl.h index d7f86d5cc8..13687d93d8 100644 --- a/include/pybind11/detail/type_caster_base-inl.h +++ b/include/pybind11/detail/type_caster_base-inl.h @@ -423,19 +423,47 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE bool isinstance_generic(handle obj, return isinstance(obj, type); } -PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle get_object_handle(const void *ptr, +PYBIND11_NOINLINE_ATTR PYBIND11_INLINE object get_object_handle(const void *ptr, const detail::type_info *type) { +#if defined(PYBIND11_OPAQUE_PYOBJECT) + // values_and_holders() takes the internals lock, which can suspend the instance map's + // critical section: take strong references first, then inspect them without the map lock. + std::vector candidates = with_instance_map(ptr, [&](instance_map &instances) { + std::vector found; + auto range = instances.equal_range(ptr); + for (auto it = range.first; it != range.second; ++it) { + PyObject *wrapper = nullptr; + if (it->second->registry_weakref != nullptr + && PyWeakref_GetRef(it->second->registry_weakref, &wrapper) == 1) { + found.emplace_back(reinterpret_steal(wrapper)); + } + } + return found; + }); + for (auto &wrapper : candidates) { + for (const auto &vh : values_and_holders(wrapper.ptr())) { + if (vh.type == type) { + return std::move(wrapper); + } + } + } + return object(); +#else return with_instance_map(ptr, [&](instance_map &instances) { auto range = instances.equal_range(ptr); for (auto it = range.first; it != range.second; ++it) { for (const auto &vh : values_and_holders(it->second)) { if (vh.type == type) { - return handle(instance_object(it->second)); + auto *wrapper = instance_object(it->second); + if (try_incref(wrapper)) { + return reinterpret_steal(wrapper); + } } } } - return handle(); + return object(); }); +#endif } PYBIND11_INLINE object cpp_conduit_method(handle self, diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index ab8b8e6268..cba3a9274e 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -280,7 +280,8 @@ class old_style_init_scope { bool was_active_ = false; }; -handle get_object_handle(const void *ptr, const detail::type_info *type); +/// The registered Python wrapper of `ptr` for `type` (a new reference), or a null object. +object get_object_handle(const void *ptr, const detail::type_info *type); // Information about how type_caster_generic::cast() can obtain its source object struct cast_sources { diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index 8da7234198..31094379a2 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -437,7 +437,7 @@ PYBIND11_NAMESPACE_BEGIN(detail) PYBIND11_INLINE function get_type_override(const void *this_ptr, const type_info *this_type, const char *name) { - handle self = get_object_handle(this_ptr, this_type); + object self = get_object_handle(this_ptr, this_type); if (!self) { return function(); } diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index c3a4f6843c..776ec8a99a 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -2647,6 +2647,10 @@ inline tuple get_bases(PyTypeObject *type) { /// `type.__mro__` as a tuple. inline tuple get_mro(PyTypeObject *type) { return reinterpret_borrow(type->tp_mro); } +/// `type.__dictoffset__` / `type.__basicsize__` (tp_dictoffset and tp_basicsize). +inline Py_ssize_t type_dictoffset(PyTypeObject *type) { return type->tp_dictoffset; } +inline Py_ssize_t type_basicsize(PyTypeObject *type) { return type->tp_basicsize; } + /// Look `name` up along the MRO of `type` without invoking descriptors, like `_PyType_Lookup`. /// Returns a null handle if the attribute is not found; never raises. (Borrowed here; the /// stable-ABI version below returns an owning `object`.) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0d8cf4ed79..23bee4d209 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -211,6 +211,10 @@ if(PYBIND11_TEST_OVERRIDE) # Instead of doing a direct override here, we iterate over the overrides without extension and # match them against entries from the PYBIND11_TEST_FILES, anything that not matches goes into the filter list. string(REGEX REPLACE "\\.[^.;]*;" ";" TEST_OVERRIDE_NO_EXT "${PYBIND11_TEST_OVERRIDE};") + # test_chrono.py imports the module built from this second translation unit. + if(test_chrono IN_LIST TEST_OVERRIDE_NO_EXT) + list(APPEND TEST_OVERRIDE_NO_EXT test_chrono_second_tu) + endif() string(REGEX REPLACE "\\.[^.;]*;" ";" TEST_FILES_NO_EXT "${PYBIND11_TEST_FILES};") # This allows the override to be done with extensions, preserving backwards compatibility. foreach(test_name ${TEST_FILES_NO_EXT}) diff --git a/tests/test_methods_and_attributes.cpp b/tests/test_methods_and_attributes.cpp index f46563cec3..6d70ee83c0 100644 --- a/tests/test_methods_and_attributes.cpp +++ b/tests/test_methods_and_attributes.cpp @@ -493,6 +493,19 @@ TEST_SUBMODULE(methods_and_attributes, m) { py::class_(m, "MetaclassOverride", py::metaclass((PyObject *) &PyType_Type)) .def_property_readonly_static("readonly", [](const py::object &) { return 1; }); +#if !defined(Py_LIMITED_API) + // test_metaclass_base_conflict: a derived class whose base has a metaclass that is neither a + // base nor a subtype of pybind11_type (rejected by PyType_FromMetaclass()). + auto type_type = py::reinterpret_borrow((PyObject *) &PyType_Type); + py::object custom_meta = type_type("CustomMeta", py::make_tuple(type_type), py::dict()); + struct MetaclassConflictBase {}; + struct MetaclassConflictDerived : MetaclassConflictBase {}; + py::class_(m, "MetaclassConflictBase", py::metaclass(custom_meta)) + .def(py::init<>()); + py::class_(m, "MetaclassConflictDerived") + .def(py::init<>()); +#endif + // test_overload_ordering m.def("overload_order", [](const std::string &) { return 1; }); m.def("overload_order", [](const std::string &) { return 2; }); diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index ded30b54f5..e5f8972fd9 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -270,6 +270,17 @@ def test_metaclass_override(): assert isinstance(m.MetaclassOverride.__dict__["readonly"], int) +@pytest.mark.skipif( + not hasattr(m, "MetaclassConflictBase"), reason="Not available under Py_LIMITED_API" +) +def test_metaclass_base_conflict(): + """A base with an unrelated metaclass does not block binding a derived class""" + + assert type(m.MetaclassConflictBase).__name__ == "CustomMeta" + assert type(m.MetaclassConflictDerived).__name__ == "pybind11_type" + assert isinstance(m.MetaclassConflictDerived(), m.MetaclassConflictBase) + + def test_no_mixed_overloads(): from pybind11_tests import detailed_error_messages_enabled diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index d7b4aed999..92468fc5a5 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -275,9 +275,10 @@ set(PYBIND11_ABI3T # True if the interpreter is a free-threaded build (needs the abi3t variant of the stable ABI). function(_pybind11_python_is_free_threaded out_var) + # SOABI is cpython-315t-... on POSIX and cp315t-win_amd64 on Windows. if(${_Python}_FREE_THREADED - OR "${${_Python}_SOABI}" MATCHES "^cpython-[0-9]+t" - OR "${PYTHON_MODULE_EXTENSION}" MATCHES "^\\.cpython-[0-9]+t") + OR "${${_Python}_SOABI}" MATCHES "^(cpython-|cp)[0-9]+t" + OR "${PYTHON_MODULE_EXTENSION}" MATCHES "^\\.(cpython-|cp)[0-9]+t") set(${out_var} ON PARENT_SCOPE) @@ -336,11 +337,24 @@ function(_pybind11_stable_abi_setup target_name) if(_abi3t AND NOT _free_threaded AND WIN32) - # Python::SABIModule is python3.lib here, but abi3t needs python3t.lib from the same directory. - target_include_directories(${target_name} SYSTEM PRIVATE ${${_Python}_INCLUDE_DIRS}) + # Python::SABIModule is python3.lib here, but abi3t needs python3t.lib from the same + # directory. Read it from the imported target: FindPython's variables are not visible when + # pybind11 was found in a subdirectory. (The headers come from pybind11::python_headers.) + set(_python3_lib "") + foreach(_prop IMPORTED_IMPLIB_RELEASE IMPORTED_IMPLIB IMPORTED_LOCATION_RELEASE + IMPORTED_LOCATION) + get_property( + _python3_lib + TARGET ${_Python}::SABIModule + PROPERTY ${_prop}) + if(_python3_lib) + break() + endif() + endforeach() + get_filename_component(_python3_lib_dir "${_python3_lib}" DIRECTORY) find_library( _pybind11_python3t_lib python3t - PATHS ${${_Python}_SABI_LIBRARY_DIRS} + PATHS ${_python3_lib_dir} ${${_Python}_SABI_LIBRARY_DIRS} NO_DEFAULT_PATH REQUIRED) target_link_libraries(${target_name} PRIVATE "${_pybind11_python3t_lib}") else()