pybind/pybind11
1
0
Fork 0
mirror of https://github.com/pybind/pybind11.git synced 2026-04-19 22:39:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make function record subinterpreter safe (#5771)

Browse Source 
* Make function_record type subinterpreter safe

* Get rid of static state in implicit conversion

* style: pre-commit fixes

* Fix lambda

* Bump ABI because we added an internals member

* Set __module__ on the type instance to get rid of DepricationWarning

* Work around internal compiler error in CUDA by not using typedef

hopefully

* Make clang-tidy happy

* Use the same __module__ as pybind11_static_property

* style: pre-commit fixes

* Oops, find-replace error

* style: pre-commit fixes

* Move the once initialization to happen more behind the scenes

* Oops, need those casts...

* Undo implicit conversion change, will do a separate PR

* Use local_internals for function_record pointer to avoid ABI bump

* style: pre-commit fixes

* Get rid of this auto for readability

* Change back to using unqualified tp_name, set __module__ attribute, explicitly add Py_TPFLAGS_HEAPTYPE → does not resolve DeprecationWarning :-(

* Revert "Change back to using unqualified tp_name, set __module__ attribute, explicitly add Py_TPFLAGS_HEAPTYPE → does not resolve DeprecationWarning :-("

This reverts commit 9ccd6de9b7.

* Add Py_TPFLAGS_HEAPTYPE to be explicit (more readable).

* Remove obsolete PYBIND11_WARNING_DISABLE_...

* Make tp_plainname_impl, tp_qualname_impl more DRY

* Change PYBIND11_INTERNAL_MODULE_NAME → PYBIND11_DUMMY_MODULE_NAME

* Add a long comment to explain the tp_qualname_impl workaround.

* Rename local_internals::function_record → function_record_py_type

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Ralf W. Grosse-Kunstleve <rwgkio@gmail.com>
Co-authored-by: Ralf W. Grosse-Kunstleve <rgrossekunst@nvidia.com>
This commit is contained in:
b-pass
2025-08-04 08:00:33 -04:00
committed by GitHub
parent 0db3d59fdf
commit c5e8fec920
5 changed files with 60 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
include/pybind11/detail/class.h
View File

@@ -98,7 +98,7 @@ inline PyTypeObject *make_static_property_type() {
pybind11_fail("make_static_property_type(): failure in PyType_Ready()!");
}
setattr((PyObject *) type, "__module__", str("pybind11_builtins"));
setattr((PyObject *) type, "__module__", str(PYBIND11_DUMMY_MODULE_NAME));
PYBIND11_SET_OLDPY_QUALNAME(type, name_obj);
return type;
@@ -282,7 +282,7 @@ inline PyTypeObject *make_default_metaclass() {
pybind11_fail("make_default_metaclass(): failure in PyType_Ready()!");
}
setattr((PyObject *) type, "__module__", str("pybind11_builtins"));
setattr((PyObject *) type, "__module__", str(PYBIND11_DUMMY_MODULE_NAME));
PYBIND11_SET_OLDPY_QUALNAME(type, name_obj);
return type;
@@ -544,7 +544,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass) {
pybind11_fail("PyType_Ready failed in make_object_base_type(): " + error_string());
}
setattr((PyObject *) type, "__module__", str("pybind11_builtins"));
setattr((PyObject *) type, "__module__", str(PYBIND11_DUMMY_MODULE_NAME));
PYBIND11_SET_OLDPY_QUALNAME(type, name_obj);
assert(!PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC));

125
include/pybind11/detail/function_record_pyobject.h
View File

@@ -45,83 +45,61 @@ static PyMethodDef tp_methods_impl[]
nullptr},
{nullptr, nullptr, 0, nullptr}};
// Python 3.12+ emits a DeprecationWarning for heap types whose tp_name does
// not contain a dot ('.') and that lack a __module__ attribute. For pybind11's
// internal function_record type, we do not have an actual module object to
// attach, so we cannot use PyType_FromModuleAndSpec (introduced in Python 3.9)
// to set __module__ automatically.
//
// As a workaround, we define a "qualified" type name that includes a dummy
// module name (PYBIND11_DUMMY_MODULE_NAME). This is non‑idiomatic but avoids
// the deprecation warning, and results in reprs like
//
// <class 'pybind11_builtins.pybind11_detail_function_record_...'>
//
// even though no real pybind11_builtins module exists. If pybind11 gains an
// actual module object in the future, this code should switch to
// PyType_FromModuleAndSpec for Python 3.9+ and drop the dummy module
// workaround.
//
// Note that this name is versioned.
constexpr char tp_name_impl[]
= "pybind11_detail_function_record_" PYBIND11_DETAIL_FUNCTION_RECORD_ABI_ID
"_" PYBIND11_PLATFORM_ABI_ID;
#define PYBIND11_DETAIL_FUNCTION_RECORD_TP_PLAINNAME \
"pybind11_detail_function_record_" PYBIND11_DETAIL_FUNCTION_RECORD_ABI_ID \
"_" PYBIND11_PLATFORM_ABI_ID
constexpr char tp_plainname_impl[] = PYBIND11_DETAIL_FUNCTION_RECORD_TP_PLAINNAME;
constexpr char tp_qualname_impl[]
= PYBIND11_DUMMY_MODULE_NAME "." PYBIND11_DETAIL_FUNCTION_RECORD_TP_PLAINNAME;
PYBIND11_NAMESPACE_END(function_record_PyTypeObject_methods)
// Designated initializers are a C++20 feature:
// https://en.cppreference.com/w/cpp/language/aggregate_initialization#Designated_initializers
// MSVC rejects them unless /std:c++20 is used (error code C7555).
PYBIND11_WARNING_PUSH
PYBIND11_WARNING_DISABLE_CLANG("-Wmissing-field-initializers")
#if defined(__GNUC__) && __GNUC__ >= 8
PYBIND11_WARNING_DISABLE_GCC("-Wmissing-field-initializers")
#endif
static PyTypeObject function_record_PyTypeObject = {
PyVarObject_HEAD_INIT(nullptr, 0)
/* const char *tp_name */ function_record_PyTypeObject_methods::tp_name_impl,
/* Py_ssize_t tp_basicsize */ sizeof(function_record_PyObject),
/* Py_ssize_t tp_itemsize */ 0,
/* destructor tp_dealloc */ function_record_PyTypeObject_methods::tp_dealloc_impl,
/* Py_ssize_t tp_vectorcall_offset */ 0,
/* getattrfunc tp_getattr */ nullptr,
/* setattrfunc tp_setattr */ nullptr,
/* PyAsyncMethods *tp_as_async */ nullptr,
/* reprfunc tp_repr */ nullptr,
/* PyNumberMethods *tp_as_number */ nullptr,
/* PySequenceMethods *tp_as_sequence */ nullptr,
/* PyMappingMethods *tp_as_mapping */ nullptr,
/* hashfunc tp_hash */ nullptr,
/* ternaryfunc tp_call */ nullptr,
/* reprfunc tp_str */ nullptr,
/* getattrofunc tp_getattro */ nullptr,
/* setattrofunc tp_setattro */ nullptr,
/* PyBufferProcs *tp_as_buffer */ nullptr,
/* unsigned long tp_flags */ Py_TPFLAGS_DEFAULT,
/* const char *tp_doc */ nullptr,
/* traverseproc tp_traverse */ nullptr,
/* inquiry tp_clear */ nullptr,
/* richcmpfunc tp_richcompare */ nullptr,
/* Py_ssize_t tp_weaklistoffset */ 0,
/* getiterfunc tp_iter */ nullptr,
/* iternextfunc tp_iternext */ nullptr,
/* struct PyMethodDef *tp_methods */ function_record_PyTypeObject_methods::tp_methods_impl,
/* struct PyMemberDef *tp_members */ nullptr,
/* struct PyGetSetDef *tp_getset */ nullptr,
/* struct _typeobject *tp_base */ nullptr,
/* PyObject *tp_dict */ nullptr,
/* descrgetfunc tp_descr_get */ nullptr,
/* descrsetfunc tp_descr_set */ nullptr,
/* Py_ssize_t tp_dictoffset */ 0,
/* initproc tp_init */ function_record_PyTypeObject_methods::tp_init_impl,
/* allocfunc tp_alloc */ function_record_PyTypeObject_methods::tp_alloc_impl,
/* newfunc tp_new */ function_record_PyTypeObject_methods::tp_new_impl,
/* freefunc tp_free */ function_record_PyTypeObject_methods::tp_free_impl,
/* inquiry tp_is_gc */ nullptr,
/* PyObject *tp_bases */ nullptr,
/* PyObject *tp_mro */ nullptr,
/* PyObject *tp_cache */ nullptr,
/* PyObject *tp_subclasses */ nullptr,
/* PyObject *tp_weaklist */ nullptr,
/* destructor tp_del */ nullptr,
/* unsigned int tp_version_tag */ 0,
/* destructor tp_finalize */ nullptr,
/* vectorcallfunc tp_vectorcall */ nullptr,
};
PYBIND11_WARNING_POP
static PyType_Slot function_record_PyType_Slots[] = {
{Py_tp_dealloc,
reinterpret_cast<void *>(function_record_PyTypeObject_methods::tp_dealloc_impl)},
{Py_tp_methods,
reinterpret_cast<void *>(function_record_PyTypeObject_methods::tp_methods_impl)},
{Py_tp_init, reinterpret_cast<void *>(function_record_PyTypeObject_methods::tp_init_impl)},
{Py_tp_alloc, reinterpret_cast<void *>(function_record_PyTypeObject_methods::tp_alloc_impl)},
{Py_tp_new, reinterpret_cast<void *>(function_record_PyTypeObject_methods::tp_new_impl)},
{Py_tp_free, reinterpret_cast<void *>(function_record_PyTypeObject_methods::tp_free_impl)},
{0, nullptr}};
static bool function_record_PyTypeObject_PyType_Ready_first_call = true;
static PyType_Spec function_record_PyType_Spec
= {function_record_PyTypeObject_methods::tp_qualname_impl,
sizeof(function_record_PyObject),
0,
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE,
function_record_PyType_Slots};
inline void function_record_PyTypeObject_PyType_Ready() {
if (function_record_PyTypeObject_PyType_Ready_first_call) {
if (PyType_Ready(&function_record_PyTypeObject) < 0) {
inline PyTypeObject *get_function_record_PyTypeObject() {
PyTypeObject *&py_type_obj = detail::get_local_internals().function_record_py_type;
if (!py_type_obj) {
PyObject *py_obj = PyType_FromSpec(&function_record_PyType_Spec);
if (py_obj == nullptr) {
throw error_already_set();
}
function_record_PyTypeObject_PyType_Ready_first_call = false;
py_type_obj = reinterpret_cast<PyTypeObject *>(py_obj);
}
return py_type_obj;
}
inline bool is_function_record_PyObject(PyObject *obj) {
@@ -129,12 +107,17 @@ inline bool is_function_record_PyObject(PyObject *obj) {
return false;
}
PyTypeObject *obj_type = Py_TYPE(obj);
PyTypeObject *frtype = get_function_record_PyTypeObject();
// Fast path (pointer comparison).
if (obj_type == &function_record_PyTypeObject) {
if (obj_type == frtype) {
return true;
}
// This works across extension modules. Note that tp_name is versioned.
if (strcmp(obj_type->tp_name, function_record_PyTypeObject.tp_name) == 0) {
if (strcmp(obj_type->tp_name, function_record_PyTypeObject_methods::tp_qualname_impl) == 0
|| strcmp(obj_type->tp_name, function_record_PyTypeObject_methods::tp_plainname_impl)
== 0) {
return true;
}
return false;
@@ -148,7 +131,7 @@ inline function_record *function_record_ptr_from_PyObject(PyObject *obj) {
}
inline object function_record_PyObject_New() {
auto *py_func_rec = PyObject_New(function_record_PyObject, &function_record_PyTypeObject);
auto *py_func_rec = PyObject_New(function_record_PyObject, get_function_record_PyTypeObject());
if (py_func_rec == nullptr) {
throw error_already_set();
}

4
include/pybind11/detail/internals.h
View File

@@ -132,7 +132,8 @@ private:
PYBIND11_NAMESPACE_BEGIN(detail)
constexpr const char *internals_function_record_capsule_name = "pybind11_function_record_capsule";
// This does NOT actually exist as a module.
#define PYBIND11_DUMMY_MODULE_NAME "pybind11_builtins"
// Forward declarations
inline PyTypeObject *make_static_property_type();
@@ -298,6 +299,7 @@ struct internals {
struct local_internals {
type_map<type_info *> registered_types_cpp;
std::forward_list<ExceptionTranslator> registered_exception_translators;
PyTypeObject *function_record_py_type = nullptr;
};
enum class holder_enum_t : uint8_t {

1
include/pybind11/pybind11.h
View File

@@ -631,7 +631,6 @@ protected:
= reinterpret_cast<PyCFunction>(reinterpret_cast<void (*)()>(dispatcher));
rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS;
detail::function_record_PyTypeObject_PyType_Ready(); // Call-once initialization.
object py_func_rec = detail::function_record_PyObject_New();
((detail::function_record_PyObject *) py_func_rec.ptr())->cpp_func_rec
= unique_rec.release();

6
tests/test_callbacks.cpp
View File

@@ -286,13 +286,7 @@ TEST_SUBMODULE(callbacks, m) {
return &def;
}();
// rec_capsule with name that has the same value (but not pointer) as our internal one
// This capsule should be detected by our code as foreign and not inspected as the pointers
// shouldn't match
constexpr const char *rec_capsule_name
= pybind11::detail::internals_function_record_capsule_name;
py::capsule rec_capsule(std::malloc(1), [](void *data) { std::free(data); });
rec_capsule.set_name(rec_capsule_name);
m.add_object("custom_function", PyCFunction_New(custom_def, rec_capsule.ptr()));
// rec_capsule with nullptr name