pybind/pybind11
1
0
Fork 0
mirror of https://github.com/pybind/pybind11.git synced 2026-04-19 22:39:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: change PYBIND11_EMBEDDED_MODULE to multiphase init (#5665)

Browse Source 
* Move embedded modules to multiphase init

So that they too can support multi-interpreter and nogil tags

* Update the multiple interpreter test for embedded module changes

* Add a note to embedded module docs about the new tags

* Oops, missed a warning pop

* Remove unused variable

* Update ci.yml

* Fix this embedded GIL test for free-threading

* Oops, need to use ptr() here

* This test created a subinterpreter when PYBIND11_SUBINTERPRETER_SUPPORT was off

So the fix is really this test should not be run in these older versions at all.

The hang was a GIL issue between the subinterpreters during pybind11::exception::what().

* fix: standard mutex for 3.13t

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

---------

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>
Co-authored-by: Henry Schreiner <HenrySchreinerIII@gmail.com>
This commit is contained in:
b-pass
2025-05-18 09:39:26 -04:00
committed by GitHub
parent 094343c74a
commit 9afc9c4ffb
6 changed files with 70 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/ci.yml vendored
View File

@@ -34,7 +34,9 @@ jobs:
python:
- '3.8'
- '3.13'
- '3.13t'
- '3.14'
- '3.14t'
- 'pypy-3.10'
- 'pypy-3.11'
- 'graalpy-24.2'

5
docs/advanced/embedding.rst
View File

@@ -212,6 +212,11 @@ naturally:
assert(locals["message"].cast<std::string>() == "1 + 2 = 3");
}
``PYBIND11_EMBEDDED_MODULE`` also accepts
:func:`py::mod_gil_not_used()`,
:func:`py::multiple_interpreters::per_interpreter_gil()`, and
:func:`py::multiple_interpreters::shared_gil()` tags just like ``PYBIND11_MODULE``.
See :ref:`misc_subinterp` and :ref:`misc_free_threading` for more information.
Interpreter lifetime
====================

4
docs/advanced/misc.rst
View File

@@ -155,6 +155,8 @@ following checklist.
within pybind11 that will throw exceptions on certain GIL handling errors
(reference counting operations).
.. _misc_free_threading:
Free-threading support
==================================================================
@@ -178,6 +180,8 @@ your code is thread safe. Modules must still be built against the Python free-t
enable free-threading, even if they specify this tag. Adding this tag does not break
compatibility with non-free-threaded Python.
.. _misc_subinterp:
Sub-interpreter support
==================================================================

37
include/pybind11/embed.h
View File

@@ -38,24 +38,43 @@
});
}
\endrst */
#define PYBIND11_EMBEDDED_MODULE(name, variable) \
PYBIND11_WARNING_PUSH
PYBIND11_WARNING_DISABLE_CLANG("-Wgnu-zero-variadic-macro-arguments")
#define PYBIND11_EMBEDDED_MODULE(name, variable, ...) \
static ::pybind11::module_::module_def PYBIND11_CONCAT(pybind11_module_def_, name); \
static ::pybind11::module_::slots_array PYBIND11_CONCAT(pybind11_module_slots_, name); \
static int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject *); \
static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &); \
static PyObject PYBIND11_CONCAT(*pybind11_init_wrapper_, name)() { \
auto m = ::pybind11::module_::create_extension_module( \
PYBIND11_TOSTRING(name), nullptr, &PYBIND11_CONCAT(pybind11_module_def_, name)); \
try { \
PYBIND11_CONCAT(pybind11_init_, name)(m); \
return m.ptr(); \
} \
PYBIND11_CATCH_INIT_EXCEPTIONS \
return nullptr; \
static auto result = []() { \
auto &slots = PYBIND11_CONCAT(pybind11_module_slots_, name); \
slots[0] = {Py_mod_exec, \
reinterpret_cast<void *>(&PYBIND11_CONCAT(pybind11_exec_, name))}; \
slots[1] = {0, nullptr}; \
return ::pybind11::module_::initialize_multiphase_module_def( \
PYBIND11_TOSTRING(name), \
nullptr, \
&PYBIND11_CONCAT(pybind11_module_def_, name), \
slots, \
##__VA_ARGS__); \
}(); \
return result.ptr(); \
} \
PYBIND11_EMBEDDED_MODULE_IMPL(name) \
::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name)( \
PYBIND11_TOSTRING(name), PYBIND11_CONCAT(pybind11_init_impl_, name)); \
int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject * pm) { \
try { \
auto m = pybind11::reinterpret_borrow<::pybind11::module_>(pm); \
PYBIND11_CONCAT(pybind11_init_, name)(m); \
return 0; \
} \
PYBIND11_CATCH_INIT_EXCEPTIONS \
return -1; \
} \
void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ \
& variable) // NOLINT(bugprone-macro-parentheses)
PYBIND11_WARNING_POP
PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
PYBIND11_NAMESPACE_BEGIN(detail)

2
include/pybind11/pybind11.h
View File

@@ -1484,7 +1484,7 @@ public:
using slots_array = std::array<PyModuleDef_Slot, 4>;
/** \rst
Initialized a module def for use with multi-phase module initialization.
Initialize a module def for use with multi-phase module initialization.
``def`` should point to a statically allocated module_def.
``slots`` must already contain a Py_mod_exec or Py_mod_create slot and will be filled with

41
tests/test_embed/test_interpreter.cpp
View File

@@ -55,7 +55,7 @@ class test_override_cache_helper_trampoline : public test_override_cache_helper
int func() override { PYBIND11_OVERRIDE(int, test_override_cache_helper, func); }
};
PYBIND11_EMBEDDED_MODULE(widget_module, m) {
PYBIND11_EMBEDDED_MODULE(widget_module, m, py::multiple_interpreters::per_interpreter_gil()) {
py::class_<Widget, PyWidget>(m, "Widget")
.def(py::init<std::string>())
.def_property_readonly("the_message", &Widget::the_message);
@@ -336,6 +336,7 @@ TEST_CASE("Restart the interpreter") {
REQUIRE(py_widget.attr("the_message").cast<std::string>() == "Hello after restart");
}
#if defined(PYBIND11_SUBINTERPRETER_SUPPORT)
TEST_CASE("Subinterpreter") {
py::module_::import("external_module"); // in the main interpreter
@@ -347,6 +348,10 @@ TEST_CASE("Subinterpreter") {
REQUIRE(m.attr("add")(1, 2).cast<int>() == 3);
}
auto main_int
= py::module_::import("external_module").attr("internals_at")().cast<uintptr_t>();
REQUIRE(has_state_dict_internals_obj());
REQUIRE(has_pybind11_internals_static());
@@ -359,7 +364,6 @@ TEST_CASE("Subinterpreter") {
// Subinterpreters get their own copy of builtins.
REQUIRE_FALSE(has_state_dict_internals_obj());
#if defined(PYBIND11_SUBINTERPRETER_SUPPORT) && PY_VERSION_HEX >= 0x030C0000
// internals hasn't been populated yet, but will be different for the subinterpreter
REQUIRE_FALSE(has_pybind11_internals_static());
@@ -369,14 +373,12 @@ TEST_CASE("Subinterpreter") {
py::detail::get_internals();
REQUIRE(has_pybind11_internals_static());
REQUIRE(get_details_as_uintptr() == ext_int);
#else
// This static is still defined
REQUIRE(has_pybind11_internals_static());
#endif
REQUIRE(main_int != ext_int);
// Modules tags should be gone.
REQUIRE_FALSE(py::hasattr(py::module_::import("__main__"), "tag"));
{
REQUIRE_NOTHROW(py::module_::import("widget_module"));
auto m = py::module_::import("widget_module");
REQUIRE_FALSE(py::hasattr(m, "extension_module_tag"));
@@ -397,7 +399,6 @@ TEST_CASE("Subinterpreter") {
REQUIRE(has_state_dict_internals_obj());
}
#if defined(PYBIND11_SUBINTERPRETER_SUPPORT)
TEST_CASE("Multiple Subinterpreters") {
// Make sure the module is in the main interpreter and save its pointer
auto *main_ext = py::module_::import("external_module").ptr();
@@ -512,10 +513,11 @@ TEST_CASE("Per-Subinterpreter GIL") {
// we have switched to the new interpreter and released the main gil
// widget_module did not provide the mod_per_interpreter_gil tag, so it cannot be imported
// trampoline_module did not provide the per_interpreter_gil tag, so it cannot be
// imported
bool caught = false;
try {
py::module_::import("widget_module");
py::module_::import("trampoline_module");
} catch (pybind11::error_already_set &pe) {
T_REQUIRE(pe.matches(PyExc_ImportError));
std::string msg(pe.what());
@@ -525,6 +527,9 @@ TEST_CASE("Per-Subinterpreter GIL") {
}
T_REQUIRE(caught);
// widget_module did provide the per_interpreter_gil tag, so it this does not throw
py::module_::import("widget_module");
T_REQUIRE(!py::hasattr(py::module_::import("external_module"), "multi_interp"));
py::module_::import("external_module").attr("multi_interp") = std::to_string(num);
@@ -547,8 +552,8 @@ TEST_CASE("Per-Subinterpreter GIL") {
Py_EndInterpreter(sub);
PyThreadState_Swap(
main_tstate); // switch back so the scoped_acquire can release the GIL properly
// switch back so the scoped_acquire can release the GIL properly
PyThreadState_Swap(main_tstate);
};
std::thread t1(thread_main, 1);
@@ -622,12 +627,26 @@ TEST_CASE("Threads") {
{
py::gil_scoped_release gil_release{};
#if defined(Py_GIL_DISABLED) && PY_VERSION_HEX < 0x030E0000
std::mutex mutex;
#endif
auto threads = std::vector<std::thread>();
for (auto i = 0; i < num_threads; ++i) {
threads.emplace_back([&]() {
py::gil_scoped_acquire gil{};
#ifdef Py_GIL_DISABLED
# if PY_VERSION_HEX < 0x030E0000
std::lock_guard<std::mutex> lock(mutex);
locals["count"] = locals["count"].cast<int>() + 1;
# else
Py_BEGIN_CRITICAL_SECTION(locals.ptr());
locals["count"] = locals["count"].cast<int>() + 1;
Py_END_CRITICAL_SECTION();
# endif
#else
locals["count"] = locals["count"].cast<int>() + 1;
#endif
});
}