From 4f81a1250748008965c59f67b7ec1c10e40ce600 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 29 Jan 2026 02:02:08 -0500 Subject: [PATCH 1/8] Fix deadlock in test with free threading (#5973) Importing "widget_module" re-enables the GIL. In current versions of CPython, this requires pausing all threads attached to all interpreters. The spinning on sync/num without a py::gil_scoped_release causes occasional deadlocks. --- tests/test_with_catch/test_subinterpreter.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/test_with_catch/test_subinterpreter.cpp b/tests/test_with_catch/test_subinterpreter.cpp index e322e0fe9..35b7f0233 100644 --- a/tests/test_with_catch/test_subinterpreter.cpp +++ b/tests/test_with_catch/test_subinterpreter.cpp @@ -501,15 +501,21 @@ TEST_CASE("Per-Subinterpreter GIL") { // wait for something to set sync to our thread number // we are holding our subinterpreter's GIL - while (sync != num) - std::this_thread::sleep_for(std::chrono::microseconds(1)); + { + py::gil_scoped_release nogil; + while (sync != num) + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } // now change it so the next thread can move on ++sync; // but keep holding the GIL until after the next thread moves on as well - while (sync == num + 1) - std::this_thread::sleep_for(std::chrono::microseconds(1)); + { + py::gil_scoped_release nogil; + while (sync == num + 1) + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } // one last check before quitting the thread, the internals should be different auto sub_int From 95d4af7ff90c4c1d98087e629a57c8d969897e45 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 1 Feb 2026 13:09:06 +0700 Subject: [PATCH 2/8] Fix CI issue: numpy requirement for Python 3.14 on ARM64 Windows (#5977) * Fix pip install conflicts in tests/requirements.txt - Fix numpy version conflict for Python 3.14 on ARM64 by excluding Python 3.14+ from the ARM64-specific numpy>=2.3.0 requirement - Add scipy requirement for Python 3.13 on Windows Co-authored-by: Cursor * Revert "Fix pip install conflicts in tests/requirements.txt" This reverts commit 1d63c73be4fca63d660488e6efab703f51405fc7. * Fix numpy requirement for Python 3.14 on ARM64 Windows Change numpy==2.4.0 to numpy>=2.4.0 for Python 3.14+ to allow pip to install numpy 2.4.1 or later versions, which are available for Python 3.14 on ARM64 Windows (MSYS2) where numpy 2.4.0 is not. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 50dd10381..39bf86785 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -10,7 +10,7 @@ numpy~=1.22.2; platform_python_implementation=="CPython" and python_version=="3. numpy~=1.26.0; platform_python_implementation=="CPython" and python_version>="3.11" and python_version<"3.13" and platform_machine!="ARM64" numpy>=2.3.0; platform_python_implementation=="CPython" and python_version>="3.11" and platform_machine=="ARM64" numpy~=2.2.0; platform_python_implementation=="CPython" and python_version=="3.13" and platform_machine!="ARM64" -numpy==2.4.0; platform_python_implementation=="CPython" and python_version>="3.14" +numpy>=2.4.0; platform_python_implementation=="CPython" and python_version>="3.14" pytest>=6 pytest-timeout scipy~=1.5.4; platform_python_implementation=="CPython" and python_version<"3.10" From 0080cae388f6e6ec8d12b51138c6389e0a0c6e9b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 31 Jan 2026 22:10:57 -0800 Subject: [PATCH 3/8] chore(deps): bump jwlawson/actions-setup-cmake in the actions group (#5978) Bumps the actions group with 1 update: [jwlawson/actions-setup-cmake](https://github.com/jwlawson/actions-setup-cmake). Updates `jwlawson/actions-setup-cmake` from 2.0 to 2.1 - [Release notes](https://github.com/jwlawson/actions-setup-cmake/releases) - [Commits](https://github.com/jwlawson/actions-setup-cmake/compare/v2.0...v2.1) --- updated-dependencies: - dependency-name: jwlawson/actions-setup-cmake dependency-version: '2.1' dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 12 ++++++------ .github/workflows/configure.yml | 2 +- .github/workflows/upstream.yml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45f867588..cda2c214d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -293,7 +293,7 @@ jobs: debug: ${{ matrix.python-debug }} - name: Update CMake - uses: jwlawson/actions-setup-cmake@v2.0 + uses: jwlawson/actions-setup-cmake@v2.1 - name: Valgrind cache if: matrix.valgrind @@ -561,7 +561,7 @@ jobs: run: python3 -m pip install --upgrade pip - name: Update CMake - uses: jwlawson/actions-setup-cmake@v2.0 + uses: jwlawson/actions-setup-cmake@v2.1 - name: Configure shell: bash @@ -897,7 +897,7 @@ jobs: ${{ matrix.python == '3.13' && runner.os == 'Windows' }} - name: Update CMake - uses: jwlawson/actions-setup-cmake@v2.0 + uses: jwlawson/actions-setup-cmake@v2.1 - name: Prepare MSVC uses: ilammy/msvc-dev-cmd@v1.13.0 @@ -947,7 +947,7 @@ jobs: architecture: x86 - name: Update CMake - uses: jwlawson/actions-setup-cmake@v2.0 + uses: jwlawson/actions-setup-cmake@v2.1 - name: Prepare MSVC uses: ilammy/msvc-dev-cmd@v1.13.0 @@ -998,7 +998,7 @@ jobs: run: python3 -m pip install -r tests/requirements.txt - name: Update CMake - uses: jwlawson/actions-setup-cmake@v2.0 + uses: jwlawson/actions-setup-cmake@v2.1 - name: Configure C++20 run: > @@ -1180,7 +1180,7 @@ jobs: python-version: ${{ matrix.python }} - name: Update CMake - uses: jwlawson/actions-setup-cmake@v2.0 + uses: jwlawson/actions-setup-cmake@v2.1 - name: Install ninja-build tool uses: seanmiddleditch/gha-setup-ninja@v6 diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 226e3f718..931c0bff2 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -64,7 +64,7 @@ jobs: # An action for adding a specific version of CMake: # https://github.com/jwlawson/actions-setup-cmake - name: Setup CMake ${{ matrix.cmake }} - uses: jwlawson/actions-setup-cmake@v2.0 + uses: jwlawson/actions-setup-cmake@v2.1 with: cmake-version: ${{ matrix.cmake }} diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 890ae0b6f..051cffc04 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -36,7 +36,7 @@ jobs: run: sudo apt-get install libboost-dev - name: Update CMake - uses: jwlawson/actions-setup-cmake@v2.0 + uses: jwlawson/actions-setup-cmake@v2.1 - name: Run pip installs run: | From e7754de037fe6bc9910e1b09691c7d0ca2ee3057 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Mon, 2 Feb 2026 13:54:05 +0800 Subject: [PATCH 4/8] Revert internals destruction and add test for internals recreation (#5972) * Bump internals version * Prevent internals destruction before all pybind11 types are destroyed * Use Py_XINCREF and Py_XDECREF * Hold GIL before decref * Use weakrefs * Remove unused code * Move code location * Move code location * Move code location * Try add tests * Fix PYTHONPATH * Fix PYTHONPATH * Skip tests for subprocess * Revert to leak internals * Revert to leak internals * Revert "Revert to leak internals" This reverts commit c5ec1cf8862b43c125bdaf582445f3cbb1347017. This reverts commit 72c2e0aa9b481debd94557b629e86b954543d64b. * Revert internals version bump * Reapply to leak internals This reverts commit 8f25a254e893817c2ed70d97cde7f62778a14de2. * Add re-entrancy detection for internals creation Prevent re-creation of internals after destruction during interpreter shutdown. If pybind11 code runs after internals have been destroyed, fail early with a clear error message instead of silently creating new empty internals that would cause type lookup failures. Co-Authored-By: Claude Opus 4.5 * Fix C++11/C++14 support * Add lock under multiple interpreters * Try fix tests * Try fix tests * Try fix tests * Update comments and assertion messages * Update comments and assertion messages * Update comments * Update lock scope * Use original pointer type for Windows * Change hard error to warning * Update lock scope * Update lock scope to resolve deadlock * Remove scope release of GIL * Update comments * Lock pp on reset * Mark content created after assignment * Update comments * Simplify implementation * Update lock scope when delete unique_ptr --------- Co-authored-by: Claude Opus 4.5 --- docs/advanced/classes.rst | 37 ++++-- include/pybind11/detail/internals.h | 169 ++++++++++++++++++---------- tests/env.py | 28 +++++ tests/test_custom_type_setup.cpp | 73 ++++++++++-- tests/test_custom_type_setup.py | 46 +++++++- tests/test_multiple_interpreters.py | 37 ++---- 6 files changed, 276 insertions(+), 114 deletions(-) diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index faaba38b8..bfbaea60a 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -1381,11 +1381,22 @@ You can do that using ``py::custom_type_setup``: .. code-block:: cpp - struct OwnsPythonObjects { - py::object value = py::none(); + struct ContainerOwnsPythonObjects { + std::vector list; + + void append(const py::object &obj) { list.emplace_back(obj); } + py::object at(py::ssize_t index) const { + if (index >= size() || index < 0) { + throw py::index_error("Index out of range"); + } + return list.at(py::size_t(index)); + } + py::ssize_t size() const { return py::ssize_t_cast(list.size()); } + void clear() { list.clear(); } }; - py::class_ cls( - m, "OwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) { + + py::class_ cls( + m, "ContainerOwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) { @@ -1394,20 +1405,28 @@ You can do that using ``py::custom_type_setup``: Py_VISIT(Py_TYPE(self_base)); #endif if (py::detail::is_holder_constructed(self_base)) { - auto &self = py::cast(py::handle(self_base)); - Py_VISIT(self.value.ptr()); + auto &self = py::cast(py::handle(self_base)); + for (auto &item : self.list) { + Py_VISIT(item.ptr()); + } } return 0; }; type->tp_clear = [](PyObject *self_base) { if (py::detail::is_holder_constructed(self_base)) { - auto &self = py::cast(py::handle(self_base)); - self.value = py::none(); + auto &self = py::cast(py::handle(self_base)); + for (auto &item : self.list) { + Py_CLEAR(item.ptr()); + } + self.list.clear(); } return 0; }; })); cls.def(py::init<>()); - cls.def_readwrite("value", &OwnsPythonObjects::value); + cls.def("append", &ContainerOwnsPythonObjects::append); + cls.def("at", &ContainerOwnsPythonObjects::at); + cls.def("size", &ContainerOwnsPythonObjects::size); + cls.def("clear", &ContainerOwnsPythonObjects::clear); .. versionadded:: 2.8 diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 9b3e69f4d..7cfa5da92 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -337,19 +337,7 @@ struct internals { internals(internals &&other) = delete; internals &operator=(const internals &other) = delete; internals &operator=(internals &&other) = delete; - ~internals() { - // Normally this destructor runs during interpreter finalization and it may DECREF things. - // In odd finalization scenarios it might end up running after the interpreter has - // completely shut down, In that case, we should not decref these objects because pymalloc - // is gone. This also applies across sub-interpreters, we should only DECREF when the - // original owning interpreter is active. - auto *cur_istate = get_interpreter_state_unchecked(); - if (cur_istate && cur_istate == istate) { - Py_CLEAR(instance_base); - Py_CLEAR(default_metaclass); - Py_CLEAR(static_property_type); - } - } + ~internals() = default; }; // the internals struct (above) is shared between all the modules. local_internals are only @@ -359,8 +347,6 @@ struct internals { // impact any other modules, because the only things accessing the local internals is the // module that contains them. struct local_internals { - local_internals() : istate(get_interpreter_state_unchecked()) {} - // It should be safe to use fast_type_map here because this entire // data structure is scoped to our single module, and thus a single // DSO and single instance of type_info for any particular type. @@ -368,19 +354,6 @@ struct local_internals { std::forward_list registered_exception_translators; PyTypeObject *function_record_py_type = nullptr; - PyInterpreterState *istate = nullptr; - - ~local_internals() { - // Normally this destructor runs during interpreter finalization and it may DECREF things. - // In odd finalization scenarios it might end up running after the interpreter has - // completely shut down, In that case, we should not decref these objects because pymalloc - // is gone. This also applies across sub-interpreters, we should only DECREF when the - // original owning interpreter is active. - auto *cur_istate = get_interpreter_state_unchecked(); - if (cur_istate && cur_istate == istate) { - Py_CLEAR(function_record_py_type); - } - } }; enum class holder_enum_t : uint8_t { @@ -576,6 +549,10 @@ inline void translate_local_exception(std::exception_ptr p) { } #endif +// Sentinel value for the `dtor` parameter of `atomic_get_or_create_in_state_dict`. +// Indicates no destructor was explicitly provided (distinct from nullptr, which means "leak"). +#define PYBIND11_DTOR_USE_DELETE (reinterpret_cast(1)) + // Get or create per-storage capsule in the current interpreter's state dict. // - The storage is interpreter-dependent: different interpreters will have different storage. // This is important when using multiple-interpreters, to avoid sharing unshareable objects @@ -592,9 +569,14 @@ inline void translate_local_exception(std::exception_ptr p) { // // Returns: pair of (pointer to storage, bool indicating if newly created). // The bool follows std::map::insert convention: true = created, false = existed. +// `dtor`: optional destructor called when the interpreter shuts down. +// - If not provided: the storage will be deleted using `delete`. +// - If nullptr: the storage will be leaked (useful for singletons that outlive the interpreter). +// - If a function: that function will be called with the capsule object. template std::pair atomic_get_or_create_in_state_dict(const char *key, - void (*dtor)(PyObject *) = nullptr) { + void (*dtor)(PyObject *) + = PYBIND11_DTOR_USE_DELETE) { error_scope err_scope; // preserve any existing Python error states auto state_dict = reinterpret_borrow(get_python_state_dict()); @@ -640,7 +622,7 @@ std::pair atomic_get_or_create_in_state_dict(const char *key, // - Otherwise, our `new_capsule` is now in the dict, and it owns the storage and the state // dict will incref it. We need to set the caller's destructor on it, which will be // called when the interpreter shuts down. - if (created && dtor) { + if (created && dtor != PYBIND11_DTOR_USE_DELETE) { if (PyCapsule_SetDestructor(capsule_obj, dtor) < 0) { throw error_already_set(); } @@ -657,6 +639,8 @@ std::pair atomic_get_or_create_in_state_dict(const char *key, return std::pair(static_cast(raw_ptr), created); } +#undef PYBIND11_DTOR_USE_DELETE + template class internals_pp_manager { public: @@ -713,42 +697,75 @@ public: // this could be called without an active interpreter, just use what was cached if (!tstate || tstate->interp == last_istate_tls()) { auto tpp = internals_p_tls(); - - delete tpp; + { + std::lock_guard lock(pp_set_mutex_); + pps_have_created_content_.erase(tpp); // untrack deleted pp + } + delete tpp; // may call back into Python } unref(); return; } #endif - delete internals_singleton_pp_; + { + std::lock_guard lock(pp_set_mutex_); + pps_have_created_content_.erase(internals_singleton_pp_); // untrack deleted pp + } + delete internals_singleton_pp_; // may call back into Python unref(); } + void create_pp_content_once(std::unique_ptr *const pp) { + // Assume the GIL is held here. May call back into Python. We cannot hold the lock with our + // mutex here. So there may be multiple threads creating the content at the same time. Only + // one will install its content to pp below. Others will be freed when going out of scope. + auto tmp = std::unique_ptr(new InternalsType()); + + { + // Lock scope must not include Python calls, which may require the GIL and cause + // deadlocks. + std::lock_guard lock(pp_set_mutex_); + + if (*pp) { + // Already created in another thread. + return; + } + + // At this point, pp->get() is nullptr. + // The content is either not yet created, or was previously destroyed via pp->reset(). + + // Detect re-creation of internals after destruction during interpreter shutdown. + // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals + // have been destroyed, a new empty internals would be created, causing type lookup + // failures. See also get_or_create_pp_in_state_dict() comments. + if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { + pybind11_fail( + "pybind11::detail::internals_pp_manager::create_pp_content_once() " + "FAILED: reentrant call detected while fetching pybind11 internals!"); + } + + // Each interpreter can only create its internals once. + pps_have_created_content_.insert(pp); + // Install the created content. + pp->swap(tmp); + } + } + private: internals_pp_manager(char const *id, on_fetch_function *on_fetch) : holder_id_(id), on_fetch_(on_fetch) {} - static void internals_shutdown(PyObject *capsule) { - auto *pp = static_cast *>( - PyCapsule_GetPointer(capsule, nullptr)); - if (pp) { - pp->reset(); - } - // We reset the unique_ptr's contents but cannot delete the unique_ptr itself here. - // The pp_manager in this module (and possibly other modules sharing internals) holds - // a raw pointer to this unique_ptr, and that pointer would dangle if we deleted it now. - // - // For pybind11-owned interpreters (via embed.h or subinterpreter.h), destroy() is - // called after Py_Finalize/Py_EndInterpreter completes, which safely deletes the - // unique_ptr. For interpreters not owned by pybind11 (e.g., a pybind11 extension - // loaded into an external interpreter), destroy() is never called and the unique_ptr - // shell (8 bytes, not its contents) is leaked. - // (See PR #5958 for ideas to eliminate this leak.) - } - std::unique_ptr *get_or_create_pp_in_state_dict() { + // The `unique_ptr` is intentionally leaked on interpreter shutdown. + // Once an instance is created, it will never be deleted until the process exits (compare + // to interpreter shutdown in multiple-interpreter scenarios). + // We cannot guarantee the destruction order of capsules in the interpreter state dict on + // interpreter shutdown, so deleting internals too early could cause undefined behavior + // when other pybind11 objects access `get_internals()` during finalization (which would + // recreate empty internals). See also create_pp_content_once() above. + // See https://github.com/pybind/pybind11/pull/5958#discussion_r2717645230. auto result = atomic_get_or_create_in_state_dict>( - holder_id_, &internals_shutdown); + holder_id_, /*dtor=*/nullptr /* leak the capsule content */); auto *pp = result.first; bool created = result.second; // Only call on_fetch_ when fetching existing internals, not when creating new ones. @@ -774,7 +791,12 @@ private: on_fetch_function *on_fetch_ = nullptr; // Pointer-to-pointer to the singleton internals for the first seen interpreter (may not be the // main interpreter) - std::unique_ptr *internals_singleton_pp_; + std::unique_ptr *internals_singleton_pp_ = nullptr; + + // Track pointer-to-pointers whose internals have been created, to detect re-entrancy. + // Use instance member over static due to singleton pattern of this class. + std::unordered_set *> pps_have_created_content_; + std::mutex pp_set_mutex_; }; // If We loaded the internals through `state_dict`, our `error_already_set` @@ -815,7 +837,8 @@ PYBIND11_NOINLINE internals &get_internals() { // Slow path, something needs fetched from the state dict or created gil_scoped_acquire_simple gil; error_scope err_scope; - internals_ptr.reset(new internals()); + + ppmgr.create_pp_content_once(&internals_ptr); if (!internals_ptr->instance_base) { // This calls get_internals, so cannot be called from within the internals constructor @@ -826,6 +849,31 @@ PYBIND11_NOINLINE internals &get_internals() { return *internals_ptr; } +/// Return the PyObject* for the internals capsule (borrowed reference). +/// Returns nullptr if the capsule doesn't exist yet. +inline PyObject *get_internals_capsule() { + auto state_dict = reinterpret_borrow(get_python_state_dict()); + return dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID); +} + +/// Return the key used for local_internals in the state dict. +/// This function ensures a consistent key is used across all call sites within the same +/// compilation unit. The key includes the address of a static variable to make it unique per +/// module (DSO), matching the behavior of get_local_internals_pp_manager(). +inline const std::string &get_local_internals_key() { + static const std::string key + = PYBIND11_MODULE_LOCAL_ID + std::to_string(reinterpret_cast(&key)); + return key; +} + +/// Return the PyObject* for the local_internals capsule (borrowed reference). +/// Returns nullptr if the capsule doesn't exist yet. +inline PyObject *get_local_internals_capsule() { + const auto &key = get_local_internals_key(); + auto state_dict = reinterpret_borrow(get_python_state_dict()); + return dict_getitemstring(state_dict.ptr(), key.c_str()); +} + inline void ensure_internals() { pybind11::detail::get_internals_pp_manager().unref(); #ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT @@ -837,12 +885,10 @@ inline void ensure_internals() { } inline internals_pp_manager &get_local_internals_pp_manager() { - // Use the address of this static itself as part of the key, so that the value is uniquely tied + // Use the address of a static variable as part of the key, so that the value is uniquely tied // to where the module is loaded in memory - static const std::string this_module_idstr - = PYBIND11_MODULE_LOCAL_ID - + std::to_string(reinterpret_cast(&this_module_idstr)); - return internals_pp_manager::get_instance(this_module_idstr.c_str(), nullptr); + return internals_pp_manager::get_instance(get_local_internals_key().c_str(), + nullptr); } /// Works like `get_internals`, but for things which are locally registered. @@ -850,7 +896,10 @@ inline local_internals &get_local_internals() { auto &ppmgr = get_local_internals_pp_manager(); auto &internals_ptr = *ppmgr.get_pp(); if (!internals_ptr) { - internals_ptr.reset(new local_internals()); + gil_scoped_acquire_simple gil; + error_scope err_scope; + + ppmgr.create_pp_content_once(&internals_ptr); } return *internals_ptr; } diff --git a/tests/env.py b/tests/env.py index ee932ad77..8c0617830 100644 --- a/tests/env.py +++ b/tests/env.py @@ -29,3 +29,31 @@ TYPES_ARE_IMMORTAL = ( or GRAALPY or (CPYTHON and PY_GIL_DISABLED and (3, 13) <= sys.version_info < (3, 14)) ) + + +def check_script_success_in_subprocess(code: str, *, rerun: int = 8) -> None: + """Runs the given code in a subprocess.""" + import os + import subprocess + import sys + import textwrap + + code = textwrap.dedent(code).strip() + try: + for _ in range(rerun): # run flakily failing test multiple times + subprocess.check_output( + [sys.executable, "-c", code], + cwd=os.getcwd(), + stderr=subprocess.STDOUT, + text=True, + ) + except subprocess.CalledProcessError as ex: + raise RuntimeError( + f"Subprocess failed with exit code {ex.returncode}.\n\n" + f"Code:\n" + f"```python\n" + f"{code}\n" + f"```\n\n" + f"Output:\n" + f"{ex.output}" + ) from None diff --git a/tests/test_custom_type_setup.cpp b/tests/test_custom_type_setup.cpp index 35d30abcf..15516b21b 100644 --- a/tests/test_custom_type_setup.cpp +++ b/tests/test_custom_type_setup.cpp @@ -7,22 +7,67 @@ BSD-style license that can be found in the LICENSE file. */ +#include #include #include "pybind11_tests.h" +#include + namespace py = pybind11; namespace { +struct ContainerOwnsPythonObjects { + std::vector list; -struct OwnsPythonObjects { - py::object value = py::none(); + void append(const py::object &obj) { list.emplace_back(obj); } + py::object at(py::ssize_t index) const { + if (index >= size() || index < 0) { + throw py::index_error("Index out of range"); + } + return list.at(py::size_t(index)); + } + py::ssize_t size() const { return py::ssize_t_cast(list.size()); } + void clear() { list.clear(); } }; + +void add_gc_checkers_with_weakrefs(const py::object &obj) { + py::handle global_capsule = py::detail::get_internals_capsule(); + if (!global_capsule) { + throw std::runtime_error("No global internals capsule found"); + } + (void) py::weakref(obj, py::cpp_function([global_capsule](py::handle weakref) -> void { + py::handle current_global_capsule = py::detail::get_internals_capsule(); + if (!current_global_capsule.is(global_capsule)) { + throw std::runtime_error( + "Global internals capsule was destroyed prematurely"); + } + weakref.dec_ref(); + })) + .release(); + + py::handle local_capsule = py::detail::get_local_internals_capsule(); + if (!local_capsule) { + throw std::runtime_error("No local internals capsule found"); + } + (void) py::weakref( + obj, py::cpp_function([local_capsule](py::handle weakref) -> void { + py::handle current_local_capsule = py::detail::get_local_internals_capsule(); + if (!current_local_capsule.is(local_capsule)) { + throw std::runtime_error("Local internals capsule was destroyed prematurely"); + } + weakref.dec_ref(); + })) + .release(); +} } // namespace TEST_SUBMODULE(custom_type_setup, m) { - py::class_ cls( - m, "OwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) { + py::class_ cls( + m, + "ContainerOwnsPythonObjects", + // Please review/update docs/advanced/classes.rst after making changes here. + py::custom_type_setup([](PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) { @@ -31,19 +76,29 @@ TEST_SUBMODULE(custom_type_setup, m) { Py_VISIT(Py_TYPE(self_base)); #endif if (py::detail::is_holder_constructed(self_base)) { - auto &self = py::cast(py::handle(self_base)); - Py_VISIT(self.value.ptr()); + auto &self = py::cast(py::handle(self_base)); + for (auto &item : self.list) { + Py_VISIT(item.ptr()); + } } return 0; }; type->tp_clear = [](PyObject *self_base) { if (py::detail::is_holder_constructed(self_base)) { - auto &self = py::cast(py::handle(self_base)); - self.value = py::none(); + auto &self = py::cast(py::handle(self_base)); + for (auto &item : self.list) { + Py_CLEAR(item.ptr()); + } + self.list.clear(); } return 0; }; })); cls.def(py::init<>()); - cls.def_readwrite("value", &OwnsPythonObjects::value); + cls.def("append", &ContainerOwnsPythonObjects::append); + cls.def("at", &ContainerOwnsPythonObjects::at); + cls.def("size", &ContainerOwnsPythonObjects::size); + cls.def("clear", &ContainerOwnsPythonObjects::clear); + + m.def("add_gc_checkers_with_weakrefs", &add_gc_checkers_with_weakrefs); } diff --git a/tests/test_custom_type_setup.py b/tests/test_custom_type_setup.py index bb2865cad..4c6b9510a 100644 --- a/tests/test_custom_type_setup.py +++ b/tests/test_custom_type_setup.py @@ -1,11 +1,14 @@ from __future__ import annotations import gc +import os +import sys import weakref import pytest -import env # noqa: F401 +import env +import pybind11_tests from pybind11_tests import custom_type_setup as m @@ -36,15 +39,46 @@ def gc_tester(): # PyPy does not seem to reliably garbage collect. @pytest.mark.skipif("env.PYPY or env.GRAALPY") def test_self_cycle(gc_tester): - obj = m.OwnsPythonObjects() - obj.value = obj + obj = m.ContainerOwnsPythonObjects() + obj.append(obj) gc_tester(obj) # PyPy does not seem to reliably garbage collect. @pytest.mark.skipif("env.PYPY or env.GRAALPY") def test_indirect_cycle(gc_tester): - obj = m.OwnsPythonObjects() - obj_list = [obj] - obj.value = obj_list + obj = m.ContainerOwnsPythonObjects() + obj.append([obj]) gc_tester(obj) + + +@pytest.mark.skipif( + env.IOS or sys.platform.startswith("emscripten"), + reason="Requires subprocess support", +) +@pytest.mark.skipif("env.PYPY or env.GRAALPY") +def test_py_cast_useable_on_shutdown(): + """Test that py::cast works during interpreter shutdown. + + See PR #5972 and https://github.com/pybind/pybind11/pull/5958#discussion_r2717645230. + """ + env.check_script_success_in_subprocess( + f""" + import sys + + sys.path.insert(0, {os.path.dirname(env.__file__)!r}) + sys.path.insert(0, {os.path.dirname(pybind11_tests.__file__)!r}) + + from pybind11_tests import custom_type_setup as m + + # Create a self-referential cycle that will be collected during shutdown. + # The tp_traverse and tp_clear callbacks call py::cast, which requires + # internals to still be valid. + obj = m.ContainerOwnsPythonObjects() + obj.append(obj) + + # Add weakref callbacks that verify the capsule is still alive when the + # pybind11 object is garbage collected during shutdown. + m.add_gc_checkers_with_weakrefs(obj) + """ + ) diff --git a/tests/test_multiple_interpreters.py b/tests/test_multiple_interpreters.py index 44877e772..56d303a36 100644 --- a/tests/test_multiple_interpreters.py +++ b/tests/test_multiple_interpreters.py @@ -3,7 +3,6 @@ from __future__ import annotations import contextlib import os import pickle -import subprocess import sys import textwrap @@ -219,6 +218,7 @@ PREAMBLE_CODE = textwrap.dedent( def test(): import sys + sys.path.insert(0, {os.path.dirname(env.__file__)!r}) sys.path.insert(0, {os.path.dirname(pybind11_tests.__file__)!r}) import collections @@ -269,36 +269,13 @@ def test_import_module_with_singleton_per_interpreter(): interp.exec(code) -def check_script_success_in_subprocess(code: str, *, rerun: int = 8) -> None: - """Runs the given code in a subprocess.""" - code = textwrap.dedent(code).strip() - try: - for _ in range(rerun): # run flakily failing test multiple times - subprocess.check_output( - [sys.executable, "-c", code], - cwd=os.getcwd(), - stderr=subprocess.STDOUT, - text=True, - ) - except subprocess.CalledProcessError as ex: - raise RuntimeError( - f"Subprocess failed with exit code {ex.returncode}.\n\n" - f"Code:\n" - f"```python\n" - f"{code}\n" - f"```\n\n" - f"Output:\n" - f"{ex.output}" - ) from None - - @pytest.mark.skipif( sys.platform.startswith("emscripten"), reason="Requires loadable modules" ) @pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+") def test_import_in_subinterpreter_after_main(): """Tests that importing a module in a subinterpreter after the main interpreter works correctly""" - check_script_success_in_subprocess( + env.check_script_success_in_subprocess( PREAMBLE_CODE + textwrap.dedent( """ @@ -319,7 +296,7 @@ def test_import_in_subinterpreter_after_main(): ) ) - check_script_success_in_subprocess( + env.check_script_success_in_subprocess( PREAMBLE_CODE + textwrap.dedent( """ @@ -354,7 +331,7 @@ def test_import_in_subinterpreter_after_main(): @pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+") def test_import_in_subinterpreter_before_main(): """Tests that importing a module in a subinterpreter before the main interpreter works correctly""" - check_script_success_in_subprocess( + env.check_script_success_in_subprocess( PREAMBLE_CODE + textwrap.dedent( """ @@ -375,7 +352,7 @@ def test_import_in_subinterpreter_before_main(): ) ) - check_script_success_in_subprocess( + env.check_script_success_in_subprocess( PREAMBLE_CODE + textwrap.dedent( """ @@ -401,7 +378,7 @@ def test_import_in_subinterpreter_before_main(): ) ) - check_script_success_in_subprocess( + env.check_script_success_in_subprocess( PREAMBLE_CODE + textwrap.dedent( """ @@ -434,7 +411,7 @@ def test_import_in_subinterpreter_before_main(): @pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+") def test_import_in_subinterpreter_concurrently(): """Tests that importing a module in multiple subinterpreters concurrently works correctly""" - check_script_success_in_subprocess( + env.check_script_success_in_subprocess( PREAMBLE_CODE + textwrap.dedent( """ From 4d7d02a8e528f3d3ba14d9b10b813557b2e9d0e2 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 2 Feb 2026 01:02:50 -0500 Subject: [PATCH 5/8] Fix race condition with py::make_key_iterator in free threading (#5971) * Fix race condition with py::make_key_iterator in free threading The creation of the iterator class needs to be synchronized. * style: pre-commit fixes * Use PyCriticalSection_BeginMutex instead of recursive mutex * style: pre-commit fixes * Make pycritical_section non-copyable and non-movable The pycritical_section class is a RAII wrapper that manages a Python critical section lifecycle: - Acquires the critical section in the constructor via PyCriticalSection_BeginMutex - Releases it in the destructor via PyCriticalSection_End - Holds a reference to a pymutex Allowing copy or move operations would be dangerous: 1. Copy: Both the original and copied objects would call PyCriticalSection_End on the same PyCriticalSection object in their destructors, leading to double-unlock and undefined behavior. 2. Move: The moved-from object's destructor would still run and attempt to end the critical section, while the moved-to object would also try to end it, again causing double-unlock. This follows the same pattern used by other RAII lock guards in the codebase, such as gil_scoped_acquire and gil_scoped_release, which also explicitly delete copy/move operations to prevent similar issues. By explicitly deleting these operations, we prevent accidental misuse and ensure the critical section is properly managed by a single RAII object throughout its lifetime. * Drop Python 3.13t support from CI Python 3.13t was experimental, while Python 3.14t is not. This PR uses PyCriticalSection_BeginMutex which is only available in Python 3.14+, making Python 3.13t incompatible with the changes. Removed all Python 3.13t CI jobs: - ubuntu-latest, 3.13t (standard-large matrix) - macos-15-intel, 3.13t (standard-large matrix) - windows-latest, 3.13t (standard-large matrix) - manylinux job testing 3.13t This aligns with the decision to drop Python 3.13t support as discussed in PR #5971. * Add Python 3.13 (default) replacement jobs for removed 3.13t jobs After removing Python 3.13t support (incompatible with PyCriticalSection_BeginMutex which requires Python 3.14+), we're adding replacement jobs using Python 3.13 (default) to maintain test coverage in key dimensions: 1. ubuntu-latest, Python 3.13: C++20 + DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION - Replaces: ubuntu-latest, 3.13t with same config - Maintains coverage for this specific configuration combination 2. macos-15-intel, Python 3.13: C++11 - Replaces: macos-15-intel, 3.13t with same config - Maintains macOS coverage for Python 3.13 3. manylinux (musllinux), Python 3.13: GIL testing - Replaces: manylinux, 3.13t job - Maintains manylinux/musllinux container testing coverage These additions are proposed to get feedback on which jobs should be kept to maintain appropriate test coverage without the experimental 3.13t builds. * ci: run in free-threading mode a bit more on 3.14 * Revert "ci: run in free-threading mode a bit more on 3.14" This reverts commit 91189c9242e787922d26ca467710dcc494871b82. Reason: https://github.com/pybind/pybind11/pull/5971#issuecomment-3831321903 * Reapply "ci: run in free-threading mode a bit more on 3.14" This reverts commit f3197de97557a86a9a73df7890be1c227a7c4c59. After #5972 is/was merged, tests should pass (already tested under #5980). See also https://github.com/pybind/pybind11/pull/5972#discussion_r2752674989 --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve Co-authored-by: Henry Schreiner Co-authored-by: Ralf W. Grosse-Kunstleve --- .github/workflows/ci.yml | 15 ++++++--------- include/pybind11/detail/internals.h | 22 ++++++++++++++++++++-- include/pybind11/pybind11.h | 1 + 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cda2c214d..a0965fa81 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: python-version: '3.12' cmake-args: -DPYBIND11_TEST_SMART_HOLDER=ON -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON - runs-on: ubuntu-latest - python-version: '3.13t' + python-version: '3.14t' cmake-args: -DCMAKE_CXX_STANDARD=20 -DPYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION=ON - runs-on: ubuntu-latest python-version: '3.14' @@ -102,12 +102,12 @@ jobs: - runs-on: macos-15-intel python-version: '3.11' cmake-args: -DPYBIND11_TEST_SMART_HOLDER=ON + - runs-on: macos-15-intel + python-version: '3.13' + cmake-args: -DCMAKE_CXX_STANDARD=11 - runs-on: macos-latest python-version: '3.12' cmake-args: -DCMAKE_CXX_STANDARD=17 -DPYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION=ON - - runs-on: macos-15-intel - python-version: '3.13t' - cmake-args: -DCMAKE_CXX_STANDARD=11 - runs-on: macos-latest python-version: '3.14t' cmake-args: -DCMAKE_CXX_STANDARD=20 @@ -138,9 +138,6 @@ jobs: - runs-on: windows-2022 python-version: '3.13' cmake-args: -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebugDLL - - runs-on: windows-latest - python-version: '3.13t' - cmake-args: -DCMAKE_CXX_STANDARD=17 - runs-on: windows-latest python-version: '3.14' cmake-args: -DCMAKE_CXX_STANDARD=20 @@ -240,7 +237,7 @@ jobs: manylinux: - name: Manylinux on 🐍 3.13t • GIL + name: Manylinux on 🐍 3.14t if: github.event.pull_request.draft == false runs-on: ubuntu-latest timeout-minutes: 40 @@ -257,7 +254,7 @@ jobs: run: uv tool install ninja - name: Configure via preset - run: cmake --preset venv -DPYBIND11_CREATE_WITH_UV=python3.13t + run: cmake --preset venv -DPYBIND11_CREATE_WITH_UV=python3.14t - name: Build C++11 run: cmake --build --preset venv diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 7cfa5da92..b9b0f08f2 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -230,6 +230,7 @@ using instance_map = std::unordered_multimap; #ifdef Py_GIL_DISABLED // Wrapper around PyMutex to provide BasicLockable semantics class pymutex { + friend class pycritical_section; PyMutex mutex; public: @@ -238,6 +239,23 @@ public: void unlock() { PyMutex_Unlock(&mutex); } }; +class pycritical_section { + pymutex &mutex; + PyCriticalSection cs; + +public: + explicit pycritical_section(pymutex &m) : mutex(m) { + PyCriticalSection_BeginMutex(&cs, &mutex.mutex); + } + ~pycritical_section() { PyCriticalSection_End(&cs); } + + // Non-copyable and non-movable to prevent double-unlock + pycritical_section(const pycritical_section &) = delete; + pycritical_section &operator=(const pycritical_section &) = delete; + pycritical_section(pycritical_section &&) = delete; + pycritical_section &operator=(pycritical_section &&) = delete; +}; + // Instance map shards are used to reduce mutex contention in free-threaded Python. struct instance_map_shard { instance_map registered_instances; @@ -905,7 +923,7 @@ inline local_internals &get_local_internals() { } #ifdef Py_GIL_DISABLED -# define PYBIND11_LOCK_INTERNALS(internals) std::unique_lock lock((internals).mutex) +# define PYBIND11_LOCK_INTERNALS(internals) pycritical_section lock((internals).mutex) #else # define PYBIND11_LOCK_INTERNALS(internals) #endif @@ -934,7 +952,7 @@ inline auto with_exception_translators(const F &cb) get_local_internals().registered_exception_translators)) { auto &internals = get_internals(); #ifdef Py_GIL_DISABLED - std::unique_lock lock((internals).exception_translator_mutex); + pycritical_section lock((internals).exception_translator_mutex); #endif auto &local_internals = get_local_internals(); return cb(internals.registered_exception_translators, diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 02d2e72c2..f88fc2027 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -3173,6 +3173,7 @@ iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) { using state = detail::iterator_state; // TODO: state captures only the types of Extra, not the values + PYBIND11_LOCK_INTERNALS(get_internals()); if (!detail::get_type_info(typeid(state), false)) { class_(handle(), "iterator", pybind11::module_local()) .def( From 8f68ecd32c8e18d3b064dbf0ea5fc31a6cb37e9a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Feb 2026 17:16:53 -0800 Subject: [PATCH 6/8] chore(deps): update pre-commit hooks (#5982) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.14.10 → v0.14.14](https://github.com/astral-sh/ruff-pre-commit/compare/v0.14.10...v0.14.14) - [github.com/Lucas-C/pre-commit-hooks: v1.5.5 → v1.5.6](https://github.com/Lucas-C/pre-commit-hooks/compare/v1.5.5...v1.5.6) - [github.com/adhtruong/mirrors-typos: v1.41.0 → v1.42.3](https://github.com/adhtruong/mirrors-typos/compare/v1.41.0...v1.42.3) - [github.com/python-jsonschema/check-jsonschema: 0.36.0 → 0.36.1](https://github.com/python-jsonschema/check-jsonschema/compare/0.36.0...0.36.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1271c2afe..05626151c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: # Ruff, the Python auto-correcting linter/formatter written in Rust - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.10 + rev: v0.14.14 hooks: - id: ruff-check args: ["--fix", "--show-fixes"] @@ -88,7 +88,7 @@ repos: # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: "v1.5.5" + rev: "v1.5.6" hooks: - id: remove-tabs exclude: (^docs/.*|\.patch)?$ @@ -122,7 +122,7 @@ repos: # Use mirror because pre-commit autoupdate confuses tags in the upstream repo. # See https://github.com/crate-ci/typos/issues/390 - repo: https://github.com/adhtruong/mirrors-typos - rev: "v1.41.0" + rev: "v1.42.3" hooks: - id: typos args: [] @@ -151,7 +151,7 @@ repos: # Check schemas on some of our YAML files - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.36.0 + rev: 0.36.1 hooks: - id: check-readthedocs - id: check-github-workflows From 5f2c678916ce2890e51333e0b684eb443e8848ed Mon Sep 17 00:00:00 2001 From: Daniel Simon Date: Sun, 8 Feb 2026 15:04:46 -0800 Subject: [PATCH 7/8] Add helpers to array that return the size and strides as a std::span (#5974) * Add helper functions to pybind11::array to return the shape and strides as a std::span. These functions are hidden with macros unless PYBIND11_CPP20 is defined and the include has been found. * style: pre-commit fixes * tests: Add unit tests for shape_span() and strides_span() Add comprehensive unit tests for the new std::span helper functions: - Test 0D, 1D, 2D, and 3D arrays - Verify spans match regular shape()/strides() methods - Test that spans can be used to construct new arrays - Tests are conditionally compiled only when PYBIND11_HAS_SPAN is defined * Use __cpp_lib_span feature test macro instead of __has_include Replace __has_include() check with __cpp_lib_span feature test macro to resolve ambiguity where some pre-C++20 systems might have a global header called that isn't the C++20 std::span. The check is moved after is included, consistent with how __cpp_lib_char8_t is handled. Co-authored-by: Cursor * Fix: Use py::ssize_t instead of ssize_t in span tests On Windows/MSVC, ssize_t is not available in the standard namespace without proper includes. Use py::ssize_t (the pybind11 typedef) instead to ensure cross-platform compatibility. Fixes compilation errors on: - Windows/MSVC 2022 (C++20) - GCC 10 (C++20) Co-authored-by: Cursor --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve Co-authored-by: Cursor --- include/pybind11/detail/common.h | 4 ++++ include/pybind11/numpy.h | 18 +++++++++++++++ tests/test_numpy_array.cpp | 17 ++++++++++++++ tests/test_numpy_array.py | 39 ++++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 19ebc8532..017d134da 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -252,6 +252,10 @@ # define PYBIND11_HAS_U8STRING 1 #endif +#if defined(PYBIND11_CPP20) && defined(__cpp_lib_span) && __cpp_lib_span >= 202002L +# define PYBIND11_HAS_SPAN 1 +#endif + // See description of PR #4246: #if !defined(PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF) && !defined(NDEBUG) \ && !defined(PYPY_VERSION) && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 6fa6c772b..408d1699c 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -29,6 +29,10 @@ #include #include +#ifdef PYBIND11_HAS_SPAN +# include +#endif + #if defined(PYBIND11_NUMPY_1_ONLY) # error "PYBIND11_NUMPY_1_ONLY is no longer supported (see PR #5595)." #endif @@ -1143,6 +1147,13 @@ public: /// Dimensions of the array const ssize_t *shape() const { return detail::array_proxy(m_ptr)->dimensions; } +#ifdef PYBIND11_HAS_SPAN + /// Dimensions of the array as a span + std::span shape_span() const { + return std::span(shape(), static_cast(ndim())); + } +#endif + /// Dimension along a given axis ssize_t shape(ssize_t dim) const { if (dim >= ndim()) { @@ -1154,6 +1165,13 @@ public: /// Strides of the array const ssize_t *strides() const { return detail::array_proxy(m_ptr)->strides; } +#ifdef PYBIND11_HAS_SPAN + /// Strides of the array as a span + std::span strides_span() const { + return std::span(strides(), static_cast(ndim())); + } +#endif + /// Stride along a given axis ssize_t strides(ssize_t dim) const { if (dim >= ndim()) { diff --git a/tests/test_numpy_array.cpp b/tests/test_numpy_array.cpp index ac6b1cfe3..3bce98835 100644 --- a/tests/test_numpy_array.cpp +++ b/tests/test_numpy_array.cpp @@ -14,6 +14,7 @@ #include #include +#include // Size / dtype checks. struct DtypeCheck { @@ -246,6 +247,22 @@ TEST_SUBMODULE(numpy_array, sm) { sm.def("nbytes", [](const arr &a) { return a.nbytes(); }); sm.def("owndata", [](const arr &a) { return a.owndata(); }); +#ifdef PYBIND11_HAS_SPAN + // test_shape_strides_span + sm.def("shape_span", [](const arr &a) { + auto span = a.shape_span(); + return std::vector(span.begin(), span.end()); + }); + sm.def("strides_span", [](const arr &a) { + auto span = a.strides_span(); + return std::vector(span.begin(), span.end()); + }); + // Test that spans can be used to construct new arrays + sm.def("array_from_spans", [](const arr &a) { + return py::array(a.dtype(), a.shape_span(), a.strides_span(), a.data(), a); + }); +#endif + // test_index_offset def_index_fn(index_at, const arr &); def_index_fn(index_at_t, const arr_t &); diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index 19c07ca11..93477aa23 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -68,6 +68,45 @@ def test_array_attributes(): assert not m.owndata(a) +@pytest.mark.skipif(not hasattr(m, "shape_span"), reason="std::span not available") +def test_shape_strides_span(): + # Test 0-dimensional array (scalar) + a = np.array(42, "f8") + assert m.ndim(a) == 0 + assert m.shape_span(a) == [] + assert m.strides_span(a) == [] + + # Test 1-dimensional array + a = np.array([1, 2, 3, 4], "u2") + assert m.ndim(a) == 1 + assert m.shape_span(a) == [4] + assert m.strides_span(a) == [2] + + # Test 2-dimensional array + a = np.array([[1, 2, 3], [4, 5, 6]], "u2").view() + a.flags.writeable = False + assert m.ndim(a) == 2 + assert m.shape_span(a) == [2, 3] + assert m.strides_span(a) == [6, 2] + + # Test 3-dimensional array + a = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], "i4") + assert m.ndim(a) == 3 + assert m.shape_span(a) == [2, 2, 2] + # Verify spans match regular shape/strides + assert list(m.shape_span(a)) == list(m.shape(a)) + assert list(m.strides_span(a)) == list(m.strides(a)) + + # Test that spans can be used to construct new arrays + original = np.array([[1, 2, 3], [4, 5, 6]], "f4") + new_array = m.array_from_spans(original) + assert new_array.shape == original.shape + assert new_array.strides == original.strides + assert new_array.dtype == original.dtype + # Verify data is shared (since we pass the same data pointer) + np.testing.assert_array_equal(new_array, original) + + @pytest.mark.parametrize( ("args", "ret"), [([], 0), ([0], 0), ([1], 3), ([0, 1], 1), ([1, 2], 5)] ) From 3ae5a173c54ab64339ad4a6c57557896b8bad6f9 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Tue, 10 Feb 2026 13:14:07 +0800 Subject: [PATCH 8/8] Add fallback implementation of `PyCriticalSection_BeginMutex` for Python 3.13t (#5981) * Add failback implementation of `PyCriticalSection_BeginMutex` for Python 3.13t * Add comment for Python version * Use `_PyCriticalSection_BeginSlow` * Add forward declaration * Fix forward declaration * Remove always true condition `defined(PY_VERSION_HEX)` * Detect musllinux * Add manylinux test * Use direct mutex locking for Python 3.13t `_PyCriticalSection_BeginSlow` is a private CPython function not exported on Linux. For Python < 3.14.0rc1, use direct `mutex.lock()`/`mutex.unlock()` instead of critical section APIs. * Empty commit to trigger CI * Empty commit to trigger CI * Empty commit to trigger CI * Run apt update before apt install * Remove unnecessary prefix * Add manylinux test with Python 3.13t * Simplify pycritical_section with std::unique_lock fallback for Python < 3.14 * Fix potential deadlock in make_iterator_impl for Python 3.13t Refactor pycritical_section into a unified class with internal version checks instead of using a type alias fallback. Skip locking in make_iterator_impl for Python < 3.14.0rc1 to avoid deadlock during type registration, as pycritical_section cannot release the mutex during Python callbacks without PyCriticalSection_BeginMutex. * Add reference for xfail message --- .github/workflows/ci.yml | 18 +++++++++++++++--- .github/workflows/reusable-standard.yml | 2 +- include/pybind11/detail/internals.h | 16 +++++++++++++++- include/pybind11/pybind11.h | 4 ++++ tests/env.py | 12 ++++++++++++ tests/test_multiple_interpreters.py | 5 +++++ 6 files changed, 52 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0965fa81..03cb0238c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -237,11 +237,23 @@ jobs: manylinux: - name: Manylinux on 🐍 3.14t + name: Manylinux on 🐍 ${{ matrix.python-version }} (${{ matrix.container }}) if: github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + include: + - container: quay.io/pypa/manylinux_2_28_x86_64:latest + python-version: '3.13t' + - container: quay.io/pypa/musllinux_1_2_x86_64:latest + python-version: '3.13t' + - container: quay.io/pypa/manylinux_2_28_x86_64:latest + python-version: '3.14t' + - container: quay.io/pypa/musllinux_1_2_x86_64:latest + python-version: '3.14t' runs-on: ubuntu-latest timeout-minutes: 40 - container: quay.io/pypa/musllinux_1_2_x86_64:latest + container: ${{ matrix.container }} steps: - uses: actions/checkout@v6 with: @@ -254,7 +266,7 @@ jobs: run: uv tool install ninja - name: Configure via preset - run: cmake --preset venv -DPYBIND11_CREATE_WITH_UV=python3.14t + run: cmake --preset venv -DPYBIND11_CREATE_WITH_UV="${{ matrix.python-version }}" - name: Build C++11 run: cmake --build --preset venv diff --git a/.github/workflows/reusable-standard.yml b/.github/workflows/reusable-standard.yml index 56d92e277..6e22d0f38 100644 --- a/.github/workflows/reusable-standard.yml +++ b/.github/workflows/reusable-standard.yml @@ -44,7 +44,7 @@ jobs: - name: Setup Boost (Linux) if: runner.os == 'Linux' - run: sudo apt-get install libboost-dev + run: sudo apt-get update && sudo apt-get install -y libboost-dev - name: Setup Boost (macOS) if: runner.os == 'macOS' diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index b9b0f08f2..b68152932 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -241,13 +241,27 @@ public: class pycritical_section { pymutex &mutex; +# if PY_VERSION_HEX >= 0x030E00C1 // 3.14.0rc1 PyCriticalSection cs; +# endif public: explicit pycritical_section(pymutex &m) : mutex(m) { + // PyCriticalSection_BeginMutex was added in Python 3.15.0a1 and backported to 3.14.0rc1 +# if PY_VERSION_HEX >= 0x030E00C1 // 3.14.0rc1 PyCriticalSection_BeginMutex(&cs, &mutex.mutex); +# else + // Fall back to direct mutex locking for older free-threaded Python versions + mutex.lock(); +# endif + } + ~pycritical_section() { +# if PY_VERSION_HEX >= 0x030E00C1 // 3.14.0rc1 + PyCriticalSection_End(&cs); +# else + mutex.unlock(); +# endif } - ~pycritical_section() { PyCriticalSection_End(&cs); } // Non-copyable and non-movable to prevent double-unlock pycritical_section(const pycritical_section &) = delete; diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index f88fc2027..0f31262c4 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -3173,7 +3173,11 @@ iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) { using state = detail::iterator_state; // TODO: state captures only the types of Extra, not the values + // For Python < 3.14.0rc1, pycritical_section uses direct mutex locking (same as a unique + // lock), which may deadlock during type registration. See detail/internals.h for details. +#if PY_VERSION_HEX >= 0x030E00C1 // 3.14.0rc1 PYBIND11_LOCK_INTERNALS(get_internals()); +#endif if (!detail::get_type_info(typeid(state), false)) { class_(handle(), "iterator", pybind11::module_local()) .def( diff --git a/tests/env.py b/tests/env.py index 8c0617830..790a0108f 100644 --- a/tests/env.py +++ b/tests/env.py @@ -11,6 +11,18 @@ MACOS = sys.platform.startswith("darwin") WIN = sys.platform.startswith("win32") or sys.platform.startswith("cygwin") FREEBSD = sys.platform.startswith("freebsd") +MUSLLINUX = False +MANYLINUX = False +if LINUX: + + def _is_musl() -> bool: + libc, _ = platform.libc_ver() + return libc == "musl" or (libc != "glibc" and libc != "") + + MUSLLINUX = _is_musl() + MANYLINUX = not MUSLLINUX + del _is_musl + CPYTHON = platform.python_implementation() == "CPython" PYPY = platform.python_implementation() == "PyPy" GRAALPY = sys.implementation.name == "graalpy" diff --git a/tests/test_multiple_interpreters.py b/tests/test_multiple_interpreters.py index 56d303a36..65b273319 100644 --- a/tests/test_multiple_interpreters.py +++ b/tests/test_multiple_interpreters.py @@ -408,6 +408,11 @@ def test_import_in_subinterpreter_before_main(): @pytest.mark.skipif( sys.platform.startswith("emscripten"), reason="Requires loadable modules" ) +@pytest.mark.xfail( + env.MUSLLINUX, + reason="Flaky on musllinux, see also: https://github.com/pybind/pybind11/pull/5972#discussion_r2755283335", + strict=False, +) @pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+") def test_import_in_subinterpreter_concurrently(): """Tests that importing a module in multiple subinterpreters concurrently works correctly"""