Merge branch 'master' into henryiii-patch-3

2026-03-14 20:27:47 +00:00 · 2026-02-10 00:28:02 -05:00
parent b1ce230e3c 3ae5a173c5
commit d7cc5b052d
18 changed files with 453 additions and 144 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -84,7 +84,7 @@ jobs:
            python-version: '3.12'
            cmake-args: -DPYBIND11_TEST_SMART_HOLDER=ON -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON
          - runs-on: ubuntu-latest
-            python-version: '3.13t'
+            python-version: '3.14t'
            cmake-args: -DCMAKE_CXX_STANDARD=20 -DPYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION=ON
          - runs-on: ubuntu-latest
            python-version: '3.14'
@@ -102,12 +102,12 @@ jobs:
          - runs-on: macos-15-intel
            python-version: '3.11'
            cmake-args: -DPYBIND11_TEST_SMART_HOLDER=ON
+          - runs-on: macos-15-intel
+            python-version: '3.13'
+            cmake-args: -DCMAKE_CXX_STANDARD=11
          - runs-on: macos-latest
            python-version: '3.12'
            cmake-args: -DCMAKE_CXX_STANDARD=17 -DPYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION=ON
-          - runs-on: macos-15-intel
-            python-version: '3.13t'
-            cmake-args: -DCMAKE_CXX_STANDARD=11
          - runs-on: macos-latest
            python-version: '3.14t'
            cmake-args: -DCMAKE_CXX_STANDARD=20
@@ -138,9 +138,6 @@ jobs:
          - runs-on: windows-2022
            python-version: '3.13'
            cmake-args: -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebugDLL
-          - runs-on: windows-latest
-            python-version: '3.13t'
-            cmake-args: -DCMAKE_CXX_STANDARD=17
          - runs-on: windows-latest
            python-version: '3.14'
            cmake-args: -DCMAKE_CXX_STANDARD=20
@@ -240,11 +237,23 @@ jobs:


  manylinux:
-    name: Manylinux on 🐍 3.13t • GIL
+    name: Manylinux on 🐍 ${{ matrix.python-version }} (${{ matrix.container }})
    if: github.event.pull_request.draft == false
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - container: quay.io/pypa/manylinux_2_28_x86_64:latest
+            python-version: '3.13t'
+          - container: quay.io/pypa/musllinux_1_2_x86_64:latest
+            python-version: '3.13t'
+          - container: quay.io/pypa/manylinux_2_28_x86_64:latest
+            python-version: '3.14t'
+          - container: quay.io/pypa/musllinux_1_2_x86_64:latest
+            python-version: '3.14t'
    runs-on: ubuntu-latest
    timeout-minutes: 40
-    container: quay.io/pypa/musllinux_1_2_x86_64:latest
+    container: ${{ matrix.container }}
    steps:
      - uses: actions/checkout@v6
        with:
@@ -257,7 +266,7 @@ jobs:
        run: uv tool install ninja

      - name: Configure via preset
-        run: cmake --preset venv -DPYBIND11_CREATE_WITH_UV=python3.13t
+        run: cmake --preset venv -DPYBIND11_CREATE_WITH_UV="${{ matrix.python-version }}"

      - name: Build C++11
        run: cmake --build --preset venv
@@ -293,7 +302,7 @@ jobs:
        debug: ${{ matrix.python-debug }}

    - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v2.0
+      uses: jwlawson/actions-setup-cmake@v2.1

    - name: Valgrind cache
      if: matrix.valgrind
@@ -561,7 +570,7 @@ jobs:
      run: python3 -m pip install --upgrade pip

    - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v2.0
+      uses: jwlawson/actions-setup-cmake@v2.1

    - name: Configure
      shell: bash
@@ -897,7 +906,7 @@ jobs:
          ${{ matrix.python == '3.13' && runner.os == 'Windows' }}

    - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v2.0
+      uses: jwlawson/actions-setup-cmake@v2.1

    - name: Prepare MSVC
      uses: ilammy/msvc-dev-cmd@v1.13.0
@@ -947,7 +956,7 @@ jobs:
        architecture: x86

    - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v2.0
+      uses: jwlawson/actions-setup-cmake@v2.1

    - name: Prepare MSVC
      uses: ilammy/msvc-dev-cmd@v1.13.0
@@ -998,7 +1007,7 @@ jobs:
      run: python3 -m pip install -r tests/requirements.txt

    - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v2.0
+      uses: jwlawson/actions-setup-cmake@v2.1

    - name: Configure C++20
      run: >
@@ -1180,7 +1189,7 @@ jobs:
          python-version: ${{ matrix.python }}

      - name: Update CMake
-        uses: jwlawson/actions-setup-cmake@v2.0
+        uses: jwlawson/actions-setup-cmake@v2.1

      - name: Install ninja-build tool
        uses: seanmiddleditch/gha-setup-ninja@v6
--- a/.github/workflows/configure.yml
+++ b/.github/workflows/configure.yml
@@ -64,7 +64,7 @@ jobs:
    # An action for adding a specific version of CMake:
    #   https://github.com/jwlawson/actions-setup-cmake
    - name: Setup CMake ${{ matrix.cmake }}
-      uses: jwlawson/actions-setup-cmake@v2.0
+      uses: jwlawson/actions-setup-cmake@v2.1
      with:
        cmake-version: ${{ matrix.cmake }}

--- a/.github/workflows/reusable-standard.yml
+++ b/.github/workflows/reusable-standard.yml
@@ -44,7 +44,7 @@ jobs:

      - name: Setup Boost (Linux)
        if: runner.os == 'Linux'
-        run: sudo apt-get install libboost-dev
+        run: sudo apt-get update && sudo apt-get install -y libboost-dev

      - name: Setup Boost (macOS)
        if: runner.os == 'macOS'
--- a/.github/workflows/upstream.yml
+++ b/.github/workflows/upstream.yml
@@ -36,7 +36,7 @@ jobs:
      run: sudo apt-get install libboost-dev

    - name: Update CMake
-      uses: jwlawson/actions-setup-cmake@v2.0
+      uses: jwlawson/actions-setup-cmake@v2.1

    - name: Run pip installs
      run: |
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -32,7 +32,7 @@ repos:

 # Ruff, the Python auto-correcting linter/formatter written in Rust
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.14.10
+  rev: v0.14.14
  hooks:
  - id: ruff-check
    args: ["--fix", "--show-fixes"]
@@ -88,7 +88,7 @@ repos:

 # Changes tabs to spaces
 - repo: https://github.com/Lucas-C/pre-commit-hooks
-  rev: "v1.5.5"
+  rev: "v1.5.6"
  hooks:
  - id: remove-tabs
    exclude: (^docs/.*|\.patch)?$
@@ -122,7 +122,7 @@ repos:
 # Use mirror because pre-commit autoupdate confuses tags in the upstream repo.
 # See https://github.com/crate-ci/typos/issues/390
 - repo: https://github.com/adhtruong/mirrors-typos
-  rev: "v1.41.0"
+  rev: "v1.42.3"
  hooks:
  - id: typos
    args: []
@@ -151,7 +151,7 @@ repos:

 # Check schemas on some of our YAML files
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.36.0
+  rev: 0.36.1
  hooks:
  - id: check-readthedocs
  - id: check-github-workflows
--- a/docs/advanced/classes.rst
+++ b/docs/advanced/classes.rst
@@ -1381,11 +1381,22 @@ You can do that using ``py::custom_type_setup``:

 .. code-block:: cpp

-   struct OwnsPythonObjects {
-       py::object value = py::none();
+   struct ContainerOwnsPythonObjects {
+       std::vector<py::object> list;
+
+       void append(const py::object &obj) { list.emplace_back(obj); }
+       py::object at(py::ssize_t index) const {
+           if (index >= size() || index < 0) {
+               throw py::index_error("Index out of range");
+           }
+           return list.at(py::size_t(index));
+       }
+       py::ssize_t size() const { return py::ssize_t_cast(list.size()); }
+       void clear() { list.clear(); }
   };
-   py::class_<OwnsPythonObjects> cls(
-       m, "OwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) {
+
+   py::class_<ContainerOwnsPythonObjects> cls(
+       m, "ContainerOwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) {
           auto *type = &heap_type->ht_type;
           type->tp_flags |= Py_TPFLAGS_HAVE_GC;
           type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) {
@@ -1394,20 +1405,28 @@ You can do that using ``py::custom_type_setup``:
               Py_VISIT(Py_TYPE(self_base));
   #endif
               if (py::detail::is_holder_constructed(self_base)) {
-                   auto &self = py::cast<OwnsPythonObjects&>(py::handle(self_base));
-                   Py_VISIT(self.value.ptr());
+                   auto &self = py::cast<ContainerOwnsPythonObjects &>(py::handle(self_base));
+                   for (auto &item : self.list) {
+                       Py_VISIT(item.ptr());
+                   }
               }
               return 0;
           };
           type->tp_clear = [](PyObject *self_base) {
               if (py::detail::is_holder_constructed(self_base)) {
-                   auto &self = py::cast<OwnsPythonObjects&>(py::handle(self_base));
-                   self.value = py::none();
+                   auto &self = py::cast<ContainerOwnsPythonObjects &>(py::handle(self_base));
+                   for (auto &item : self.list) {
+                       Py_CLEAR(item.ptr());
+                   }
+                   self.list.clear();
               }
               return 0;
           };
       }));
   cls.def(py::init<>());
-   cls.def_readwrite("value", &OwnsPythonObjects::value);
+   cls.def("append", &ContainerOwnsPythonObjects::append);
+   cls.def("at", &ContainerOwnsPythonObjects::at);
+   cls.def("size", &ContainerOwnsPythonObjects::size);
+   cls.def("clear", &ContainerOwnsPythonObjects::clear);

 .. versionadded:: 2.8
--- a/include/pybind11/detail/common.h
+++ b/include/pybind11/detail/common.h
@@ -252,6 +252,10 @@
 #    define PYBIND11_HAS_U8STRING 1
 #endif

+#if defined(PYBIND11_CPP20) && defined(__cpp_lib_span) && __cpp_lib_span >= 202002L
+#    define PYBIND11_HAS_SPAN 1
+#endif
+
 // See description of PR #4246:
 #if !defined(PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF) && !defined(NDEBUG)                       \
    && !defined(PYPY_VERSION) && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF)
--- a/include/pybind11/detail/internals.h
+++ b/include/pybind11/detail/internals.h
@@ -230,6 +230,7 @@ using instance_map = std::unordered_multimap<const void *, instance *>;
 #ifdef Py_GIL_DISABLED
 // Wrapper around PyMutex to provide BasicLockable semantics
 class pymutex {
+    friend class pycritical_section;
    PyMutex mutex;

 public:
@@ -238,6 +239,37 @@ public:
    void unlock() { PyMutex_Unlock(&mutex); }
 };

+class pycritical_section {
+    pymutex &mutex;
+#    if PY_VERSION_HEX >= 0x030E00C1 // 3.14.0rc1
+    PyCriticalSection cs;
+#    endif
+
+public:
+    explicit pycritical_section(pymutex &m) : mutex(m) {
+        // PyCriticalSection_BeginMutex was added in Python 3.15.0a1 and backported to 3.14.0rc1
+#    if PY_VERSION_HEX >= 0x030E00C1 // 3.14.0rc1
+        PyCriticalSection_BeginMutex(&cs, &mutex.mutex);
+#    else
+        // Fall back to direct mutex locking for older free-threaded Python versions
+        mutex.lock();
+#    endif
+    }
+    ~pycritical_section() {
+#    if PY_VERSION_HEX >= 0x030E00C1 // 3.14.0rc1
+        PyCriticalSection_End(&cs);
+#    else
+        mutex.unlock();
+#    endif
+    }
+
+    // Non-copyable and non-movable to prevent double-unlock
+    pycritical_section(const pycritical_section &) = delete;
+    pycritical_section &operator=(const pycritical_section &) = delete;
+    pycritical_section(pycritical_section &&) = delete;
+    pycritical_section &operator=(pycritical_section &&) = delete;
+};
+
 // Instance map shards are used to reduce mutex contention in free-threaded Python.
 struct instance_map_shard {
    instance_map registered_instances;
@@ -337,19 +369,7 @@ struct internals {
    internals(internals &&other) = delete;
    internals &operator=(const internals &other) = delete;
    internals &operator=(internals &&other) = delete;
-    ~internals() {
-        // Normally this destructor runs during interpreter finalization and it may DECREF things.
-        // In odd finalization scenarios it might end up running after the interpreter has
-        // completely shut down, In that case, we should not decref these objects because pymalloc
-        // is gone.  This also applies across sub-interpreters, we should only DECREF when the
-        // original owning interpreter is active.
-        auto *cur_istate = get_interpreter_state_unchecked();
-        if (cur_istate && cur_istate == istate) {
-            Py_CLEAR(instance_base);
-            Py_CLEAR(default_metaclass);
-            Py_CLEAR(static_property_type);
-        }
-    }
+    ~internals() = default;
 };

 // the internals struct (above) is shared between all the modules. local_internals are only
@@ -359,8 +379,6 @@ struct internals {
 // impact any other modules, because the only things accessing the local internals is the
 // module that contains them.
 struct local_internals {
-    local_internals() : istate(get_interpreter_state_unchecked()) {}
-
    // It should be safe to use fast_type_map here because this entire
    // data structure is scoped to our single module, and thus a single
    // DSO and single instance of type_info for any particular type.
@@ -368,19 +386,6 @@ struct local_internals {

    std::forward_list<ExceptionTranslator> registered_exception_translators;
    PyTypeObject *function_record_py_type = nullptr;
-    PyInterpreterState *istate = nullptr;
-
-    ~local_internals() {
-        // Normally this destructor runs during interpreter finalization and it may DECREF things.
-        // In odd finalization scenarios it might end up running after the interpreter has
-        // completely shut down, In that case, we should not decref these objects because pymalloc
-        // is gone.  This also applies across sub-interpreters, we should only DECREF when the
-        // original owning interpreter is active.
-        auto *cur_istate = get_interpreter_state_unchecked();
-        if (cur_istate && cur_istate == istate) {
-            Py_CLEAR(function_record_py_type);
-        }
-    }
 };

 enum class holder_enum_t : uint8_t {
@@ -576,6 +581,10 @@ inline void translate_local_exception(std::exception_ptr p) {
 }
 #endif

+// Sentinel value for the `dtor` parameter of `atomic_get_or_create_in_state_dict`.
+// Indicates no destructor was explicitly provided (distinct from nullptr, which means "leak").
+#define PYBIND11_DTOR_USE_DELETE (reinterpret_cast<void (*)(PyObject *)>(1))
+
 // Get or create per-storage capsule in the current interpreter's state dict.
 //   - The storage is interpreter-dependent: different interpreters will have different storage.
 //     This is important when using multiple-interpreters, to avoid sharing unshareable objects
@@ -592,9 +601,14 @@ inline void translate_local_exception(std::exception_ptr p) {
 //
 // Returns: pair of (pointer to storage, bool indicating if newly created).
 //          The bool follows std::map::insert convention: true = created, false = existed.
+// `dtor`: optional destructor called when the interpreter shuts down.
+//   - If not provided: the storage will be deleted using `delete`.
+//   - If nullptr: the storage will be leaked (useful for singletons that outlive the interpreter).
+//   - If a function: that function will be called with the capsule object.
 template <typename Payload>
 std::pair<Payload *, bool> atomic_get_or_create_in_state_dict(const char *key,
-                                                              void (*dtor)(PyObject *) = nullptr) {
+                                                              void (*dtor)(PyObject *)
+                                                              = PYBIND11_DTOR_USE_DELETE) {
    error_scope err_scope; // preserve any existing Python error states

    auto state_dict = reinterpret_borrow<dict>(get_python_state_dict());
@@ -640,7 +654,7 @@ std::pair<Payload *, bool> atomic_get_or_create_in_state_dict(const char *key,
        // - Otherwise, our `new_capsule` is now in the dict, and it owns the storage and the state
        //   dict will incref it.  We need to set the caller's destructor on it, which will be
        //   called when the interpreter shuts down.
-        if (created && dtor) {
+        if (created && dtor != PYBIND11_DTOR_USE_DELETE) {
            if (PyCapsule_SetDestructor(capsule_obj, dtor) < 0) {
                throw error_already_set();
            }
@@ -657,6 +671,8 @@ std::pair<Payload *, bool> atomic_get_or_create_in_state_dict(const char *key,
    return std::pair<Payload *, bool>(static_cast<Payload *>(raw_ptr), created);
 }

+#undef PYBIND11_DTOR_USE_DELETE
+
 template <typename InternalsType>
 class internals_pp_manager {
 public:
@@ -713,42 +729,75 @@ public:
            // this could be called without an active interpreter, just use what was cached
            if (!tstate || tstate->interp == last_istate_tls()) {
                auto tpp = internals_p_tls();
-
-                delete tpp;
+                {
+                    std::lock_guard<std::mutex> lock(pp_set_mutex_);
+                    pps_have_created_content_.erase(tpp); // untrack deleted pp
+                }
+                delete tpp; // may call back into Python
            }
            unref();
            return;
        }
 #endif
-        delete internals_singleton_pp_;
+        {
+            std::lock_guard<std::mutex> lock(pp_set_mutex_);
+            pps_have_created_content_.erase(internals_singleton_pp_); // untrack deleted pp
+        }
+        delete internals_singleton_pp_; // may call back into Python
        unref();
    }

+    void create_pp_content_once(std::unique_ptr<InternalsType> *const pp) {
+        // Assume the GIL is held here. May call back into Python. We cannot hold the lock with our
+        // mutex here. So there may be multiple threads creating the content at the same time. Only
+        // one will install its content to pp below. Others will be freed when going out of scope.
+        auto tmp = std::unique_ptr<InternalsType>(new InternalsType());
+
+        {
+            // Lock scope must not include Python calls, which may require the GIL and cause
+            // deadlocks.
+            std::lock_guard<std::mutex> lock(pp_set_mutex_);
+
+            if (*pp) {
+                // Already created in another thread.
+                return;
+            }
+
+            // At this point, pp->get() is nullptr.
+            // The content is either not yet created, or was previously destroyed via pp->reset().
+
+            // Detect re-creation of internals after destruction during interpreter shutdown.
+            // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals
+            // have been destroyed, a new empty internals would be created, causing type lookup
+            // failures. See also get_or_create_pp_in_state_dict() comments.
+            if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) {
+                pybind11_fail(
+                    "pybind11::detail::internals_pp_manager::create_pp_content_once() "
+                    "FAILED: reentrant call detected while fetching pybind11 internals!");
+            }
+
+            // Each interpreter can only create its internals once.
+            pps_have_created_content_.insert(pp);
+            // Install the created content.
+            pp->swap(tmp);
+        }
+    }
+
 private:
    internals_pp_manager(char const *id, on_fetch_function *on_fetch)
        : holder_id_(id), on_fetch_(on_fetch) {}

-    static void internals_shutdown(PyObject *capsule) {
-        auto *pp = static_cast<std::unique_ptr<InternalsType> *>(
-            PyCapsule_GetPointer(capsule, nullptr));
-        if (pp) {
-            pp->reset();
-        }
-        // We reset the unique_ptr's contents but cannot delete the unique_ptr itself here.
-        // The pp_manager in this module (and possibly other modules sharing internals) holds
-        // a raw pointer to this unique_ptr, and that pointer would dangle if we deleted it now.
-        //
-        // For pybind11-owned interpreters (via embed.h or subinterpreter.h), destroy() is
-        // called after Py_Finalize/Py_EndInterpreter completes, which safely deletes the
-        // unique_ptr. For interpreters not owned by pybind11 (e.g., a pybind11 extension
-        // loaded into an external interpreter), destroy() is never called and the unique_ptr
-        // shell (8 bytes, not its contents) is leaked.
-        // (See PR #5958 for ideas to eliminate this leak.)
-    }
-
    std::unique_ptr<InternalsType> *get_or_create_pp_in_state_dict() {
+        // The `unique_ptr<InternalsType>` is intentionally leaked on interpreter shutdown.
+        // Once an instance is created, it will never be deleted until the process exits (compare
+        // to interpreter shutdown in multiple-interpreter scenarios).
+        // We cannot guarantee the destruction order of capsules in the interpreter state dict on
+        // interpreter shutdown, so deleting internals too early could cause undefined behavior
+        // when other pybind11 objects access `get_internals()` during finalization (which would
+        // recreate empty internals). See also create_pp_content_once() above.
+        // See https://github.com/pybind/pybind11/pull/5958#discussion_r2717645230.
        auto result = atomic_get_or_create_in_state_dict<std::unique_ptr<InternalsType>>(
-            holder_id_, &internals_shutdown);
+            holder_id_, /*dtor=*/nullptr /* leak the capsule content */);
        auto *pp = result.first;
        bool created = result.second;
        // Only call on_fetch_ when fetching existing internals, not when creating new ones.
@@ -774,7 +823,12 @@ private:
    on_fetch_function *on_fetch_ = nullptr;
    // Pointer-to-pointer to the singleton internals for the first seen interpreter (may not be the
    // main interpreter)
-    std::unique_ptr<InternalsType> *internals_singleton_pp_;
+    std::unique_ptr<InternalsType> *internals_singleton_pp_ = nullptr;
+
+    // Track pointer-to-pointers whose internals have been created, to detect re-entrancy.
+    // Use instance member over static due to singleton pattern of this class.
+    std::unordered_set<std::unique_ptr<InternalsType> *> pps_have_created_content_;
+    std::mutex pp_set_mutex_;
 };

 // If We loaded the internals through `state_dict`, our `error_already_set`
@@ -815,7 +869,8 @@ PYBIND11_NOINLINE internals &get_internals() {
        // Slow path, something needs fetched from the state dict or created
        gil_scoped_acquire_simple gil;
        error_scope err_scope;
-        internals_ptr.reset(new internals());
+
+        ppmgr.create_pp_content_once(&internals_ptr);

        if (!internals_ptr->instance_base) {
            // This calls get_internals, so cannot be called from within the internals constructor
@@ -826,6 +881,31 @@ PYBIND11_NOINLINE internals &get_internals() {
    return *internals_ptr;
 }

+/// Return the PyObject* for the internals capsule (borrowed reference).
+/// Returns nullptr if the capsule doesn't exist yet.
+inline PyObject *get_internals_capsule() {
+    auto state_dict = reinterpret_borrow<dict>(get_python_state_dict());
+    return dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID);
+}
+
+/// Return the key used for local_internals in the state dict.
+/// This function ensures a consistent key is used across all call sites within the same
+/// compilation unit. The key includes the address of a static variable to make it unique per
+/// module (DSO), matching the behavior of get_local_internals_pp_manager().
+inline const std::string &get_local_internals_key() {
+    static const std::string key
+        = PYBIND11_MODULE_LOCAL_ID + std::to_string(reinterpret_cast<uintptr_t>(&key));
+    return key;
+}
+
+/// Return the PyObject* for the local_internals capsule (borrowed reference).
+/// Returns nullptr if the capsule doesn't exist yet.
+inline PyObject *get_local_internals_capsule() {
+    const auto &key = get_local_internals_key();
+    auto state_dict = reinterpret_borrow<dict>(get_python_state_dict());
+    return dict_getitemstring(state_dict.ptr(), key.c_str());
+}
+
 inline void ensure_internals() {
    pybind11::detail::get_internals_pp_manager().unref();
 #ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT
@@ -837,12 +917,10 @@ inline void ensure_internals() {
 }

 inline internals_pp_manager<local_internals> &get_local_internals_pp_manager() {
-    // Use the address of this static itself as part of the key, so that the value is uniquely tied
+    // Use the address of a static variable as part of the key, so that the value is uniquely tied
    // to where the module is loaded in memory
-    static const std::string this_module_idstr
-        = PYBIND11_MODULE_LOCAL_ID
-          + std::to_string(reinterpret_cast<uintptr_t>(&this_module_idstr));
-    return internals_pp_manager<local_internals>::get_instance(this_module_idstr.c_str(), nullptr);
+    return internals_pp_manager<local_internals>::get_instance(get_local_internals_key().c_str(),
+                                                               nullptr);
 }

 /// Works like `get_internals`, but for things which are locally registered.
@@ -850,13 +928,16 @@ inline local_internals &get_local_internals() {
    auto &ppmgr = get_local_internals_pp_manager();
    auto &internals_ptr = *ppmgr.get_pp();
    if (!internals_ptr) {
-        internals_ptr.reset(new local_internals());
+        gil_scoped_acquire_simple gil;
+        error_scope err_scope;
+
+        ppmgr.create_pp_content_once(&internals_ptr);
    }
    return *internals_ptr;
 }

 #ifdef Py_GIL_DISABLED
-#    define PYBIND11_LOCK_INTERNALS(internals) std::unique_lock<pymutex> lock((internals).mutex)
+#    define PYBIND11_LOCK_INTERNALS(internals) pycritical_section lock((internals).mutex)
 #else
 #    define PYBIND11_LOCK_INTERNALS(internals)
 #endif
@@ -885,7 +966,7 @@ inline auto with_exception_translators(const F &cb)
                   get_local_internals().registered_exception_translators)) {
    auto &internals = get_internals();
 #ifdef Py_GIL_DISABLED
-    std::unique_lock<pymutex> lock((internals).exception_translator_mutex);
+    pycritical_section lock((internals).exception_translator_mutex);
 #endif
    auto &local_internals = get_local_internals();
    return cb(internals.registered_exception_translators,
--- a/include/pybind11/numpy.h
+++ b/include/pybind11/numpy.h
@@ -29,6 +29,10 @@
 #include <utility>
 #include <vector>

+#ifdef PYBIND11_HAS_SPAN
+#    include <span>
+#endif
+
 #if defined(PYBIND11_NUMPY_1_ONLY)
 #    error "PYBIND11_NUMPY_1_ONLY is no longer supported (see PR #5595)."
 #endif
@@ -1143,6 +1147,13 @@ public:
    /// Dimensions of the array
    const ssize_t *shape() const { return detail::array_proxy(m_ptr)->dimensions; }

+#ifdef PYBIND11_HAS_SPAN
+    /// Dimensions of the array as a span
+    std::span<const ssize_t, std::dynamic_extent> shape_span() const {
+        return std::span(shape(), static_cast<std::size_t>(ndim()));
+    }
+#endif
+
    /// Dimension along a given axis
    ssize_t shape(ssize_t dim) const {
        if (dim >= ndim()) {
@@ -1154,6 +1165,13 @@ public:
    /// Strides of the array
    const ssize_t *strides() const { return detail::array_proxy(m_ptr)->strides; }

+#ifdef PYBIND11_HAS_SPAN
+    /// Strides of the array as a span
+    std::span<const ssize_t, std::dynamic_extent> strides_span() const {
+        return std::span(strides(), static_cast<std::size_t>(ndim()));
+    }
+#endif
+
    /// Stride along a given axis
    ssize_t strides(ssize_t dim) const {
        if (dim >= ndim()) {
--- a/include/pybind11/pybind11.h
+++ b/include/pybind11/pybind11.h
@@ -3173,6 +3173,11 @@ iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) {
    using state = detail::iterator_state<Access, Policy, Iterator, Sentinel, ValueType, Extra...>;
    // TODO: state captures only the types of Extra, not the values

+    // For Python < 3.14.0rc1, pycritical_section uses direct mutex locking (same as a unique
+    // lock), which may deadlock during type registration. See detail/internals.h for details.
+#if PY_VERSION_HEX >= 0x030E00C1 // 3.14.0rc1
+    PYBIND11_LOCK_INTERNALS(get_internals());
+#endif
    if (!detail::get_type_info(typeid(state), false)) {
        class_<state>(handle(), "iterator", pybind11::module_local())
            .def(
--- a/tests/env.py
+++ b/tests/env.py
@@ -11,6 +11,18 @@ MACOS = sys.platform.startswith("darwin")
 WIN = sys.platform.startswith("win32") or sys.platform.startswith("cygwin")
 FREEBSD = sys.platform.startswith("freebsd")

+MUSLLINUX = False
+MANYLINUX = False
+if LINUX:
+
+    def _is_musl() -> bool:
+        libc, _ = platform.libc_ver()
+        return libc == "musl" or (libc != "glibc" and libc != "")
+
+    MUSLLINUX = _is_musl()
+    MANYLINUX = not MUSLLINUX
+    del _is_musl
+
 CPYTHON = platform.python_implementation() == "CPython"
 PYPY = platform.python_implementation() == "PyPy"
 GRAALPY = sys.implementation.name == "graalpy"
@@ -29,3 +41,31 @@ TYPES_ARE_IMMORTAL = (
    or GRAALPY
    or (CPYTHON and PY_GIL_DISABLED and (3, 13) <= sys.version_info < (3, 14))
 )
+
+
+def check_script_success_in_subprocess(code: str, *, rerun: int = 8) -> None:
+    """Runs the given code in a subprocess."""
+    import os
+    import subprocess
+    import sys
+    import textwrap
+
+    code = textwrap.dedent(code).strip()
+    try:
+        for _ in range(rerun):  # run flakily failing test multiple times
+            subprocess.check_output(
+                [sys.executable, "-c", code],
+                cwd=os.getcwd(),
+                stderr=subprocess.STDOUT,
+                text=True,
+            )
+    except subprocess.CalledProcessError as ex:
+        raise RuntimeError(
+            f"Subprocess failed with exit code {ex.returncode}.\n\n"
+            f"Code:\n"
+            f"```python\n"
+            f"{code}\n"
+            f"```\n\n"
+            f"Output:\n"
+            f"{ex.output}"
+        ) from None
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -10,7 +10,7 @@ numpy~=1.22.2; platform_python_implementation=="CPython" and python_version=="3.
 numpy~=1.26.0; platform_python_implementation=="CPython" and python_version>="3.11" and python_version<"3.13" and platform_machine!="ARM64"
 numpy>=2.3.0; platform_python_implementation=="CPython" and python_version>="3.11" and platform_machine=="ARM64"
 numpy~=2.2.0; platform_python_implementation=="CPython" and python_version=="3.13" and platform_machine!="ARM64"
-numpy==2.4.0; platform_python_implementation=="CPython" and python_version>="3.14"
+numpy>=2.4.0; platform_python_implementation=="CPython" and python_version>="3.14"
 pytest>=6
 pytest-timeout
 scipy~=1.5.4; platform_python_implementation=="CPython" and python_version<"3.10"
--- a/tests/test_custom_type_setup.cpp
+++ b/tests/test_custom_type_setup.cpp
@@ -7,22 +7,67 @@
    BSD-style license that can be found in the LICENSE file.
 */

+#include <pybind11/detail/internals.h>
 #include <pybind11/pybind11.h>

 #include "pybind11_tests.h"

+#include <vector>
+
 namespace py = pybind11;

 namespace {
+struct ContainerOwnsPythonObjects {
+    std::vector<py::object> list;

-struct OwnsPythonObjects {
-    py::object value = py::none();
+    void append(const py::object &obj) { list.emplace_back(obj); }
+    py::object at(py::ssize_t index) const {
+        if (index >= size() || index < 0) {
+            throw py::index_error("Index out of range");
+        }
+        return list.at(py::size_t(index));
+    }
+    py::ssize_t size() const { return py::ssize_t_cast(list.size()); }
+    void clear() { list.clear(); }
 };
+
+void add_gc_checkers_with_weakrefs(const py::object &obj) {
+    py::handle global_capsule = py::detail::get_internals_capsule();
+    if (!global_capsule) {
+        throw std::runtime_error("No global internals capsule found");
+    }
+    (void) py::weakref(obj, py::cpp_function([global_capsule](py::handle weakref) -> void {
+                           py::handle current_global_capsule = py::detail::get_internals_capsule();
+                           if (!current_global_capsule.is(global_capsule)) {
+                               throw std::runtime_error(
+                                   "Global internals capsule was destroyed prematurely");
+                           }
+                           weakref.dec_ref();
+                       }))
+        .release();
+
+    py::handle local_capsule = py::detail::get_local_internals_capsule();
+    if (!local_capsule) {
+        throw std::runtime_error("No local internals capsule found");
+    }
+    (void) py::weakref(
+        obj, py::cpp_function([local_capsule](py::handle weakref) -> void {
+            py::handle current_local_capsule = py::detail::get_local_internals_capsule();
+            if (!current_local_capsule.is(local_capsule)) {
+                throw std::runtime_error("Local internals capsule was destroyed prematurely");
+            }
+            weakref.dec_ref();
+        }))
+        .release();
+}
 } // namespace

 TEST_SUBMODULE(custom_type_setup, m) {
-    py::class_<OwnsPythonObjects> cls(
-        m, "OwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) {
+    py::class_<ContainerOwnsPythonObjects> cls(
+        m,
+        "ContainerOwnsPythonObjects",
+        // Please review/update docs/advanced/classes.rst after making changes here.
+        py::custom_type_setup([](PyHeapTypeObject *heap_type) {
            auto *type = &heap_type->ht_type;
            type->tp_flags |= Py_TPFLAGS_HAVE_GC;
            type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) {
@@ -31,19 +76,29 @@ TEST_SUBMODULE(custom_type_setup, m) {
                Py_VISIT(Py_TYPE(self_base));
 #endif
                if (py::detail::is_holder_constructed(self_base)) {
-                    auto &self = py::cast<OwnsPythonObjects &>(py::handle(self_base));
-                    Py_VISIT(self.value.ptr());
+                    auto &self = py::cast<ContainerOwnsPythonObjects &>(py::handle(self_base));
+                    for (auto &item : self.list) {
+                        Py_VISIT(item.ptr());
+                    }
                }
                return 0;
            };
            type->tp_clear = [](PyObject *self_base) {
                if (py::detail::is_holder_constructed(self_base)) {
-                    auto &self = py::cast<OwnsPythonObjects &>(py::handle(self_base));
-                    self.value = py::none();
+                    auto &self = py::cast<ContainerOwnsPythonObjects &>(py::handle(self_base));
+                    for (auto &item : self.list) {
+                        Py_CLEAR(item.ptr());
+                    }
+                    self.list.clear();
                }
                return 0;
            };
        }));
    cls.def(py::init<>());
-    cls.def_readwrite("value", &OwnsPythonObjects::value);
+    cls.def("append", &ContainerOwnsPythonObjects::append);
+    cls.def("at", &ContainerOwnsPythonObjects::at);
+    cls.def("size", &ContainerOwnsPythonObjects::size);
+    cls.def("clear", &ContainerOwnsPythonObjects::clear);
+
+    m.def("add_gc_checkers_with_weakrefs", &add_gc_checkers_with_weakrefs);
 }
--- a/tests/test_custom_type_setup.py
+++ b/tests/test_custom_type_setup.py
@@ -1,11 +1,14 @@
 from __future__ import annotations

 import gc
+import os
+import sys
 import weakref

 import pytest

-import env  # noqa: F401
+import env
+import pybind11_tests
 from pybind11_tests import custom_type_setup as m


@@ -36,15 +39,46 @@ def gc_tester():
 # PyPy does not seem to reliably garbage collect.
@pytest.mark.skipif("env.PYPY or env.GRAALPY")
 def test_self_cycle(gc_tester):
-    obj = m.OwnsPythonObjects()
-    obj.value = obj
+    obj = m.ContainerOwnsPythonObjects()
+    obj.append(obj)
    gc_tester(obj)


 # PyPy does not seem to reliably garbage collect.
@pytest.mark.skipif("env.PYPY or env.GRAALPY")
 def test_indirect_cycle(gc_tester):
-    obj = m.OwnsPythonObjects()
-    obj_list = [obj]
-    obj.value = obj_list
+    obj = m.ContainerOwnsPythonObjects()
+    obj.append([obj])
    gc_tester(obj)
+
+
+@pytest.mark.skipif(
+    env.IOS or sys.platform.startswith("emscripten"),
+    reason="Requires subprocess support",
+)
+@pytest.mark.skipif("env.PYPY or env.GRAALPY")
+def test_py_cast_useable_on_shutdown():
+    """Test that py::cast works during interpreter shutdown.
+
+    See PR #5972 and https://github.com/pybind/pybind11/pull/5958#discussion_r2717645230.
+    """
+    env.check_script_success_in_subprocess(
+        f"""
+        import sys
+
+        sys.path.insert(0, {os.path.dirname(env.__file__)!r})
+        sys.path.insert(0, {os.path.dirname(pybind11_tests.__file__)!r})
+
+        from pybind11_tests import custom_type_setup as m
+
+        # Create a self-referential cycle that will be collected during shutdown.
+        # The tp_traverse and tp_clear callbacks call py::cast, which requires
+        # internals to still be valid.
+        obj = m.ContainerOwnsPythonObjects()
+        obj.append(obj)
+
+        # Add weakref callbacks that verify the capsule is still alive when the
+        # pybind11 object is garbage collected during shutdown.
+        m.add_gc_checkers_with_weakrefs(obj)
+        """
+    )
--- a/tests/test_multiple_interpreters.py
+++ b/tests/test_multiple_interpreters.py
@@ -3,7 +3,6 @@ from __future__ import annotations
 import contextlib
 import os
 import pickle
-import subprocess
 import sys
 import textwrap

@@ -219,6 +218,7 @@ PREAMBLE_CODE = textwrap.dedent(
    def test():
        import sys

+        sys.path.insert(0, {os.path.dirname(env.__file__)!r})
        sys.path.insert(0, {os.path.dirname(pybind11_tests.__file__)!r})

        import collections
@@ -269,36 +269,13 @@ def test_import_module_with_singleton_per_interpreter():
        interp.exec(code)


-def check_script_success_in_subprocess(code: str, *, rerun: int = 8) -> None:
-    """Runs the given code in a subprocess."""
-    code = textwrap.dedent(code).strip()
-    try:
-        for _ in range(rerun):  # run flakily failing test multiple times
-            subprocess.check_output(
-                [sys.executable, "-c", code],
-                cwd=os.getcwd(),
-                stderr=subprocess.STDOUT,
-                text=True,
-            )
-    except subprocess.CalledProcessError as ex:
-        raise RuntimeError(
-            f"Subprocess failed with exit code {ex.returncode}.\n\n"
-            f"Code:\n"
-            f"```python\n"
-            f"{code}\n"
-            f"```\n\n"
-            f"Output:\n"
-            f"{ex.output}"
-        ) from None
-
-
@pytest.mark.skipif(
    sys.platform.startswith("emscripten"), reason="Requires loadable modules"
 )
@pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+")
 def test_import_in_subinterpreter_after_main():
    """Tests that importing a module in a subinterpreter after the main interpreter works correctly"""
-    check_script_success_in_subprocess(
+    env.check_script_success_in_subprocess(
        PREAMBLE_CODE
        + textwrap.dedent(
            """
@@ -319,7 +296,7 @@ def test_import_in_subinterpreter_after_main():
        )
    )

-    check_script_success_in_subprocess(
+    env.check_script_success_in_subprocess(
        PREAMBLE_CODE
        + textwrap.dedent(
            """
@@ -354,7 +331,7 @@ def test_import_in_subinterpreter_after_main():
@pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+")
 def test_import_in_subinterpreter_before_main():
    """Tests that importing a module in a subinterpreter before the main interpreter works correctly"""
-    check_script_success_in_subprocess(
+    env.check_script_success_in_subprocess(
        PREAMBLE_CODE
        + textwrap.dedent(
            """
@@ -375,7 +352,7 @@ def test_import_in_subinterpreter_before_main():
        )
    )

-    check_script_success_in_subprocess(
+    env.check_script_success_in_subprocess(
        PREAMBLE_CODE
        + textwrap.dedent(
            """
@@ -401,7 +378,7 @@ def test_import_in_subinterpreter_before_main():
        )
    )

-    check_script_success_in_subprocess(
+    env.check_script_success_in_subprocess(
        PREAMBLE_CODE
        + textwrap.dedent(
            """
@@ -431,10 +408,15 @@ def test_import_in_subinterpreter_before_main():
@pytest.mark.skipif(
    sys.platform.startswith("emscripten"), reason="Requires loadable modules"
 )
+@pytest.mark.xfail(
+    env.MUSLLINUX,
+    reason="Flaky on musllinux, see also: https://github.com/pybind/pybind11/pull/5972#discussion_r2755283335",
+    strict=False,
+)
@pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+")
 def test_import_in_subinterpreter_concurrently():
    """Tests that importing a module in multiple subinterpreters concurrently works correctly"""
-    check_script_success_in_subprocess(
+    env.check_script_success_in_subprocess(
        PREAMBLE_CODE
        + textwrap.dedent(
            """
--- a/tests/test_numpy_array.cpp
+++ b/tests/test_numpy_array.cpp
@@ -14,6 +14,7 @@

 #include <cstdint>
 #include <utility>
+#include <vector>

 // Size / dtype checks.
 struct DtypeCheck {
@@ -246,6 +247,22 @@ TEST_SUBMODULE(numpy_array, sm) {
    sm.def("nbytes", [](const arr &a) { return a.nbytes(); });
    sm.def("owndata", [](const arr &a) { return a.owndata(); });

+#ifdef PYBIND11_HAS_SPAN
+    // test_shape_strides_span
+    sm.def("shape_span", [](const arr &a) {
+        auto span = a.shape_span();
+        return std::vector<py::ssize_t>(span.begin(), span.end());
+    });
+    sm.def("strides_span", [](const arr &a) {
+        auto span = a.strides_span();
+        return std::vector<py::ssize_t>(span.begin(), span.end());
+    });
+    // Test that spans can be used to construct new arrays
+    sm.def("array_from_spans", [](const arr &a) {
+        return py::array(a.dtype(), a.shape_span(), a.strides_span(), a.data(), a);
+    });
+#endif
+
    // test_index_offset
    def_index_fn(index_at, const arr &);
    def_index_fn(index_at_t, const arr_t &);
--- a/tests/test_numpy_array.py
+++ b/tests/test_numpy_array.py
@@ -68,6 +68,45 @@ def test_array_attributes():
    assert not m.owndata(a)


+@pytest.mark.skipif(not hasattr(m, "shape_span"), reason="std::span not available")
+def test_shape_strides_span():
+    # Test 0-dimensional array (scalar)
+    a = np.array(42, "f8")
+    assert m.ndim(a) == 0
+    assert m.shape_span(a) == []
+    assert m.strides_span(a) == []
+
+    # Test 1-dimensional array
+    a = np.array([1, 2, 3, 4], "u2")
+    assert m.ndim(a) == 1
+    assert m.shape_span(a) == [4]
+    assert m.strides_span(a) == [2]
+
+    # Test 2-dimensional array
+    a = np.array([[1, 2, 3], [4, 5, 6]], "u2").view()
+    a.flags.writeable = False
+    assert m.ndim(a) == 2
+    assert m.shape_span(a) == [2, 3]
+    assert m.strides_span(a) == [6, 2]
+
+    # Test 3-dimensional array
+    a = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], "i4")
+    assert m.ndim(a) == 3
+    assert m.shape_span(a) == [2, 2, 2]
+    # Verify spans match regular shape/strides
+    assert list(m.shape_span(a)) == list(m.shape(a))
+    assert list(m.strides_span(a)) == list(m.strides(a))
+
+    # Test that spans can be used to construct new arrays
+    original = np.array([[1, 2, 3], [4, 5, 6]], "f4")
+    new_array = m.array_from_spans(original)
+    assert new_array.shape == original.shape
+    assert new_array.strides == original.strides
+    assert new_array.dtype == original.dtype
+    # Verify data is shared (since we pass the same data pointer)
+    np.testing.assert_array_equal(new_array, original)
+
+
@pytest.mark.parametrize(
    ("args", "ret"), [([], 0), ([0], 0), ([1], 3), ([0, 1], 1), ([1, 2], 5)]
 )
--- a/tests/test_with_catch/test_subinterpreter.cpp
+++ b/tests/test_with_catch/test_subinterpreter.cpp
@@ -501,15 +501,21 @@ TEST_CASE("Per-Subinterpreter GIL") {

            // wait for something to set sync to our thread number
            // we are holding our subinterpreter's GIL
-            while (sync != num)
-                std::this_thread::sleep_for(std::chrono::microseconds(1));
+            {
+                py::gil_scoped_release nogil;
+                while (sync != num)
+                    std::this_thread::sleep_for(std::chrono::microseconds(1));
+            }

            // now change it so the next thread can move on
            ++sync;

            // but keep holding the GIL until after the next thread moves on as well
-            while (sync == num + 1)
-                std::this_thread::sleep_for(std::chrono::microseconds(1));
+            {
+                py::gil_scoped_release nogil;
+                while (sync == num + 1)
+                    std::this_thread::sleep_for(std::chrono::microseconds(1));
+            }

            // one last check before quitting the thread, the internals should be different
            auto sub_int