From 1dc76208d5822e78fc8129552b4d622c78b7ce64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Dec 2025 09:18:17 -0800 Subject: [PATCH 01/30] chore(deps): bump urllib3 from 2.5.0 to 2.6.0 in /docs (#5920) Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.5.0 to 2.6.0. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/2.5.0...2.6.0) --- updated-dependencies: - dependency-name: urllib3 dependency-version: 2.6.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 0bbb01ac7..9d2ea4f85 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -23,8 +23,6 @@ idna==3.7 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==8.7.0 - # via sphinx jinja2==3.1.6 # via # myst-parser @@ -85,7 +83,5 @@ sphinxcontrib-serializinghtml==1.1.10 # via sphinx sphinxcontrib-svg2pdfconverter==1.2.2 # via -r requirements.in -urllib3==2.5.0 +urllib3==2.6.0 # via requests -zipp==3.23.0 - # via importlib-metadata From 1c1d01c8a1d71436b2c669be8fa76987e81b1946 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Mon, 8 Dec 2025 18:17:17 +0100 Subject: [PATCH 02/30] Use well-known labels in `project.urls` (#5921) https://packaging.python.org/en/latest/specifications/well-known-project-urls/#well-known-labels --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a4b43d3b0..dd58cd5b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ requires-python = ">=3.8" [project.urls] Homepage = "https://github.com/pybind/pybind11" Documentation = "https://pybind11.readthedocs.io/" -"Bug Tracker" = "https://github.com/pybind/pybind11/issues" +"Issue Tracker" = "https://github.com/pybind/pybind11/issues" Discussions = "https://github.com/pybind/pybind11/discussions" Changelog = "https://pybind11.readthedocs.io/en/latest/changelog.html" Chat = "https://gitter.im/pybind/Lobby" From 66515309638bee5a4b06dc65c2d01ceffaf7f563 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Mon, 8 Dec 2025 18:25:25 +0100 Subject: [PATCH 03/30] Enforce more ruff rules (#5922) * Apply ruff/flake8-implicit-str-concat rule ISC003 ISC003 Explicitly concatenated string should be implicitly concatenated * Enforce ruff/flake8-implicit-str-concat rules (ISC) * Apply ruff/Perflint rule PERF102 PERF102 When using only the values of a dict use the `values()` method * Apply ruff/Perflint rule PERF401 PERF401 Use a list comprehension to create a transformed list * Enforce ruff/Perflint rules (PERF) --- pyproject.toml | 2 ++ tests/test_class_sh_basic.py | 4 ++-- tests/test_iostream.py | 6 +----- tests/test_stl_binders.py | 6 +++--- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dd58cd5b3..274de222a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -152,6 +152,8 @@ extend-select = [ "C4", # flake8-comprehensions "EM", # flake8-errmsg "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "PERF", # Perflint "PGH", # pygrep-hooks "PIE", # flake8-pie "PL", # pylint diff --git a/tests/test_class_sh_basic.py b/tests/test_class_sh_basic.py index 18f0b0646..aaea87b8f 100644 --- a/tests/test_class_sh_basic.py +++ b/tests/test_class_sh_basic.py @@ -112,8 +112,8 @@ def test_pass_unique_ptr_disowns(pass_f, rtrn_f, expected): pass_f(obj) assert str(exc_info.value) == ( "Missing value for wrapped C++ type" - + " `pybind11_tests::class_sh_basic::atyp`:" - + " Python instance was disowned." + " `pybind11_tests::class_sh_basic::atyp`:" + " Python instance was disowned." ) diff --git a/tests/test_iostream.py b/tests/test_iostream.py index 00b24ab70..791b9e048 100644 --- a/tests/test_iostream.py +++ b/tests/test_iostream.py @@ -288,11 +288,7 @@ def test_redirect_both(capfd): def test_threading(): with m.ostream_redirect(stdout=True, stderr=False): # start some threads - threads = [] - - # start some threads - for _j in range(20): - threads.append(m.TestThread()) + threads = [m.TestThread() for _j in range(20)] # give the threads some time to fail threads[0].sleep() diff --git a/tests/test_stl_binders.py b/tests/test_stl_binders.py index 9856ba462..518f2df2b 100644 --- a/tests/test_stl_binders.py +++ b/tests/test_stl_binders.py @@ -258,7 +258,7 @@ def test_noncopyable_containers(): assert nvnc[i][j].value == j + 1 # Note: maps do not have .values() - for _, v in nvnc.items(): + for v in nvnc.values(): for i, j in enumerate(v, start=1): assert j.value == i @@ -269,7 +269,7 @@ def test_noncopyable_containers(): assert nmnc[i][j].value == 10 * j vsum = 0 - for _, v_o in nmnc.items(): + for v_o in nmnc.values(): for k_i, v_i in v_o.items(): assert v_i.value == 10 * k_i vsum += v_i.value @@ -283,7 +283,7 @@ def test_noncopyable_containers(): assert numnc[i][j].value == 10 * j vsum = 0 - for _, v_o in numnc.items(): + for v_o in numnc.values(): for k_i, v_i in v_o.items(): assert v_i.value == 10 * k_i vsum += v_i.value From 3ebbecb8af053d8ee1530a141dc3c46ac5846e4b Mon Sep 17 00:00:00 2001 From: Yuanyuan Chen Date: Tue, 9 Dec 2025 01:36:51 +0800 Subject: [PATCH 04/30] Add more readability tidy rules (#5924) * Apply clang-tidy readibility fixes Signed-off-by: Yuanyuan Chen * Add checks Signed-off-by: Yuanyuan Chen * More fixes Signed-off-by: cyy --------- Signed-off-by: Yuanyuan Chen Signed-off-by: cyy --- .clang-tidy | 3 +++ include/pybind11/cast.h | 2 +- include/pybind11/detail/common.h | 8 ++------ include/pybind11/detail/internals.h | 4 ++-- include/pybind11/gil_safe_call_once.h | 2 +- include/pybind11/numpy.h | 2 +- include/pybind11/pybind11.h | 9 ++++----- include/pybind11/pytypes.h | 10 +++++----- include/pybind11/stl_bind.h | 2 +- tests/test_builtin_casters.cpp | 2 +- tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp | 6 +++--- tests/test_factory_constructors.cpp | 2 +- tests/test_numpy_array.cpp | 8 ++++---- tests/test_sequences_and_iterators.cpp | 2 +- tests/test_smart_ptr.cpp | 7 ++++--- tests/test_stl_binders.cpp | 4 ++-- 16 files changed, 36 insertions(+), 37 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 3a1995c32..a375f7614 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -48,7 +48,10 @@ Checks: | readability-misplaced-array-index, readability-non-const-parameter, readability-qualified-auto, + readability-redundant-casting, readability-redundant-function-ptr-dereference, + readability-redundant-inline-specifier, + readability-redundant-member-init, readability-redundant-smartptr-get, readability-redundant-string-cstr, readability-simplify-subscript-expr, diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 5ecded36f..3acb560b0 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -475,7 +475,7 @@ public: private: // Test if an object is a NumPy boolean (without fetching the type). - static inline bool is_numpy_bool(handle object) { + static bool is_numpy_bool(handle object) { const char *type_name = Py_TYPE(object.ptr())->tp_name; // Name changed to `numpy.bool` in NumPy 2, `numpy.bool_` is needed for 1.x support return std::strcmp("numpy.bool", type_name) == 0 diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 05d675589..ce9014a7e 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -590,14 +590,10 @@ enum class return_value_policy : uint8_t { PYBIND11_NAMESPACE_BEGIN(detail) -inline static constexpr int log2(size_t n, int k = 0) { - return (n <= 1) ? k : log2(n >> 1, k + 1); -} +static constexpr int log2(size_t n, int k = 0) { return (n <= 1) ? k : log2(n >> 1, k + 1); } // Returns the size as a multiple of sizeof(void *), rounded up. -inline static constexpr size_t size_in_ptrs(size_t s) { - return 1 + ((s - 1) >> log2(sizeof(void *))); -} +static constexpr size_t size_in_ptrs(size_t s) { return 1 + ((s - 1) >> log2(sizeof(void *))); } /** * The space to allocate for simple layout instance holders (see below) in multiple of the size of diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 2600d4356..4d6c147db 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -186,7 +186,7 @@ template using type_map = std::unordered_map; struct override_hash { - inline size_t operator()(const std::pair &v) const { + size_t operator()(const std::pair &v) const { size_t value = std::hash()(v.first); value ^= std::hash()(v.second) + 0x9e3779b9 + (value << 6) + (value >> 2); return value; @@ -555,7 +555,7 @@ class internals_pp_manager { public: using on_fetch_function = void(InternalsType *); - inline static internals_pp_manager &get_instance(char const *id, on_fetch_function *on_fetch) { + static internals_pp_manager &get_instance(char const *id, on_fetch_function *on_fetch) { static internals_pp_manager instance(id, on_fetch); return instance; } diff --git a/include/pybind11/gil_safe_call_once.h b/include/pybind11/gil_safe_call_once.h index 44e68f029..2abd8fc32 100644 --- a/include/pybind11/gil_safe_call_once.h +++ b/include/pybind11/gil_safe_call_once.h @@ -87,7 +87,7 @@ public: private: alignas(T) char storage_[sizeof(T)] = {}; - std::once_flag once_flag_ = {}; + std::once_flag once_flag_; #ifdef Py_GIL_DISABLED std::atomic_bool #else diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 7f62157f5..236d90bd3 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1860,7 +1860,7 @@ public: using value_type = container_type::value_type; using size_type = container_type::size_type; - common_iterator() : m_strides() {} + common_iterator() = default; common_iterator(void *ptr, const container_type &strides, const container_type &shape) : p_ptr(reinterpret_cast(ptr)), m_strides(strides.size()) { diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 60db0a087..a2043b264 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -811,9 +811,9 @@ protected: // so they cannot be freed. Once the function has been created, they can. // Check `make_function_record` for more details. if (free_strings) { - std::free((char *) rec->name); - std::free((char *) rec->doc); - std::free((char *) rec->signature); + std::free(rec->name); + std::free(rec->doc); + std::free(rec->signature); for (auto &arg : rec->args) { std::free(const_cast(arg.name)); std::free(const_cast(arg.descr)); @@ -2486,8 +2486,7 @@ private: static void init_holder_from_existing(const detail::value_and_holder &v_h, const holder_type *holder_ptr, std::true_type /*is_copy_constructible*/) { - new (std::addressof(v_h.holder())) - holder_type(*reinterpret_cast(holder_ptr)); + new (std::addressof(v_h.holder())) holder_type(*holder_ptr); } static void init_holder_from_existing(const detail::value_and_holder &v_h, diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index cee4ab562..b28692fd7 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -861,7 +861,7 @@ bool isinstance(handle obj) { } template <> -inline bool isinstance(handle) = delete; +bool isinstance(handle) = delete; template <> inline bool isinstance(handle obj) { return obj.ptr() != nullptr; @@ -994,7 +994,7 @@ inline PyObject *dict_getitem(PyObject *v, PyObject *key) { inline PyObject *dict_getitemstringref(PyObject *v, const char *key) { #if PY_VERSION_HEX >= 0x030D0000 - PyObject *rv; + PyObject *rv = nullptr; if (PyDict_GetItemStringRef(v, key, &rv) < 0) { throw error_already_set(); } @@ -1555,7 +1555,7 @@ private: } private: - object value = {}; + object value; }; class type : public object { @@ -1914,7 +1914,7 @@ public: } } // NOLINTNEXTLINE(google-explicit-constructor) - float_(double value = .0) : object(PyFloat_FromDouble((double) value), stolen_t{}) { + float_(double value = .0) : object(PyFloat_FromDouble(value), stolen_t{}) { if (!m_ptr) { pybind11_fail("Could not allocate float object!"); } @@ -1922,7 +1922,7 @@ public: // NOLINTNEXTLINE(google-explicit-constructor) operator float() const { return (float) PyFloat_AsDouble(m_ptr); } // NOLINTNEXTLINE(google-explicit-constructor) - operator double() const { return (double) PyFloat_AsDouble(m_ptr); } + operator double() const { return PyFloat_AsDouble(m_ptr); } }; class weakref : public object { diff --git a/include/pybind11/stl_bind.h b/include/pybind11/stl_bind.h index 3eb1e53f4..8202300c7 100644 --- a/include/pybind11/stl_bind.h +++ b/include/pybind11/stl_bind.h @@ -244,7 +244,7 @@ void vector_modifiers( } auto *seq = new Vector(); - seq->reserve((size_t) slicelength); + seq->reserve(slicelength); for (size_t i = 0; i < slicelength; ++i) { seq->push_back(v[start]); diff --git a/tests/test_builtin_casters.cpp b/tests/test_builtin_casters.cpp index 1aa9f89b4..6cde727ad 100644 --- a/tests/test_builtin_casters.cpp +++ b/tests/test_builtin_casters.cpp @@ -367,7 +367,7 @@ TEST_SUBMODULE(builtin_casters, m) { m.def("complex_noconvert", [](std::complex x) { return x; }, py::arg{}.noconvert()); // test int vs. long (Python 2) - m.def("int_cast", []() { return (int) 42; }); + m.def("int_cast", []() { return 42; }); m.def("long_cast", []() { return (long) 42; }); m.def("longlong_cast", []() { return ULLONG_MAX; }); diff --git a/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp b/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp index 5580848c6..6936379c2 100644 --- a/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp +++ b/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp @@ -36,9 +36,9 @@ struct PySpBase : SpBase, py::trampoline_self_life_support { struct SpBaseTester { std::shared_ptr get_object() const { return m_obj; } void set_object(std::shared_ptr obj) { m_obj = std::move(obj); } - bool is_base_used() { return m_obj->is_base_used(); } - bool has_instance() { return (bool) m_obj; } - bool has_python_instance() { return m_obj && m_obj->has_python_instance(); } + bool is_base_used() const { return m_obj->is_base_used(); } + bool has_instance() const { return (bool) m_obj; } + bool has_python_instance() const { return m_obj && m_obj->has_python_instance(); } void set_nonpython_instance() { m_obj = std::make_shared(); } std::shared_ptr m_obj; }; diff --git a/tests/test_factory_constructors.cpp b/tests/test_factory_constructors.cpp index e50494b33..c96a3a31f 100644 --- a/tests/test_factory_constructors.cpp +++ b/tests/test_factory_constructors.cpp @@ -73,7 +73,7 @@ public: // Inheritance test class TestFactory4 : public TestFactory3 { public: - TestFactory4() : TestFactory3() { print_default_created(this); } + TestFactory4() { print_default_created(this); } explicit TestFactory4(int v) : TestFactory3(v) { print_created(this, v); } ~TestFactory4() override { print_destroyed(this); } }; diff --git a/tests/test_numpy_array.cpp b/tests/test_numpy_array.cpp index 28359c46d..ac6b1cfe3 100644 --- a/tests/test_numpy_array.cpp +++ b/tests/test_numpy_array.cpp @@ -17,8 +17,8 @@ // Size / dtype checks. struct DtypeCheck { - py::dtype numpy{}; - py::dtype pybind11{}; + py::dtype numpy; + py::dtype pybind11; }; template @@ -43,11 +43,11 @@ std::vector get_concrete_dtype_checks() { } struct DtypeSizeCheck { - std::string name{}; + std::string name; int size_cpp{}; int size_numpy{}; // For debugging. - py::dtype dtype{}; + py::dtype dtype; }; template diff --git a/tests/test_sequences_and_iterators.cpp b/tests/test_sequences_and_iterators.cpp index ccb0d1499..3f03daf38 100644 --- a/tests/test_sequences_and_iterators.cpp +++ b/tests/test_sequences_and_iterators.cpp @@ -556,7 +556,7 @@ TEST_SUBMODULE(sequences_and_iterators, m) { }); m.def("count_nonzeros", [](const py::dict &d) { - return std::count_if(d.begin(), d.end(), [](std::pair p) { + return std::count_if(d.begin(), d.end(), [](const std::pair &p) { return p.second.cast() != 0; }); }); diff --git a/tests/test_smart_ptr.cpp b/tests/test_smart_ptr.cpp index 2e98d469f..0ac1a41bd 100644 --- a/tests/test_smart_ptr.cpp +++ b/tests/test_smart_ptr.cpp @@ -220,7 +220,7 @@ struct SharedPtrRef { ~A() { print_destroyed(this); } }; - A value = {}; + A value; std::shared_ptr shared = std::make_shared(); }; @@ -228,13 +228,14 @@ struct SharedPtrRef { struct SharedFromThisRef { struct B : std::enable_shared_from_this { B() { print_created(this); } - // NOLINTNEXTLINE(bugprone-copy-constructor-init) + // NOLINTNEXTLINE(bugprone-copy-constructor-init, readability-redundant-member-init) B(const B &) : std::enable_shared_from_this() { print_copy_created(this); } + // NOLINTNEXTLINE(readability-redundant-member-init) B(B &&) noexcept : std::enable_shared_from_this() { print_move_created(this); } ~B() { print_destroyed(this); } }; - B value = {}; + B value; std::shared_ptr shared = std::make_shared(); }; diff --git a/tests/test_stl_binders.cpp b/tests/test_stl_binders.cpp index f846ae848..f399ec0e4 100644 --- a/tests/test_stl_binders.cpp +++ b/tests/test_stl_binders.cpp @@ -55,7 +55,7 @@ template Map *times_ten(int n) { auto *m = new Map(); for (int i = 1; i <= n; i++) { - m->emplace(int(i), E_nc(10 * i)); + m->emplace(i, E_nc(10 * i)); } return m; } @@ -65,7 +65,7 @@ NestMap *times_hundred(int n) { auto *m = new NestMap(); for (int i = 1; i <= n; i++) { for (int j = 1; j <= n; j++) { - (*m)[i].emplace(int(j * 10), E_nc(100 * j)); + (*m)[i].emplace(j * 10, E_nc(100 * j)); } } return m; From 228f56361016ab9e27d5ef21853542dab3e37693 Mon Sep 17 00:00:00 2001 From: Yuanyuan Chen Date: Tue, 9 Dec 2025 08:00:51 +0800 Subject: [PATCH 05/30] Skip cross module exception translation on FreeBSD (#5925) * Skip cross module exception translation on FreeBSD Signed-off-by: cyy * Link to PR number Signed-off-by: cyy --------- Signed-off-by: cyy --- tests/env.py | 1 + tests/test_exceptions.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/env.py b/tests/env.py index ccb1fd30b..4b48e9193 100644 --- a/tests/env.py +++ b/tests/env.py @@ -8,6 +8,7 @@ ANDROID = sys.platform.startswith("android") LINUX = sys.platform.startswith("linux") MACOS = sys.platform.startswith("darwin") WIN = sys.platform.startswith("win32") or sys.platform.startswith("cygwin") +FREEBSD = sys.platform.startswith("freebsd") CPYTHON = platform.python_implementation() == "CPython" PYPY = platform.python_implementation() == "PyPy" diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 79b387903..59845b441 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -76,9 +76,9 @@ def test_cross_module_exceptions(msg): # TODO: FIXME @pytest.mark.xfail( - "(env.MACOS and env.PYPY) or env.ANDROID", + "(env.MACOS and env.PYPY) or env.ANDROID or env.FREEBSD", raises=RuntimeError, - reason="See Issue #2847, PR #2999, PR #4324", + reason="See Issue #2847, PR #2999, PR #4324, PR #5925", strict=not env.PYPY, # PR 5569 ) def test_cross_module_exception_translator(): From 032e73d5632a99196a33ca6ae1ff7b8fdfe31cdf Mon Sep 17 00:00:00 2001 From: Yuanyuan Chen Date: Sat, 13 Dec 2025 13:42:36 +0800 Subject: [PATCH 06/30] Replace C-style casts to static_cast and reinterpret_cast (#5930) * Replace C-style casts to static_cast and reinterpret_cast Signed-off-by: cyy * style: pre-commit fixes --------- Signed-off-by: cyy Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/attr.h | 6 ++- include/pybind11/buffer_info.h | 7 +-- include/pybind11/cast.h | 5 +- include/pybind11/detail/argument_vector.h | 8 +-- include/pybind11/detail/class.h | 32 ++++++------ include/pybind11/detail/cpp_conduit.h | 4 +- .../detail/function_record_pyobject.h | 4 +- include/pybind11/detail/init.h | 4 +- include/pybind11/detail/internals.h | 5 +- include/pybind11/detail/type_caster_base.h | 12 ++--- include/pybind11/detail/value_and_holder.h | 6 ++- include/pybind11/pybind11.h | 51 ++++++++++--------- include/pybind11/pytypes.h | 33 +++++++----- 13 files changed, 95 insertions(+), 82 deletions(-) diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index f902c7c60..b4486dc0f 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -373,7 +373,7 @@ struct type_record { + (base_has_unique_ptr_holder ? "does not" : "does")); } - bases.append((PyObject *) base_info->type); + bases.append(reinterpret_cast(base_info->type)); #ifdef PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET dynamic_attr |= base_info->type->tp_dictoffset != 0; @@ -721,7 +721,9 @@ template ::value...)> constexpr bool expected_num_args(size_t nargs, bool has_args, bool has_kwargs) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(nargs, has_args, has_kwargs); - return named == 0 || (self + named + size_t(has_args) + size_t(has_kwargs)) == nargs; + return named == 0 + || (self + named + static_cast(has_args) + static_cast(has_kwargs)) + == nargs; } PYBIND11_NAMESPACE_END(detail) diff --git a/include/pybind11/buffer_info.h b/include/pybind11/buffer_info.h index 75aec0ba3..10fa825a8 100644 --- a/include/pybind11/buffer_info.h +++ b/include/pybind11/buffer_info.h @@ -66,10 +66,11 @@ struct buffer_info { bool readonly = false) : ptr(ptr), itemsize(itemsize), size(1), format(format), ndim(ndim), shape(std::move(shape_in)), strides(std::move(strides_in)), readonly(readonly) { - if (ndim != (ssize_t) shape.size() || ndim != (ssize_t) strides.size()) { + if (ndim != static_cast(shape.size()) + || ndim != static_cast(strides.size())) { pybind11_fail("buffer_info: ndim doesn't match shape and/or strides length"); } - for (size_t i = 0; i < (size_t) ndim; ++i) { + for (size_t i = 0; i < static_cast(ndim); ++i) { size *= shape[i]; } } @@ -195,7 +196,7 @@ struct compare_buffer_info { template struct compare_buffer_info::value>> { static bool compare(const buffer_info &b) { - return (size_t) b.itemsize == sizeof(T) + return static_cast(b.itemsize) == sizeof(T) && (b.format == format_descriptor::value || ((sizeof(T) == sizeof(long)) && b.format == (std::is_unsigned::value ? "L" : "l")) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 3acb560b0..310b77b34 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -394,7 +394,8 @@ public: } /* Check if this is a C++ type */ - const auto &bases = all_type_info((PyTypeObject *) type::handle_of(h).ptr()); + const auto &bases + = all_type_info(reinterpret_cast(type::handle_of(h).ptr())); if (bases.size() == 1) { // Only allowing loading from a single-value type value = values_and_holders(reinterpret_cast(h.ptr())).begin()->value_ptr(); return true; @@ -541,7 +542,7 @@ struct string_caster { const auto *buffer = reinterpret_cast(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); - size_t length = (size_t) PYBIND11_BYTES_SIZE(utfNbytes.ptr()) / sizeof(CharT); + size_t length = static_cast(PYBIND11_BYTES_SIZE(utfNbytes.ptr())) / sizeof(CharT); // Skip BOM for UTF-16/32 if (UTF_N > 8) { buffer++; diff --git a/include/pybind11/detail/argument_vector.h b/include/pybind11/detail/argument_vector.h index e9bfe064d..e15a3cfab 100644 --- a/include/pybind11/detail/argument_vector.h +++ b/include/pybind11/detail/argument_vector.h @@ -231,7 +231,7 @@ public: new (&m_repr.hvector) typename repr_type::heap_vector(count, value); } else { auto &inline_arr = m_repr.iarray; - inline_arr.arr.fill(value ? std::size_t(-1) : 0); + inline_arr.arr.fill(value ? static_cast(-1) : 0); inline_arr.size = static_cast(count); } } @@ -273,9 +273,9 @@ public: assert(wbi.word < kWords); assert(wbi.bit < kBitsPerWord); if (b) { - ha.arr[wbi.word] |= (std::size_t(1) << wbi.bit); + ha.arr[wbi.word] |= (static_cast(1) << wbi.bit); } else { - ha.arr[wbi.word] &= ~(std::size_t(1) << wbi.bit); + ha.arr[wbi.word] &= ~(static_cast(1) << wbi.bit); } assert(operator[](ha.size - 1) == b); } @@ -300,7 +300,7 @@ private: const auto wbi = word_and_bit_index(idx); assert(wbi.word < kWords); assert(wbi.bit < kBitsPerWord); - return m_repr.iarray.arr[wbi.word] & (std::size_t(1) << wbi.bit); + return m_repr.iarray.arr[wbi.word] & (static_cast(1) << wbi.bit); } PYBIND11_NOINLINE void move_to_heap_vector_with_reserved_size(std::size_t reserved_size) { diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 7fe692856..21e966cfe 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -71,7 +71,7 @@ inline PyTypeObject *make_static_property_type() { issue no Python C API calls which could potentially invoke the garbage collector (the GC will call type_traverse(), which will in turn find the newly constructed type in an invalid state) */ - auto *heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); + auto *heap_type = reinterpret_cast(PyType_Type.tp_alloc(&PyType_Type, 0)); if (!heap_type) { pybind11_fail("make_static_property_type(): error allocating type!"); } @@ -98,7 +98,7 @@ inline PyTypeObject *make_static_property_type() { pybind11_fail("make_static_property_type(): failure in PyType_Ready()!"); } - setattr((PyObject *) type, "__module__", str(PYBIND11_DUMMY_MODULE_NAME)); + setattr(reinterpret_cast(type), "__module__", str(PYBIND11_DUMMY_MODULE_NAME)); PYBIND11_SET_OLDPY_QUALNAME(type, name_obj); return type; @@ -265,7 +265,7 @@ inline PyTypeObject *make_default_metaclass() { issue no Python C API calls which could potentially invoke the garbage collector (the GC will call type_traverse(), which will in turn find the newly constructed type in an invalid state) */ - auto *heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); + auto *heap_type = reinterpret_cast(PyType_Type.tp_alloc(&PyType_Type, 0)); if (!heap_type) { pybind11_fail("make_default_metaclass(): error allocating metaclass!"); } @@ -291,7 +291,7 @@ inline PyTypeObject *make_default_metaclass() { pybind11_fail("make_default_metaclass(): failure in PyType_Ready()!"); } - setattr((PyObject *) type, "__module__", str(PYBIND11_DUMMY_MODULE_NAME)); + setattr(reinterpret_cast(type), "__module__", str(PYBIND11_DUMMY_MODULE_NAME)); PYBIND11_SET_OLDPY_QUALNAME(type, name_obj); return type; @@ -306,7 +306,7 @@ inline void traverse_offset_bases(void *valueptr, instance *self, bool (*f)(void * /*parentptr*/, instance * /*self*/)) { for (handle h : reinterpret_borrow(tinfo->type->tp_bases)) { - if (auto *parent_tinfo = get_type_info((PyTypeObject *) h.ptr())) { + if (auto *parent_tinfo = get_type_info(reinterpret_cast(h.ptr()))) { for (auto &c : parent_tinfo->implicit_casts) { if (c.first == tinfo->cpptype) { auto *parentptr = c.second(valueptr); @@ -530,7 +530,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass) { issue no Python C API calls which could potentially invoke the garbage collector (the GC will call type_traverse(), which will in turn find the newly constructed type in an invalid state) */ - auto *heap_type = (PyHeapTypeObject *) metaclass->tp_alloc(metaclass, 0); + auto *heap_type = reinterpret_cast(metaclass->tp_alloc(metaclass, 0)); if (!heap_type) { pybind11_fail("make_object_base_type(): error allocating type!"); } @@ -557,11 +557,11 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass) { pybind11_fail("PyType_Ready failed in make_object_base_type(): " + error_string()); } - setattr((PyObject *) type, "__module__", str(PYBIND11_DUMMY_MODULE_NAME)); + setattr(reinterpret_cast(type), "__module__", str(PYBIND11_DUMMY_MODULE_NAME)); PYBIND11_SET_OLDPY_QUALNAME(type, name_obj); assert(!PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC)); - return (PyObject *) heap_type; + return reinterpret_cast(heap_type); } /// dynamic_attr: Allow the garbage collector to traverse the internal instance `__dict__`. @@ -746,7 +746,7 @@ inline PyObject *make_new_python_type(const type_record &rec) { /* Allocate memory for docstring (Python will free this later on) */ size_t size = std::strlen(rec.doc) + 1; #if PY_VERSION_HEX >= 0x030D0000 - tp_doc = (char *) PyMem_MALLOC(size); + tp_doc = static_cast(PyMem_MALLOC(size)); #else tp_doc = (char *) PyObject_MALLOC(size); #endif @@ -761,10 +761,10 @@ inline PyObject *make_new_python_type(const type_record &rec) { issue no Python C API calls which could potentially invoke the garbage collector (the GC will call type_traverse(), which will in turn find the newly constructed type in an invalid state) */ - auto *metaclass - = rec.metaclass.ptr() ? (PyTypeObject *) rec.metaclass.ptr() : internals.default_metaclass; + auto *metaclass = rec.metaclass.ptr() ? reinterpret_cast(rec.metaclass.ptr()) + : internals.default_metaclass; - auto *heap_type = (PyHeapTypeObject *) metaclass->tp_alloc(metaclass, 0); + auto *heap_type = reinterpret_cast(metaclass->tp_alloc(metaclass, 0)); if (!heap_type) { pybind11_fail(std::string(rec.name) + ": Unable to create type object!"); } @@ -777,7 +777,7 @@ inline PyObject *make_new_python_type(const type_record &rec) { auto *type = &heap_type->ht_type; type->tp_name = full_name; type->tp_doc = tp_doc; - type->tp_base = type_incref((PyTypeObject *) base); + type->tp_base = type_incref(reinterpret_cast(base)); type->tp_basicsize = static_cast(sizeof(instance)); if (!bases.empty()) { type->tp_bases = bases.release().ptr(); @@ -818,18 +818,18 @@ inline PyObject *make_new_python_type(const type_record &rec) { /* Register type with the parent scope */ if (rec.scope) { - setattr(rec.scope, rec.name, (PyObject *) type); + setattr(rec.scope, rec.name, reinterpret_cast(type)); } else { Py_INCREF(type); // Keep it alive forever (reference leak) } if (module_) { // Needed by pydoc - setattr((PyObject *) type, "__module__", module_); + setattr(reinterpret_cast(type), "__module__", module_); } PYBIND11_SET_OLDPY_QUALNAME(type, qualname); - return (PyObject *) type; + return reinterpret_cast(type); } PYBIND11_NAMESPACE_END(detail) diff --git a/include/pybind11/detail/cpp_conduit.h b/include/pybind11/detail/cpp_conduit.h index a06b9b21a..49c199e14 100644 --- a/include/pybind11/detail/cpp_conduit.h +++ b/include/pybind11/detail/cpp_conduit.h @@ -21,13 +21,13 @@ inline bool type_is_managed_by_our_internals(PyTypeObject *type_obj) { return bool(internals.registered_types_py.find(type_obj) != internals.registered_types_py.end()); #else - return bool(type_obj->tp_new == pybind11_object_new); + return (type_obj->tp_new == pybind11_object_new); #endif } inline bool is_instance_method_of_type(PyTypeObject *type_obj, PyObject *attr_name) { PyObject *descr = _PyType_Lookup(type_obj, attr_name); - return bool((descr != nullptr) && PyInstanceMethod_Check(descr)); + return ((descr != nullptr) && PyInstanceMethod_Check(descr)); } inline object try_get_cpp_conduit_method(PyObject *obj) { diff --git a/include/pybind11/detail/function_record_pyobject.h b/include/pybind11/detail/function_record_pyobject.h index 694625f89..94d27ad17 100644 --- a/include/pybind11/detail/function_record_pyobject.h +++ b/include/pybind11/detail/function_record_pyobject.h @@ -126,7 +126,7 @@ inline bool is_function_record_PyObject(PyObject *obj) { inline function_record *function_record_ptr_from_PyObject(PyObject *obj) { if (is_function_record_PyObject(obj)) { - return ((detail::function_record_PyObject *) obj)->cpp_func_rec; + return (reinterpret_cast(obj))->cpp_func_rec; } return nullptr; } @@ -137,7 +137,7 @@ inline object function_record_PyObject_New() { throw error_already_set(); } py_func_rec->cpp_func_rec = nullptr; // For clarity/purity. Redundant in practice. - return reinterpret_steal((PyObject *) py_func_rec); + return reinterpret_steal(reinterpret_cast(py_func_rec)); } PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods) diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index d7c84cb84..b7f8d5a52 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -476,9 +476,9 @@ void setstate(value_and_holder &v_h, std::pair &&result, bool need_alias) return; } // Our tests never run into an unset dict, but being careful here for now (see #5658) - auto dict = getattr((PyObject *) v_h.inst, "__dict__", none()); + auto dict = getattr(reinterpret_cast(v_h.inst), "__dict__", none()); if (dict.is_none()) { - setattr((PyObject *) v_h.inst, "__dict__", d); + setattr(reinterpret_cast(v_h.inst), "__dict__", d); } else { // Keep the original object dict and just update it if (PyDict_Update(dict.ptr(), d.ptr()) < 0) { diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 4d6c147db..858de6752 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -606,9 +606,8 @@ public: // this could be called without an active interpreter, just use what was cached if (!tstate || tstate->interp == last_istate_tls()) { auto tpp = internals_p_tls(); - if (tpp) { - delete tpp; - } + + delete tpp; } unref(); return; diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index c6b80734b..b0c59e113 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -125,7 +125,7 @@ PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector check; for (handle parent : reinterpret_borrow(t->tp_bases)) { - check.push_back((PyTypeObject *) parent.ptr()); + check.push_back(reinterpret_cast(parent.ptr())); } auto const &type_dict = get_internals().registered_types_py; for (size_t i = 0; i < check.size(); i++) { @@ -168,7 +168,7 @@ PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector(type->tp_bases)) { - check.push_back((PyTypeObject *) parent.ptr()); + check.push_back(reinterpret_cast(parent.ptr())); } } } @@ -286,7 +286,7 @@ PYBIND11_NOINLINE detail::type_info *get_type_info(const std::type_info &tp, PYBIND11_NOINLINE handle get_type_handle(const std::type_info &tp, bool throw_if_missing) { detail::type_info *type_info = get_type_info(tp, throw_if_missing); - return handle(type_info ? ((PyObject *) type_info->type) : nullptr); + return handle(type_info ? (reinterpret_cast(type_info->type)) : nullptr); } inline bool try_incref(PyObject *obj) { @@ -506,7 +506,7 @@ PYBIND11_NOINLINE void instance::allocate_layout() { // efficient for small allocations like the one we're doing here; // for larger allocations they are just wrappers around malloc. // TODO: is this still true for pure Python 3.6? - nonsimple.values_and_holders = (void **) PyMem_Calloc(space, sizeof(void *)); + nonsimple.values_and_holders = static_cast(PyMem_Calloc(space, sizeof(void *))); if (!nonsimple.values_and_holders) { throw std::bad_alloc(); } @@ -537,7 +537,7 @@ PYBIND11_NOINLINE handle get_object_handle(const void *ptr, const detail::type_i for (auto it = range.first; it != range.second; ++it) { for (const auto &vh : values_and_holders(it->second)) { if (vh.type == type) { - return handle((PyObject *) it->second); + return handle(reinterpret_cast(it->second)); } } } @@ -1700,7 +1700,7 @@ inline std::string quote_cpp_type_name(const std::string &cpp_type_name) { PYBIND11_NOINLINE std::string type_info_description(const std::type_info &ti) { if (auto *type_data = get_type_info(ti)) { - handle th((PyObject *) type_data->type); + handle th(reinterpret_cast(type_data->type)); return th.attr("__module__").cast() + '.' + th.attr("__qualname__").cast(); } diff --git a/include/pybind11/detail/value_and_holder.h b/include/pybind11/detail/value_and_holder.h index 87c92f8e4..b24551e67 100644 --- a/include/pybind11/detail/value_and_holder.h +++ b/include/pybind11/detail/value_and_holder.h @@ -54,7 +54,8 @@ struct value_and_holder { } else if (v) { inst->nonsimple.status[index] |= instance::status_holder_constructed; } else { - inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_holder_constructed; + inst->nonsimple.status[index] + &= static_cast(~instance::status_holder_constructed); } } bool instance_registered() const { @@ -69,7 +70,8 @@ struct value_and_holder { } else if (v) { inst->nonsimple.status[index] |= instance::status_instance_registered; } else { - inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_instance_registered; + inst->nonsimple.status[index] + &= static_cast(~instance::status_instance_registered); } } }; diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index a2043b264..91b38d91e 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -159,7 +159,7 @@ inline std::string generate_function_signature(const char *type_caster_name_fiel pybind11_fail("Internal error while parsing type signature (1)"); } if (auto *tinfo = detail::get_type_info(*t)) { - handle th((PyObject *) tinfo->type); + handle th(reinterpret_cast(tinfo->type)); signature += th.attr("__module__").cast() + "." + th.attr("__qualname__").cast(); } else if (auto th = detail::global_internals_native_enum_type_map_get_item(*t)) { @@ -642,7 +642,7 @@ protected: rec->signature = guarded_strdup(signature.c_str()); rec->args.shrink_to_fit(); - rec->nargs = (std::uint16_t) args; + rec->nargs = static_cast(args); if (rec->sibling && PYBIND11_INSTANCE_METHOD_CHECK(rec->sibling.ptr())) { rec->sibling = PYBIND11_INSTANCE_METHOD_GET_FUNCTION(rec->sibling.ptr()); @@ -681,7 +681,7 @@ protected: rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS; object py_func_rec = detail::function_record_PyObject_New(); - ((detail::function_record_PyObject *) py_func_rec.ptr())->cpp_func_rec + (reinterpret_cast(py_func_rec.ptr()))->cpp_func_rec = unique_rec.release(); guarded_strdup.release(); @@ -715,8 +715,8 @@ protected: // chain. chain_start = rec; rec->next = chain; - auto *py_func_rec - = (detail::function_record_PyObject *) PyCFunction_GET_SELF(m_ptr); + auto *py_func_rec = reinterpret_cast( + PyCFunction_GET_SELF(m_ptr)); py_func_rec->cpp_func_rec = unique_rec.release(); guarded_strdup.release(); } else { @@ -776,7 +776,7 @@ protected: } } - auto *func = (PyCFunctionObject *) m_ptr; + auto *func = reinterpret_cast(m_ptr); // Install docstring if it's non-empty (when at least one option is enabled) auto *doc = signatures.empty() ? nullptr : PYBIND11_COMPAT_STRDUP(signatures.c_str()); std::free(const_cast(PYBIND11_PYCFUNCTION_GET_DOC(func))); @@ -851,7 +851,7 @@ protected: /* Need to know how many arguments + keyword arguments there are to pick the right overload */ - const auto n_args_in = (size_t) PyTuple_GET_SIZE(args_in); + const auto n_args_in = static_cast(PyTuple_GET_SIZE(args_in)); handle parent = n_args_in > 0 ? PyTuple_GET_ITEM(args_in, 0) : nullptr, result = PYBIND11_TRY_NEXT_OVERLOAD; @@ -865,7 +865,8 @@ protected: return nullptr; } - auto *const tinfo = get_type_info((PyTypeObject *) overloads->scope.ptr()); + auto *const tinfo + = get_type_info(reinterpret_cast(overloads->scope.ptr())); auto *const pi = reinterpret_cast(parent.ptr()); self_value_and_holder = pi->get_value_and_holder(tinfo, true); @@ -1296,7 +1297,7 @@ PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods) // This implementation needs the definition of `class cpp_function`. inline void tp_dealloc_impl(PyObject *self) { - auto *py_func_rec = (function_record_PyObject *) self; + auto *py_func_rec = reinterpret_cast(self); cpp_function::destruct(py_func_rec->cpp_func_rec); py_func_rec->cpp_func_rec = nullptr; } @@ -1669,7 +1670,7 @@ protected: /* Register supplemental type information in C++ dict */ auto *tinfo = new detail::type_info(); - tinfo->type = (PyTypeObject *) m_ptr; + tinfo->type = reinterpret_cast(m_ptr); tinfo->cpptype = rec.type; tinfo->type_size = rec.type_size; tinfo->type_align = rec.type_align; @@ -1704,7 +1705,7 @@ protected: PYBIND11_WARNING_DISABLE_GCC("-Warray-bounds") PYBIND11_WARNING_DISABLE_GCC("-Wstringop-overread") #endif - internals.registered_types_py[(PyTypeObject *) m_ptr] = {tinfo}; + internals.registered_types_py[reinterpret_cast(m_ptr)] = {tinfo}; PYBIND11_WARNING_POP }); @@ -1712,7 +1713,8 @@ protected: mark_parents_nonsimple(tinfo->type); tinfo->simple_ancestors = false; } else if (rec.bases.size() == 1) { - auto *parent_tinfo = get_type_info((PyTypeObject *) rec.bases[0].ptr()); + auto *parent_tinfo + = get_type_info(reinterpret_cast(rec.bases[0].ptr())); assert(parent_tinfo != nullptr); bool parent_simple_ancestors = parent_tinfo->simple_ancestors; tinfo->simple_ancestors = parent_simple_ancestors; @@ -1731,17 +1733,17 @@ protected: void mark_parents_nonsimple(PyTypeObject *value) { auto t = reinterpret_borrow(value->tp_bases); for (handle h : t) { - auto *tinfo2 = get_type_info((PyTypeObject *) h.ptr()); + auto *tinfo2 = get_type_info(reinterpret_cast(h.ptr())); if (tinfo2) { tinfo2->simple_type = false; } - mark_parents_nonsimple((PyTypeObject *) h.ptr()); + mark_parents_nonsimple(reinterpret_cast(h.ptr())); } } void install_buffer_funcs(buffer_info *(*get_buffer)(PyObject *, void *), void *get_buffer_data) { - auto *type = (PyHeapTypeObject *) m_ptr; + auto *type = reinterpret_cast(m_ptr); auto *tinfo = detail::get_type_info(&type->ht_type); if (!type->ht_type.tp_as_buffer) { @@ -1763,8 +1765,8 @@ protected: const auto is_static = (rec_func != nullptr) && !(rec_func->is_method && rec_func->scope); const auto has_doc = (rec_func != nullptr) && (rec_func->doc != nullptr) && pybind11::options::show_user_defined_docstrings(); - auto property = handle( - (PyObject *) (is_static ? get_internals().static_property_type : &PyProperty_Type)); + auto property = handle(reinterpret_cast( + is_static ? get_internals().static_property_type : &PyProperty_Type)); attr(name) = property(fget.ptr() ? fget : none(), fset.ptr() ? fset : none(), /*deleter*/ none(), @@ -2698,8 +2700,9 @@ struct enum_base { PYBIND11_NOINLINE void init(bool is_arithmetic, bool is_convertible) { m_base.attr("__entries") = dict(); - auto property = handle((PyObject *) &PyProperty_Type); - auto static_property = handle((PyObject *) get_internals().static_property_type); + auto property = handle(reinterpret_cast(&PyProperty_Type)); + auto static_property + = handle(reinterpret_cast(get_internals().static_property_type)); m_base.attr("__repr__") = cpp_function( [](const object &arg) -> str { @@ -2730,7 +2733,7 @@ struct enum_base { [](handle arg) -> std::string { std::string docstring; dict entries = arg.attr("__entries"); - if (((PyTypeObject *) arg.ptr())->tp_doc) { + if ((reinterpret_cast(arg.ptr()))->tp_doc) { docstring += std::string( reinterpret_cast(arg.ptr())->tp_doc); docstring += "\n\n"; @@ -2856,7 +2859,7 @@ struct enum_base { dict entries = m_base.attr("__entries"); str name(name_); if (entries.contains(name)) { - std::string type_name = (std::string) str(m_base.attr("__name__")); + std::string type_name = std::string(str(m_base.attr("__name__"))); throw value_error(std::move(type_name) + ": element \"" + std::string(name_) + "\" already exists!"); } @@ -3051,7 +3054,7 @@ all_type_info_get_cache(PyTypeObject *type) { if (res.second) { // New cache entry created; set up a weak reference to automatically remove it if the type // gets destroyed: - weakref((PyObject *) type, cpp_function([type](handle wr) { + weakref(reinterpret_cast(type), cpp_function([type](handle wr) { with_internals([type](internals &internals) { internals.registered_types_py.erase(type); @@ -3292,7 +3295,7 @@ void implicitly_convertible() { } tuple args(1); args[0] = obj; - PyObject *result = PyObject_Call((PyObject *) type, args.ptr(), nullptr); + PyObject *result = PyObject_Call(reinterpret_cast(type), args.ptr(), nullptr); if (result == nullptr) { PyErr_Clear(); } @@ -3508,7 +3511,7 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char * if (frame != nullptr) { PyCodeObject *f_code = PyFrame_GetCode(frame); // f_code is guaranteed to not be NULL - if ((std::string) str(f_code->co_name) == name && f_code->co_argcount > 0) { + if (std::string(str(f_code->co_name)) == name && f_code->co_argcount > 0) { # if PY_VERSION_HEX >= 0x030d0000 PyObject *locals = PyEval_GetFrameLocals(); # else diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index b28692fd7..538ee1c75 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -547,7 +547,7 @@ struct error_fetch_and_normalize { // The presence of __notes__ is likely due to exception normalization // errors, although that is not necessarily true, therefore insert a // hint only: - if (PyObject_HasAttrString(m_value.ptr(), "__notes__")) { + if (PyObject_HasAttrString(m_value.ptr(), "__notes__") != 0) { m_lazy_error_string += "[WITH __notes__]"; } #else @@ -1563,7 +1563,9 @@ public: PYBIND11_OBJECT(type, object, PyType_Check) /// Return a type handle from a handle or an object - static handle handle_of(handle h) { return handle((PyObject *) Py_TYPE(h.ptr())); } + static handle handle_of(handle h) { + return handle(reinterpret_cast(Py_TYPE(h.ptr()))); + } /// Return a type object from a handle or an object static type of(handle h) { return type(type::handle_of(h), borrowed_t{}); } @@ -1661,7 +1663,7 @@ public: if (PyBytes_AsStringAndSize(temp.ptr(), &buffer, &length) != 0) { throw error_already_set(); } - return std::string(buffer, (size_t) length); + return std::string(buffer, static_cast(length)); } template @@ -1861,10 +1863,12 @@ template Unsigned as_unsigned(PyObject *o) { if (sizeof(Unsigned) <= sizeof(unsigned long)) { unsigned long v = PyLong_AsUnsignedLong(o); - return v == (unsigned long) -1 && PyErr_Occurred() ? (Unsigned) -1 : (Unsigned) v; + return v == static_cast(-1) && PyErr_Occurred() ? (Unsigned) -1 + : (Unsigned) v; } unsigned long long v = PyLong_AsUnsignedLongLong(o); - return v == (unsigned long long) -1 && PyErr_Occurred() ? (Unsigned) -1 : (Unsigned) v; + return v == static_cast(-1) && PyErr_Occurred() ? (Unsigned) -1 + : (Unsigned) v; } PYBIND11_NAMESPACE_END(detail) @@ -1908,7 +1912,7 @@ public: PYBIND11_OBJECT_CVT(float_, object, PyFloat_Check, PyNumber_Float) // Allow implicit conversion from float/double: // NOLINTNEXTLINE(google-explicit-constructor) - float_(float value) : object(PyFloat_FromDouble((double) value), stolen_t{}) { + float_(float value) : object(PyFloat_FromDouble(static_cast(value)), stolen_t{}) { if (!m_ptr) { pybind11_fail("Could not allocate float object!"); } @@ -1920,7 +1924,7 @@ public: } } // NOLINTNEXTLINE(google-explicit-constructor) - operator float() const { return (float) PyFloat_AsDouble(m_ptr); } + operator float() const { return static_cast(PyFloat_AsDouble(m_ptr)); } // NOLINTNEXTLINE(google-explicit-constructor) operator double() const { return PyFloat_AsDouble(m_ptr); } }; @@ -2122,7 +2126,7 @@ public: pybind11_fail("Could not allocate tuple object!"); } } - size_t size() const { return (size_t) PyTuple_Size(m_ptr); } + size_t size() const { return static_cast(PyTuple_Size(m_ptr)); } bool empty() const { return size() == 0; } detail::tuple_accessor operator[](size_t index) const { return {*this, index}; } template ::value, int> = 0> @@ -2156,7 +2160,7 @@ public: typename collector = detail::deferred_t, Args...>> explicit dict(Args &&...args) : dict(collector(std::forward(args)...).kwargs()) {} - size_t size() const { return (size_t) PyDict_Size(m_ptr); } + size_t size() const { return static_cast(PyDict_Size(m_ptr)); } bool empty() const { return size() == 0; } detail::dict_iterator begin() const { return {*this, 0}; } detail::dict_iterator end() const { return {}; } @@ -2176,7 +2180,8 @@ private: if (PyDict_Check(op)) { return handle(op).inc_ref().ptr(); } - return PyObject_CallFunctionObjArgs((PyObject *) &PyDict_Type, op, nullptr); + return PyObject_CallFunctionObjArgs( + reinterpret_cast(&PyDict_Type), op, nullptr); } }; @@ -2188,7 +2193,7 @@ public: if (result == -1) { throw error_already_set(); } - return (size_t) result; + return static_cast(result); } bool empty() const { return size() == 0; } detail::sequence_accessor operator[](size_t index) const { return {*this, index}; } @@ -2211,7 +2216,7 @@ public: pybind11_fail("Could not allocate list object!"); } } - size_t size() const { return (size_t) PyList_Size(m_ptr); } + size_t size() const { return static_cast(PyList_Size(m_ptr)); } bool empty() const { return size() == 0; } detail::list_accessor operator[](size_t index) const { return {*this, index}; } template ::value, int> = 0> @@ -2497,7 +2502,7 @@ inline size_t len(handle h) { if (result < 0) { throw error_already_set(); } - return (size_t) result; + return static_cast(result); } /// Get the length hint of a Python object. @@ -2510,7 +2515,7 @@ inline size_t len_hint(handle h) { PyErr_Clear(); return 0; } - return (size_t) result; + return static_cast(result); } inline str repr(handle h) { From 41a4d0c4b689a57f0829d7cca5fe0db9ad65421b Mon Sep 17 00:00:00 2001 From: Kyle Shores Date: Sat, 13 Dec 2025 01:57:05 -0600 Subject: [PATCH 07/30] Add Windows arm tests (#5932) * adding windows arm test - excluding numpy 2.2.0 for arm64 builds * adding windows arm msys2 test * testing mingw python * unnamed namespace test on windows arm with clang and mingw * Revert "unnamed namespace test on windows arm with clang and mingw" This reverts commit 08abf889ae4422b86d92302597423cdb4c7e7896. * bumping c++ version * commenting out other tests * Ignore unnmaed namespace on arm windows with mingw - Updatig XFAIL condition to expect a failure on windows arm with mingw and clang - setting python home and path variables in c++ tests * Revert "commenting out other tests" This reverts commit dc75243963f96295c2ce9e2a5180cd6ac384a4ed. * removing windows-11-arm from big test, push * removing redundant shell * removing trailing whitespace * Clarify Windows ARM clang job naming Rename the Windows ARM clang jobs to windows_arm_clang_msvc and windows_arm_clang_msys2 and adjust their display names to clang-msvc / clang-msys2. The first runs clang against the MSVC/Windows SDK toolchain and python.org CPython for Windows ARM, while the second runs clang inside the MSYS2/MinGW-w64 CLANGARM64 environment with MSYS2 Python. Using clearly distinguished job names makes it easier to discuss failures and behavior in each environment without ambiguity, both in logs and in PR review discussions. * Reduce Windows ARM clang matrix size Limit the windows_arm_clang_msvc job to Python 3.13 and the windows_arm_clang_msys2 job to Python 3.12 to stay within our constrained GitHub Actions resources. Keep both jobs using a matrix over os and python so their structure stays aligned and it remains easy to expand coverage when needed. * Remove matrix.python from windows_arm_clang_msys2 job: the Python version is determined by the msys2/setup-msys2 action and cannot be changed --------- Co-authored-by: Ralf W. Grosse-Kunstleve --- .github/workflows/ci.yml | 131 ++++++++++++++++++++++++++++++ tests/requirements.txt | 5 +- tests/test_unnamed_namespace_a.py | 5 +- 3 files changed, 138 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f2b688a9..d00ee6d27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1205,3 +1205,134 @@ jobs: - name: Clean directory run: git clean -fdx + + # Clang with MSVC/Windows SDK toolchain + python.org CPython (Windows ARM) + windows_arm_clang_msvc: + if: github.event.pull_request.draft == false + + strategy: + fail-fast: false + matrix: + os: [windows-11-arm] + python: ['3.13'] + + runs-on: "${{ matrix.os }}" + timeout-minutes: 90 + + name: "🐍 ${{ matrix.python }} • ${{ matrix.os }} • clang-msvc" + + steps: + - name: Show env + run: env + + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python }} + architecture: arm64 + + - name: Run pip installs + run: | + python -m pip install --upgrade pip + python -m pip install -r tests/requirements.txt + + - name: Configure CMake + run: > + cmake -G Ninja -S . -B . + -DPYBIND11_WERROR=OFF + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=20 + -DPython_ROOT_DIR="$env:Python_ROOT_DIR" + + - name: Build + run: cmake --build . -j 2 + + - name: Python tests + run: cmake --build . --target pytest -j 2 + + - name: C++ tests + run: cmake --build . --target cpptest -j 2 + + - name: Interface test + run: cmake --build . --target test_cmake_build -j 2 + + - name: Visibility test + run: cmake --build . --target test_cross_module_rtti -j 2 + + # Clang in MSYS2/MinGW-w64 CLANGARM64 toolchain + MSYS2 Python (Windows ARM) + windows_arm_clang_msys2: + if: github.event.pull_request.draft == false + + strategy: + fail-fast: false + matrix: + os: [windows-11-arm] + + runs-on: "${{ matrix.os }}" + timeout-minutes: 90 + + name: "${{ matrix.os }} • clang-msys2" + + defaults: + run: + shell: msys2 {0} + + steps: + - uses: actions/checkout@v6 + with: + submodules: true + + - uses: msys2/setup-msys2@v2 + with: + msystem: CLANGARM64 + update: true + install: | + mingw-w64-clang-aarch64-cmake + mingw-w64-clang-aarch64-clang + mingw-w64-clang-aarch64-ninja + mingw-w64-clang-aarch64-python-pip + mingw-w64-clang-aarch64-python-pytest + mingw-w64-clang-aarch64-python-numpy + + - name: Debug info + run: | + clang++ --version + cmake --version + ninja --version + python --version + + - name: Run pip installs + run: | + python -m pip install --upgrade pip + python -m pip install -r tests/requirements.txt + + - name: Configure CMake + run: >- + cmake -S . -B build + -DPYBIND11_WERROR=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=20 + -DPYTHON_EXECUTABLE=$(python -c "import sys; print(sys.executable)") + + - name: Build + run: cmake --build build -j 2 + + - name: Python tests + run: cmake --build build --target pytest -j 2 + + - name: C++ tests + run: PYTHONHOME=/clangarm64 PYTHONPATH=/clangarm64 cmake --build build --target cpptest -j 2 + + - name: Interface test + run: cmake --build build --target test_cmake_build -j 2 + + - name: Visibility test + run: cmake --build build --target test_cross_module_rtti -j 2 diff --git a/tests/requirements.txt b/tests/requirements.txt index 6e3a260b1..41fc9f143 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -7,8 +7,9 @@ numpy~=2.2.0; python_version=="3.10" and platform_python_implementation=="PyPy" numpy~=1.26.0; platform_python_implementation=="GraalVM" and sys_platform=="linux" numpy~=1.21.5; platform_python_implementation=="CPython" and python_version>="3.8" and python_version<"3.10" numpy~=1.22.2; platform_python_implementation=="CPython" and python_version=="3.10" -numpy~=1.26.0; platform_python_implementation=="CPython" and python_version>="3.11" and python_version<"3.13" -numpy~=2.2.0; platform_python_implementation=="CPython" and python_version=="3.13" +numpy~=1.26.0; platform_python_implementation=="CPython" and python_version>="3.11" and python_version<"3.13" and platform_machine!="ARM64" +numpy~=2.3.0; platform_python_implementation=="CPython" and python_version>="3.11" and platform_machine=="ARM64" +numpy~=2.2.0; platform_python_implementation=="CPython" and python_version=="3.13" and platform_machine!="ARM64" pytest>=6 pytest-timeout scipy~=1.5.4; platform_python_implementation=="CPython" and python_version<"3.10" diff --git a/tests/test_unnamed_namespace_a.py b/tests/test_unnamed_namespace_a.py index fabf1312a..514a81272 100644 --- a/tests/test_unnamed_namespace_a.py +++ b/tests/test_unnamed_namespace_a.py @@ -5,7 +5,10 @@ import pytest from pybind11_tests import unnamed_namespace_a as m from pybind11_tests import unnamed_namespace_b as mb -XFAIL_CONDITION = "not m.defined_WIN32_or__WIN32 and (m.defined___clang__ or m.defined__LIBCPP_VERSION)" +XFAIL_CONDITION = ( + "m.defined__LIBCPP_VERSION or " + "(not m.defined_WIN32_or__WIN32 and m.defined___clang__)" +) XFAIL_REASON = "Known issues: https://github.com/pybind/pybind11/pull/4319" From 1006933415f13fdae2509169dfc62837e3018f6f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 13 Dec 2025 05:17:08 -0500 Subject: [PATCH 08/30] chore: also use typos (#5931) Signed-off-by: Henry Schreiner Co-authored-by: Ralf W. Grosse-Kunstleve --- .github/CONTRIBUTING.md | 2 +- .pre-commit-config.yaml | 9 ++++++++- docs/advanced/embedding.rst | 2 +- docs/changelog.md | 4 ++-- pyproject.toml | 23 +++++++++++++++++++++++ tests/conftest.py | 6 +++--- tests/test_docstring_options.py | 2 +- 7 files changed, 39 insertions(+), 9 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index fbdf075f0..a09b630a2 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -165,7 +165,7 @@ The valid options are: * Use `cmake --build build -j12` to build with 12 cores (for example). * Use `-G` and the name of a generator to use something different. `cmake --help` lists the generators available. - - On Unix, setting `CMAKE_GENERATER=Ninja` in your environment will give + - On Unix, setting `CMAKE_GENERATOR=Ninja` in your environment will give you automatic multithreading on all your CMake projects! * Open the `CMakeLists.txt` with QtCreator to generate for that IDE. * You can use `-DCMAKE_EXPORT_COMPILE_COMMANDS=ON` to generate the `.json` file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ba6f3829a..aa7b58cc4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -115,9 +115,16 @@ repos: rev: "v2.4.1" hooks: - id: codespell - exclude: ".supp$" + exclude: "(.supp|^pyproject.toml)$" args: ["-x.codespell-ignore-lines", "-Lccompiler,intstruct"] +# Also check spelling +- repo: https://github.com/crate-ci/typos + rev: v1.40.0 + hooks: + - id: typos + args: [] + # Check for common shell mistakes - repo: https://github.com/shellcheck-py/shellcheck-py rev: "v0.11.0.1" diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index 3ac057938..7d770faa2 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -302,7 +302,7 @@ Activating a Sub-interpreter Once a sub-interpreter is created, you can "activate" it on a thread (and acquire its GIL) by creating a :class:`subinterpreter_scoped_activate` -instance and passing it the sub-intepreter to be activated. The function +instance and passing it the sub-interpreter to be activated. The function will acquire the sub-interpreter's GIL and make the sub-interpreter the current active interpreter on the current thread for the lifetime of the instance. When the :class:`subinterpreter_scoped_activate` instance goes out diff --git a/docs/changelog.md b/docs/changelog.md index c8d631879..4bf23a697 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -167,7 +167,7 @@ New Features: [#5665](https://github.com/pybind/pybind11/pull/5665) and consolidate code [#5670](https://github.com/pybind/pybind11/pull/5670). -- Added API in `pybind11/subinterpreter.h` for embedding sub-intepreters (requires Python 3.12+). +- Added API in `pybind11/subinterpreter.h` for embedding sub-interpreters (requires Python 3.12+). [#5666](https://github.com/pybind/pybind11/pull/5666) - `py::native_enum` was added, for conversions between Python's native @@ -1213,7 +1213,7 @@ Performance and style: - Optimize Eigen sparse matrix casting by removing unnecessary temporary. [#4064](https://github.com/pybind/pybind11/pull/4064) - Avoid potential implicit copy/assignment constructors causing double - free in `strdup_gaurd`. + free in `strdup_guard`. [#3905](https://github.com/pybind/pybind11/pull/3905) - Enable clang-tidy checks `misc-definitions-in-headers`, `modernize-loop-convert`, and `modernize-use-nullptr`. diff --git a/pyproject.toml b/pyproject.toml index 274de222a..7a12eed1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,3 +184,26 @@ isort.required-imports = ["from __future__ import annotations"] [tool.repo-review] ignore = ["PP"] + +[tool.typos] +files.extend-exclude = ["/cpython"] + +[tool.typos.default.extend-identifiers] +ser_no = "ser_no" +SerNo = "SerNo" +StrLits = "StrLits" + +[tool.typos.default.extend-words] +nd = "nd" +valu = "valu" +fo = "fo" +quater = "quater" +optin = "optin" +othr = "othr" + +#[tool.typos.type.cpp.extend-words] +setp = "setp" +ot = "ot" + +[tool.typos.type.json.extend-words] +ba = "ba" diff --git a/tests/conftest.py b/tests/conftest.py index 39de4e138..9d9815b88 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -304,10 +304,10 @@ def backport_typehints() -> Callable[[SanitizedString], SanitizedString]: if sys.version_info < (3, 10): d["typing_extensions.TypeGuard"] = "typing.TypeGuard" - def backport(sanatized_string: SanitizedString) -> SanitizedString: + def backport(sanitized_string: SanitizedString) -> SanitizedString: for old, new in d.items(): - sanatized_string.string = sanatized_string.string.replace(old, new) + sanitized_string.string = sanitized_string.string.replace(old, new) - return sanatized_string + return sanitized_string return backport diff --git a/tests/test_docstring_options.py b/tests/test_docstring_options.py index 802a1ec9e..ffd1a739d 100644 --- a/tests/test_docstring_options.py +++ b/tests/test_docstring_options.py @@ -48,7 +48,7 @@ def test_docstring_options(): assert not m.DocstringTestFoo.__doc__ assert not m.DocstringTestFoo.value_prop.__doc__ - # Check existig behaviour of enum docstings + # Check existing behaviour of enum docstings assert ( m.DocstringTestEnum1.__doc__ == "Enum docstring\n\nMembers:\n\n Member1\n\n Member2" From 5b379161aa3fc861d9c5b6052f0a11476cc67598 Mon Sep 17 00:00:00 2001 From: Yuanyuan Chen Date: Sun, 14 Dec 2025 08:20:26 +0800 Subject: [PATCH 09/30] Apply clang-tidy fixes to subinterpreter support code (#5929) * Fix PyObject_HasAttrString return value Signed-off-by: cyy * Tidy unchecked files Signed-off-by: cyy * [skip ci] Handle PyObject_HasAttrString error when probing __notes__ PyObject_HasAttrString may return -1 to signal an error and set a Python exception. The previous logic only checked for "!= 0", which meant that the error path was treated the same as "attribute exists", causing two problems: misreporting the presence of __notes__ and leaving a spurious exception pending. The earlier PR tightened the condition to "== 1" so that only a successful lookup marks the error string as [WITH __notes__], but it still left the -1 case unhandled. In the context of error_fetch_and_normalize, we are already dealing with an active exception and only want to best-effort detect whether normalization attached any __notes__. If the attribute probe itself fails, we do not want that secondary failure to affect later C-API calls or the error we ultimately report. This change stores the PyObject_HasAttrString return value, treats "== 1" as "has __notes__", and explicitly calls PyErr_Clear() when it returns -1. That way, we avoid leaking a secondary error while still preserving the original exception information and hinting [WITH __notes__] only when we can determine it reliably. * Run clang-tidy with -DPYBIND11_HAS_SUBINTERPRETER_SUPPORT * [skip ci] Revert "Run clang-tidy with -DPYBIND11_HAS_SUBINTERPRETER_SUPPORT" This reverts commit bb6e751de4a96d491fcbf5a989cf38aab31694d3. --------- Signed-off-by: cyy Co-authored-by: Ralf W. Grosse-Kunstleve --- include/pybind11/pytypes.h | 7 ++++++- include/pybind11/subinterpreter.h | 33 ++++++++++++++++--------------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 538ee1c75..0ab0b73e1 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -547,8 +547,13 @@ struct error_fetch_and_normalize { // The presence of __notes__ is likely due to exception normalization // errors, although that is not necessarily true, therefore insert a // hint only: - if (PyObject_HasAttrString(m_value.ptr(), "__notes__") != 0) { + const int has_notes = PyObject_HasAttrString(m_value.ptr(), "__notes__"); + if (has_notes == 1) { m_lazy_error_string += "[WITH __notes__]"; + } else if (has_notes == -1) { + // Ignore secondary errors when probing for __notes__ to avoid leaking a + // spurious exception while still reporting the original error. + PyErr_Clear(); } #else // PyErr_NormalizeException() may change the exception type if there are cascading diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index 5d2f0a839..aaf520457 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -22,11 +22,11 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) inline PyInterpreterState *get_interpreter_state_unchecked() { - auto cur_tstate = get_thread_state_unchecked(); - if (cur_tstate) + auto *cur_tstate = get_thread_state_unchecked(); + if (cur_tstate) { return cur_tstate->interp; - else - return nullptr; + } + return nullptr; } PYBIND11_NAMESPACE_END(detail) @@ -76,7 +76,7 @@ public: /// Create a new subinterpreter with the specified configuration /// @note This function acquires (and then releases) the main interpreter GIL, but the main /// interpreter and its GIL are not required to be held prior to calling this function. - static inline subinterpreter create(PyInterpreterConfig const &cfg) { + static subinterpreter create(PyInterpreterConfig const &cfg) { error_scope err_scope; subinterpreter result; @@ -84,7 +84,7 @@ public: // we must hold the main GIL in order to create a subinterpreter subinterpreter_scoped_activate main_guard(main()); - auto prev_tstate = PyThreadState_Get(); + auto *prev_tstate = PyThreadState_Get(); PyStatus status; @@ -103,7 +103,7 @@ public: } // this doesn't raise a normal Python exception, it provides an exit() status code. - if (PyStatus_Exception(status)) { + if (PyStatus_Exception(status) != 0) { pybind11_fail("failed to create new sub-interpreter"); } @@ -128,7 +128,7 @@ public: /// Calls create() with a default configuration of an isolated interpreter that disallows fork, /// exec, and Python threads. - static inline subinterpreter create() { + static subinterpreter create() { // same as the default config in the python docs PyInterpreterConfig cfg; std::memset(&cfg, 0, sizeof(cfg)); @@ -144,8 +144,8 @@ public: return; } - PyThreadState *destroy_tstate; - PyThreadState *old_tstate; + PyThreadState *destroy_tstate = nullptr; + PyThreadState *old_tstate = nullptr; // Python 3.12 requires us to keep the original PyThreadState alive until we are ready to // destroy the interpreter. We prefer to use that to destroy the interpreter. @@ -173,7 +173,7 @@ public: old_tstate = PyThreadState_Swap(destroy_tstate); #endif - bool switch_back = old_tstate && old_tstate->interp != istate_; + bool switch_back = (old_tstate != nullptr) && old_tstate->interp != istate_; // Internals always exists in the subinterpreter, this class enforces it when it creates // the subinterpreter. Even if it didn't, this only creates the pointer-to-pointer, not the @@ -190,8 +190,9 @@ public: detail::get_local_internals_pp_manager().destroy(); // switch back to the old tstate and old GIL (if there was one) - if (switch_back) + if (switch_back) { PyThreadState_Swap(old_tstate); + } } /// Get a handle to the main interpreter that can be used with subinterpreter_scoped_activate @@ -214,11 +215,11 @@ public: /// Get the numerical identifier for the sub-interpreter int64_t id() const { - if (istate_ != nullptr) + if (istate_ != nullptr) { return PyInterpreterState_GetID(istate_); - else - return -1; // CPython uses one-up numbers from 0, so negative should be safe to return - // here. + } + return -1; // CPython uses one-up numbers from 0, so negative should be safe to return + // here. } /// Get the interpreter's state dict. This interpreter's GIL must be held before calling! From d4f9cfbc2866f2156e1b17cb478a67088c6063f6 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 14 Dec 2025 19:01:34 -0800 Subject: [PATCH 10/30] Modernize NVHPC CI job (to make it working again): Ubuntu-24.04 runner, NVHPC 25.11 (#5935) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Limit busy-wait loops in per-subinterpreter GIL test Add explicit timeouts to the busy-wait coordination loops in the Per-Subinterpreter GIL test in tests/test_with_catch/test_subinterpreter.cpp. Previously those loops spun indefinitely waiting for shared atomics like `started` and `sync` to change, which is fine when CPython's free-threading and per-interpreter GIL behavior matches the test's expectations but becomes pathologically bad when that behavior regresses: the `test_with_catch` executable can then hang forever, causing our 3.14t CI jobs to time out after 90 minutes. This change keeps the structure and intent of the test but adds a std::chrono::steady_clock deadline to each of the coordination loops, using a conservative 10 second bound. Worker threads record a failure and return if they hit the timeout, while the main thread fails the test via Catch2 instead of hanging. That way, if future CPython free-threading patches change the semantics again, the test will fail quickly and produced a diagnosable error instead of wedging the CI job. * Revert "Limit busy-wait loops in per-subinterpreter GIL test" This reverts commit 7847adacda5d35582c9f3e5720199ae001681781. * Add progress reporter for test_with_catch Catch runner Introduce a custom Catch2 reporter for tests/test_with_catch that prints a simple one-line status for each test case as it starts and ends, and wire the cpptest CMake target to invoke test_with_catch with -r progress. This makes it much easier to see where the embedded/interpreter test binary is spending its time in CI logs, and in particular to pinpoint which test case is stuck when the free-threading builds hang. Compared to adding ad hoc timeouts around potentially infinite busy-wait loops in individual tests, a progress reporter is a more general and robust approach: it gives visibility into all tests (including future ones) without changing their behavior, and turns otherwise opaque 90-minute timeouts into locatable issues in the Catch output. * Temporarily limit CI to Python 3.14t free-threading jobs * Temporarily remove non-CI GitHub workflow files * Temporarily disable AppVeyor builds via skip_commits * Add DEBUG_LOOK in TEST_CASE("Move Subinterpreter") * Add Python version banner to Catch progress reporter Print the CPython version once at the start of the Catch-based interpreter tests using Py_GetVersion(). This makes it trivial to confirm which free-threaded build a failing run is using when inspecting CI or local logs. * Revert "Add DEBUG_LOOK in TEST_CASE("Move Subinterpreter")" This reverts commit ad3e1c34ceeb072c5812f385f38360387fbbcf68. * Pin CI free-threaded runs to CPython 3.14.0t Update the standard-small and standard-large GitHub Actions jobs to request python-version 3.14.0t instead of 3.14t. This forces setup-python to use the last-known-good 3.14.0 free-threaded build rather than the newer 3.14.1+ builds where subinterpreter finalization regressed. * Revert "Pin CI free-threaded runs to CPython 3.14.0t" This reverts commit 5281e1c20c7477878df53fd6a09395f8bb6dcc09. * Revert "Temporarily disable AppVeyor builds via skip_commits" This reverts commit ed112926365544bfdb6d5153f8947474d8c4caf2. * Revert "Temporarily remove non-CI GitHub workflow files" This reverts commit 0fe6a42a0406f32178429fa8b50de206b6db37d2. * Revert "Temporarily limit CI to Python 3.14t free-threading jobs" This reverts commit 60ae0e8f744904ba2499cf44a43fdf91049f6e4a. * Pin CI free-threaded runs to CPython 3.14.0t Update the standard-small and standard-large GitHub Actions jobs to request python-version 3.14.0t instead of 3.14t. This forces setup-python to use the last-known-good 3.14.0 free-threaded build rather than the newer 3.14.1+ builds where subinterpreter finalization regressed. * Switch NVHPC job to ubuntu-24.04 and disable AppVeyor * Temporarily trim workflows to focus on NVHPC job * First restore ci.yml from test-with-catch-timeouts branch, then delete all jobs except ubuntu-nvhpc7 * Change runner to ubuntu-24.04 * Use nvhpc-25-11 * Undo ALL changes relative to master (i.e. this branch is now an exact copy of master) * Change runner to ubuntu-24.04 * Use nvhpc-25-11 * Remove misleading 7 from job name (i.e. ubuntu-nvhpc7 → ubuntu-nvhpc) --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d00ee6d27..c5a200e32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -470,10 +470,10 @@ jobs: # Testing on Ubuntu + NVHPC (previous PGI) compilers, which seems to require more workarounds - ubuntu-nvhpc7: + ubuntu-nvhpc: if: github.event.pull_request.draft == false - runs-on: ubuntu-22.04 - name: "🐍 3 • NVHPC 23.5 • C++17 • x64" + runs-on: ubuntu-24.04 + name: "🐍 3 • NVHPC 25.11 • C++17 • x64" timeout-minutes: 90 env: @@ -491,7 +491,7 @@ jobs: run: | sudo apt-get update -y && \ sudo apt-get install -y cmake environment-modules git python3-dev python3-pip python3-numpy && \ - sudo apt-get install -y --no-install-recommends nvhpc-23-5 && \ + sudo apt-get install -y --no-install-recommends nvhpc-25-11 && \ sudo rm -rf /var/lib/apt/lists/* python3 -m pip install --upgrade pip python3 -m pip install --upgrade pytest @@ -502,7 +502,7 @@ jobs: shell: bash run: | source /etc/profile.d/modules.sh - module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/23.5 + module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/25.11 cmake -S . -B build -DDOWNLOAD_CATCH=ON \ -DCMAKE_CXX_STANDARD=17 \ -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \ @@ -510,7 +510,7 @@ jobs: -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp" - name: Build - run: cmake --build build -j 2 --verbose + run: cmake --build build -j $(nproc) --verbose - name: Python tests run: cmake --build build --target pytest From 3aeb113b0a904bb277a23038c0ed5170122fdaa0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Dec 2025 19:53:24 -0800 Subject: [PATCH 11/30] chore(deps): bump the actions group across 1 directory with 5 updates (#5941) * chore(deps): bump the actions group across 1 directory with 5 updates Bumps the actions group with 5 updates in the / directory: | Package | From | To | | --- | --- | --- | | [actions/checkout](https://github.com/actions/checkout) | `1` | `6` | | [actions/cache](https://github.com/actions/cache) | `4` | `5` | | [actions/upload-artifact](https://github.com/actions/upload-artifact) | `5` | `6` | | [actions/download-artifact](https://github.com/actions/download-artifact) | `6` | `7` | | [scientific-python/upload-nightly-action](https://github.com/scientific-python/upload-nightly-action) | `0.6.2` | `0.6.3` | Updates `actions/checkout` from 1 to 6 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v1...v6) Updates `actions/cache` from 4 to 5 - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v4...v5) Updates `actions/upload-artifact` from 5 to 6 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v5...v6) Updates `actions/download-artifact` from 6 to 7 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v6...v7) Updates `scientific-python/upload-nightly-action` from 0.6.2 to 0.6.3 - [Release notes](https://github.com/scientific-python/upload-nightly-action/releases) - [Commits](https://github.com/scientific-python/upload-nightly-action/compare/b36e8c0c10dbcfd2e05bf95f17ef8c14fd708dbf...5748273c71e2d8d3a61f3a11a16421c8954f9ecf) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: actions/cache dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: actions/upload-artifact dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: actions/download-artifact dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: scientific-python/upload-nightly-action dependency-version: 0.6.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] * Reset install-classic actions/checkout to @v1 * Pin actions/checkout@v1 to SHA, switch dependabot to monthly Pin the i386/debian container checkout to SHA to prevent dependabot from updating it (v1 is required for that container). Remove the non-working ignore block and change schedule from weekly to monthly. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- .github/dependabot.yml | 6 +----- .github/workflows/ci.yml | 5 +++-- .github/workflows/nightlies.yml | 6 +++--- .github/workflows/pip.yml | 6 +++--- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 22c34bd74..8b13673c1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,12 +4,8 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" + interval: "monthly" groups: actions: patterns: - "*" - ignore: - - dependency-name: actions/checkout - versions: - - "<5" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5a200e32..86eea6805 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -296,7 +296,7 @@ jobs: - name: Valgrind cache if: matrix.valgrind - uses: actions/cache@v4 + uses: actions/cache@v5 id: cache-valgrind with: path: valgrind @@ -778,7 +778,8 @@ jobs: timeout-minutes: 90 steps: - - uses: actions/checkout@v1 # v1 is required to run inside docker + # v1 required for i386/debian container; pinned to SHA to prevent dependabot updates + - uses: actions/checkout@544eadc6bf3d226fd7a7a9f0dc5b5bf7ca0675b9 # v1 - name: Install requirements run: | diff --git a/.github/workflows/nightlies.yml b/.github/workflows/nightlies.yml index ad4a35152..9b4f933e4 100644 --- a/.github/workflows/nightlies.yml +++ b/.github/workflows/nightlies.yml @@ -33,7 +33,7 @@ jobs: nox -s build nox -s build_global - - uses: actions/upload-artifact@v5 + - uses: actions/upload-artifact@v6 with: name: Packages path: dist/* @@ -44,7 +44,7 @@ jobs: needs: [build_wheel] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v6 + - uses: actions/download-artifact@v7 with: name: Packages path: dist @@ -53,7 +53,7 @@ jobs: run: ls -lha dist/*.whl - name: Upload wheel to Anaconda Cloud as nightly - uses: scientific-python/upload-nightly-action@b36e8c0c10dbcfd2e05bf95f17ef8c14fd708dbf # 0.6.2 + uses: scientific-python/upload-nightly-action@5748273c71e2d8d3a61f3a11a16421c8954f9ecf # 0.6.3 with: artifacts_path: dist anaconda_nightly_upload_token: ${{ secrets.ANACONDA_ORG_UPLOAD_TOKEN }} diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 8df91a00f..b7555a5a7 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -72,13 +72,13 @@ jobs: run: twine check dist/* - name: Save standard package - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v6 with: name: standard path: dist/pybind11-* - name: Save global package - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v6 with: name: global path: dist/*global-* @@ -100,7 +100,7 @@ jobs: steps: # Downloads all to directories matching the artifact names - - uses: actions/download-artifact@v6 + - uses: actions/download-artifact@v7 - name: Generate artifact attestation for sdist and wheel uses: actions/attest-build-provenance@v3 From 78381e5e281f3d329f0bc85f43a2da8a476968a1 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 22 Dec 2025 12:25:06 +0700 Subject: [PATCH 12/30] Improve C++ test infrastructure: progress reporter, timeouts, and skip hanging Move Subinterpreter test (#5942) * Improve C++ test infrastructure and disable hanging test This commit improves the C++ test infrastructure to ensure test output is visible in CI logs, and disables a test that hangs on free-threaded Python 3.14+. Changes: ## CI/test infrastructure improvements - .github/workflows: Added `timeout-minutes: 3` to all C++ test steps to prevent indefinite hangs. - tests/**/CMakeLists.txt: Added `USES_TERMINAL` to C++ test targets (cpptest, test_cross_module_rtti, test_pure_cpp) to ensure output is shown immediately rather than buffered and possibly lost on crash/timeout. - tests/test_with_catch/catch.cpp: Added a custom Catch2 progress reporter with timestamps, Python version info, and a SIGTERM handler to make test execution and failures clearly visible in CI logs. ## Disabled hanging test - The "Move Subinterpreter" test is disabled on free-threaded Python 3.14+ due to a hang in Py_EndInterpreter() when the subinterpreter is destroyed from a different thread than it was created on. Work on fixing the underlying issue will continue under PR #5940. Context: We were in the dark for months (since we started testing with Python 3.14t) because CI logs gave no clue about the root cause of hangs. This led to ignoring intermittent hangs (mostly on macOS). Our hand was forced only with the Python 3.14.1 release, when hangs became predictable on all platforms. For the full development history of these changes, see PR #5933. * Add test summary to progress reporter Print the total number of test cases and assertions at the end of the test run, making it easy to spot if tests are disabled or added. Example output: [ PASSED ] 20 test cases, 1589 assertions. * Add PYBIND11_CATCH2_SKIP_IF macro to skip tests at runtime Catch2 v2 doesn't have native skip support (v3 does with SKIP()). This macro allows tests to be skipped with a visible message while still appearing in the test list. Use this for the Move Subinterpreter test on free-threaded Python 3.14+ so it shows as skipped rather than being conditionally compiled out. Example output: [ RUN ] Move Subinterpreter [ SKIPPED ] Skipped on free-threaded Python 3.14+ (see PR #5940) [ OK ] Move Subinterpreter * Fix clang-tidy bugprone-macro-parentheses warning in PYBIND11_CATCH2_SKIP_IF --- .github/workflows/ci.yml | 15 ++ .github/workflows/reusable-standard.yml | 1 + .github/workflows/upstream.yml | 2 + tests/pure_cpp/CMakeLists.txt | 4 +- tests/test_cross_module_rtti/CMakeLists.txt | 4 +- tests/test_with_catch/CMakeLists.txt | 4 +- tests/test_with_catch/catch.cpp | 132 ++++++++++++++++++ tests/test_with_catch/catch_skip.h | 16 +++ tests/test_with_catch/test_subinterpreter.cpp | 9 ++ 9 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 tests/test_with_catch/catch_skip.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86eea6805..f5569e3a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,6 +229,7 @@ jobs: run: cmake --build . --target pytest - name: Compiled tests + timeout-minutes: 3 run: cmake --build . --target cpptest - name: Interface test @@ -334,6 +335,7 @@ jobs: run: cmake --build --preset default --target pytest - name: C++ tests + timeout-minutes: 3 run: cmake --build --preset default --target cpptest - name: Visibility test @@ -393,6 +395,7 @@ jobs: run: cmake --build build --target pytest - name: C++ tests + timeout-minutes: 3 run: cmake --build build --target cpptest - name: Interface test @@ -516,6 +519,7 @@ jobs: run: cmake --build build --target pytest - name: C++ tests + timeout-minutes: 3 run: cmake --build build --target cpptest - name: Interface test @@ -570,6 +574,7 @@ jobs: run: cmake --build build --target pytest - name: C++ tests + timeout-minutes: 3 run: cmake --build build --target cpptest - name: Interface test @@ -652,6 +657,7 @@ jobs: cmake --build build-11 --target check - name: C++ tests C++11 + timeout-minutes: 3 run: | set +e; source /opt/intel/oneapi/setvars.sh; set -e cmake --build build-11 --target cpptest @@ -689,6 +695,7 @@ jobs: cmake --build build-17 --target check - name: C++ tests C++17 + timeout-minutes: 3 run: | set +e; source /opt/intel/oneapi/setvars.sh; set -e cmake --build build-17 --target cpptest @@ -760,6 +767,7 @@ jobs: run: cmake --build build --target pytest - name: C++ tests + timeout-minutes: 3 run: cmake --build build --target cpptest - name: Interface test @@ -1001,6 +1009,7 @@ jobs: run: cmake --build build --target pytest - name: C++20 tests + timeout-minutes: 3 run: cmake --build build --target cpptest -j 2 - name: Interface test C++20 @@ -1077,6 +1086,7 @@ jobs: run: cmake --build build --target pytest -j 2 - name: C++11 tests + timeout-minutes: 3 run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build --target cpptest -j 2 - name: Interface test C++11 @@ -1101,6 +1111,7 @@ jobs: run: cmake --build build2 --target pytest -j 2 - name: C++14 tests + timeout-minutes: 3 run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build2 --target cpptest -j 2 - name: Interface test C++14 @@ -1125,6 +1136,7 @@ jobs: run: cmake --build build3 --target pytest -j 2 - name: C++17 tests + timeout-minutes: 3 run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build3 --target cpptest -j 2 - name: Interface test C++17 @@ -1196,6 +1208,7 @@ jobs: run: cmake --build . --target pytest -j 2 - name: C++ tests + timeout-minutes: 3 run: cmake --build . --target cpptest -j 2 - name: Interface test @@ -1258,6 +1271,7 @@ jobs: run: cmake --build . --target pytest -j 2 - name: C++ tests + timeout-minutes: 3 run: cmake --build . --target cpptest -j 2 - name: Interface test @@ -1330,6 +1344,7 @@ jobs: run: cmake --build build --target pytest -j 2 - name: C++ tests + timeout-minutes: 3 run: PYTHONHOME=/clangarm64 PYTHONPATH=/clangarm64 cmake --build build --target cpptest -j 2 - name: Interface test diff --git a/.github/workflows/reusable-standard.yml b/.github/workflows/reusable-standard.yml index 96b14bdfb..56d92e277 100644 --- a/.github/workflows/reusable-standard.yml +++ b/.github/workflows/reusable-standard.yml @@ -83,6 +83,7 @@ jobs: run: cmake --build build --target pytest - name: C++ tests + timeout-minutes: 3 run: cmake --build build --target cpptest - name: Interface test diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 15ede7a85..890ae0b6f 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -66,6 +66,7 @@ jobs: run: cmake --build build11 --target pytest -j 2 - name: C++11 tests + timeout-minutes: 3 run: cmake --build build11 --target cpptest -j 2 - name: Interface test C++11 @@ -87,6 +88,7 @@ jobs: run: cmake --build build17 --target pytest - name: C++17 tests + timeout-minutes: 3 run: cmake --build build17 --target cpptest # Third build - C++17 mode with unstable ABI diff --git a/tests/pure_cpp/CMakeLists.txt b/tests/pure_cpp/CMakeLists.txt index 1150cb405..d2757db76 100644 --- a/tests/pure_cpp/CMakeLists.txt +++ b/tests/pure_cpp/CMakeLists.txt @@ -15,6 +15,8 @@ target_link_libraries(smart_holder_poc_test PRIVATE pybind11::headers Catch2::Ca add_custom_target( test_pure_cpp COMMAND "$" - WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + USES_TERMINAL # Ensures output is shown immediately (not buffered and possibly lost on crash) +) add_dependencies(check test_pure_cpp) diff --git a/tests/test_cross_module_rtti/CMakeLists.txt b/tests/test_cross_module_rtti/CMakeLists.txt index 97d2c780c..c9b95bfba 100644 --- a/tests/test_cross_module_rtti/CMakeLists.txt +++ b/tests/test_cross_module_rtti/CMakeLists.txt @@ -60,7 +60,9 @@ add_custom_target( test_cross_module_rtti COMMAND "$" DEPENDS test_cross_module_rtti_main - WORKING_DIRECTORY "$") + WORKING_DIRECTORY "$" + USES_TERMINAL # Ensures output is shown immediately (not buffered and possibly lost on crash) +) set_target_properties(test_cross_module_rtti_bindings PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") diff --git a/tests/test_with_catch/CMakeLists.txt b/tests/test_with_catch/CMakeLists.txt index 136537e67..e6a9f67aa 100644 --- a/tests/test_with_catch/CMakeLists.txt +++ b/tests/test_with_catch/CMakeLists.txt @@ -47,7 +47,9 @@ add_custom_target( cpptest COMMAND "$" DEPENDS test_with_catch - WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + USES_TERMINAL # Ensures output is shown immediately (not buffered and possibly lost on crash) +) pybind11_add_module(external_module THIN_LTO external_module.cpp) set_target_properties(external_module PROPERTIES LIBRARY_OUTPUT_DIRECTORY diff --git a/tests/test_with_catch/catch.cpp b/tests/test_with_catch/catch.cpp index 5bd8b3880..895959318 100644 --- a/tests/test_with_catch/catch.cpp +++ b/tests/test_with_catch/catch.cpp @@ -3,6 +3,17 @@ #include +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +# include +#endif + // Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to // catch 2.0.1; this should be fixed in the next catch release after 2.0.1). PYBIND11_WARNING_DISABLE_MSVC(4996) @@ -13,11 +24,126 @@ PYBIND11_WARNING_DISABLE_MSVC(4996) #endif #define CATCH_CONFIG_RUNNER +#define CATCH_CONFIG_DEFAULT_REPORTER "progress" +#include "catch_skip.h" + #include namespace py = pybind11; +// Simple progress reporter that prints a line per test case. +namespace { + +class ProgressReporter : public Catch::StreamingReporterBase { +public: + using StreamingReporterBase::StreamingReporterBase; + + static std::string getDescription() { return "Simple progress reporter (one line per test)"; } + + void testCaseStarting(Catch::TestCaseInfo const &testInfo) override { + print_python_version_once(); + auto &os = Catch::cout(); + os << "[ RUN ] " << testInfo.name << '\n'; + os.flush(); + } + + void testCaseEnded(Catch::TestCaseStats const &stats) override { + bool failed = stats.totals.assertions.failed > 0; + auto &os = Catch::cout(); + os << (failed ? "[ FAILED ] " : "[ OK ] ") << stats.testInfo.name << '\n'; + os.flush(); + } + + void noMatchingTestCases(std::string const &spec) override { + auto &os = Catch::cout(); + os << "[ NO TEST ] no matching test cases for spec: " << spec << '\n'; + os.flush(); + } + + void reportInvalidArguments(std::string const &arg) override { + auto &os = Catch::cout(); + os << "[ ERROR ] invalid Catch2 arguments: " << arg << '\n'; + os.flush(); + } + + void assertionStarting(Catch::AssertionInfo const &) override {} + + bool assertionEnded(Catch::AssertionStats const &) override { return false; } + + void testRunEnded(Catch::TestRunStats const &stats) override { + auto &os = Catch::cout(); + auto passed = stats.totals.testCases.passed; + auto failed = stats.totals.testCases.failed; + auto total = passed + failed; + auto assertions = stats.totals.assertions.passed + stats.totals.assertions.failed; + if (failed == 0) { + os << "[ PASSED ] " << total << " test cases, " << assertions << " assertions.\n"; + } else { + os << "[ FAILED ] " << failed << " of " << total << " test cases, " << assertions + << " assertions.\n"; + } + os.flush(); + } + +private: + void print_python_version_once() { + if (printed_) { + return; + } + printed_ = true; + auto &os = Catch::cout(); + os << "[ PYTHON ] " << Py_GetVersion() << '\n'; + os.flush(); + } + + bool printed_ = false; +}; + +} // namespace + +CATCH_REGISTER_REPORTER("progress", ProgressReporter) + +namespace { + +std::string get_utc_timestamp() { + auto now = std::chrono::system_clock::now(); + auto time_t_now = std::chrono::system_clock::to_time_t(now); + auto ms = std::chrono::duration_cast(now.time_since_epoch()) % 1000; + + std::tm utc_tm{}; +#if defined(_WIN32) + gmtime_s(&utc_tm, &time_t_now); +#else + gmtime_r(&time_t_now, &utc_tm); +#endif + + std::ostringstream oss; + oss << std::put_time(&utc_tm, "%Y-%m-%d %H:%M:%S") << '.' << std::setfill('0') << std::setw(3) + << ms.count() << 'Z'; + return oss.str(); +} + +#ifndef _WIN32 +// Signal handler to print a message when the process is terminated. +// Uses only async-signal-safe functions. +void termination_signal_handler(int sig) { + const char *msg = "[ SIGNAL ] Process received SIGTERM\n"; + // write() is async-signal-safe, unlike std::cout + ssize_t written = write(STDOUT_FILENO, msg, strlen(msg)); + (void) written; // suppress "unused variable" warnings + // Re-raise with default handler to get proper exit status + std::signal(sig, SIG_DFL); + std::raise(sig); +} +#endif + +} // namespace + int main(int argc, char *argv[]) { +#ifndef _WIN32 + std::signal(SIGTERM, termination_signal_handler); +#endif + // Setup for TEST_CASE in test_interpreter.cpp, tagging on a large random number: std::string updated_pythonpath("pybind11_test_with_catch_PYTHONPATH_2099743835476552"); const char *preexisting_pythonpath = getenv("PYTHONPATH"); @@ -35,9 +161,15 @@ int main(int argc, char *argv[]) { setenv("PYTHONPATH", updated_pythonpath.c_str(), /*replace=*/1); #endif + std::cout << "[ STARTING ] " << get_utc_timestamp() << '\n'; + std::cout.flush(); + py::scoped_interpreter guard{}; auto result = Catch::Session().run(argc, argv); + std::cout << "[ DONE ] " << get_utc_timestamp() << " (result " << result << ")\n"; + std::cout.flush(); + return result < 0xff ? result : 0xff; } diff --git a/tests/test_with_catch/catch_skip.h b/tests/test_with_catch/catch_skip.h new file mode 100644 index 000000000..72ffdb62b --- /dev/null +++ b/tests/test_with_catch/catch_skip.h @@ -0,0 +1,16 @@ +// Macro to skip a test at runtime with a visible message. +// Catch2 v2 doesn't have native skip support (v3 does with SKIP()). +// The test will count as "passed" in totals, but the output clearly shows it was skipped. + +#pragma once + +#include + +#define PYBIND11_CATCH2_SKIP_IF(condition, reason) \ + do { \ + if (condition) { \ + Catch::cout() << "[ SKIPPED ] " << (reason) << '\n'; \ + Catch::cout().flush(); \ + return; \ + } \ + } while (0) diff --git a/tests/test_with_catch/test_subinterpreter.cpp b/tests/test_with_catch/test_subinterpreter.cpp index 3c7c35be1..26e059758 100644 --- a/tests/test_with_catch/test_subinterpreter.cpp +++ b/tests/test_with_catch/test_subinterpreter.cpp @@ -6,6 +6,8 @@ // catch 2.0.1; this should be fixed in the next catch release after 2.0.1). PYBIND11_WARNING_DISABLE_MSVC(4996) +# include "catch_skip.h" + # include # include # include @@ -92,6 +94,13 @@ TEST_CASE("Single Subinterpreter") { # if PY_VERSION_HEX >= 0x030D0000 TEST_CASE("Move Subinterpreter") { + // Test is skipped on free-threaded Python 3.14+ due to a hang in Py_EndInterpreter() + // when the subinterpreter is destroyed from a different thread than it was created on. + // See: https://github.com/pybind/pybind11/pull/5940 +# if PY_VERSION_HEX >= 0x030E0000 && defined(Py_GIL_DISABLED) + PYBIND11_CATCH2_SKIP_IF(true, "Skipped on free-threaded Python 3.14+ (see PR #5940)"); +# endif + std::unique_ptr sub(new py::subinterpreter(py::subinterpreter::create())); // on this thread, use the subinterpreter and import some non-trivial junk From 7ae61bfb82d7d477771ac873a9c4b031cf21f0bb Mon Sep 17 00:00:00 2001 From: KenLee Date: Tue, 23 Dec 2025 11:07:33 +0800 Subject: [PATCH 13/30] Avoid LNK2001 in c++20 when /GL (Whole program optimization) is on with MSVC Update pybind11.h (#5939) --- include/pybind11/pybind11.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 91b38d91e..f93272628 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -43,6 +43,12 @@ PYBIND11_WARNING_DISABLE_CLANG("-Wgnu-zero-variadic-macro-arguments") # include #endif +#if defined(__cpp_if_constexpr) && __cpp_if_constexpr >= 201606 +# define PYBIND11_MAYBE_CONSTEXPR constexpr +#else +# define PYBIND11_MAYBE_CONSTEXPR +#endif + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) /* https://stackoverflow.com/questions/46798456/handling-gccs-noexcept-type-warning @@ -3606,13 +3612,15 @@ function get_override(const T *this_ptr, const char *name) { auto o = override(__VA_ARGS__); \ PYBIND11_WARNING_PUSH \ PYBIND11_WARNING_DISABLE_MSVC(4127) \ - if (pybind11::detail::cast_is_temporary_value_reference::value \ + if PYBIND11_MAYBE_CONSTEXPR ( \ + pybind11::detail::cast_is_temporary_value_reference::value \ && !pybind11::detail::is_same_ignoring_cvref::value) { \ static pybind11::detail::override_caster_t caster; \ return pybind11::detail::cast_ref(std::move(o), caster); \ + } else { \ + return pybind11::detail::cast_safe(std::move(o)); \ } \ PYBIND11_WARNING_POP \ - return pybind11::detail::cast_safe(std::move(o)); \ } \ } while (false) From 799f591ec3487b34f9f10965ed1f90ce3930b98b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 23 Dec 2025 12:32:41 +0700 Subject: [PATCH 14/30] Re-enable Move Subinterpreter test for free-threaded Python 3.14 (#5940) * Remove skip for Move Subinterpreter test on free-threaded Python 3.14+ * Fix deadlock by detaching from the main interpreter before joining the thread. * style: pre-commit fixes --------- Co-authored-by: b-pass Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/advanced/embedding.rst | 4 ++++ tests/test_with_catch/test_subinterpreter.cpp | 18 +++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index 7d770faa2..145471b7f 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -492,4 +492,8 @@ Best Practices for sub-interpreter safety So you must still consider the thread safety of your C++ code. Remember, in Python 3.12 sub-interpreters must be destroyed on the same thread that they were created on. +- When using sub-interpreters in free-threaded python builds, note that creating and destroying + sub-interpreters may initiate a "stop-the-world". Be sure to detach long-running C++ threads + from Python thread state (similar to releasing the GIL) to avoid deadlocks. + - Familiarize yourself with :ref:`misc_concurrency`. diff --git a/tests/test_with_catch/test_subinterpreter.cpp b/tests/test_with_catch/test_subinterpreter.cpp index 26e059758..278313388 100644 --- a/tests/test_with_catch/test_subinterpreter.cpp +++ b/tests/test_with_catch/test_subinterpreter.cpp @@ -94,13 +94,6 @@ TEST_CASE("Single Subinterpreter") { # if PY_VERSION_HEX >= 0x030D0000 TEST_CASE("Move Subinterpreter") { - // Test is skipped on free-threaded Python 3.14+ due to a hang in Py_EndInterpreter() - // when the subinterpreter is destroyed from a different thread than it was created on. - // See: https://github.com/pybind/pybind11/pull/5940 -# if PY_VERSION_HEX >= 0x030E0000 && defined(Py_GIL_DISABLED) - PYBIND11_CATCH2_SKIP_IF(true, "Skipped on free-threaded Python 3.14+ (see PR #5940)"); -# endif - std::unique_ptr sub(new py::subinterpreter(py::subinterpreter::create())); // on this thread, use the subinterpreter and import some non-trivial junk @@ -113,14 +106,21 @@ TEST_CASE("Move Subinterpreter") { py::module_::import("external_module"); } - std::thread([&]() { + auto t = std::thread([&]() { // Use it again { py::subinterpreter_scoped_activate activate(*sub); py::module_::import("external_module"); } sub.reset(); - }).join(); + }); + + // on 3.14.1+ destructing a sub-interpreter does a stop-the-world. we need to detach our + // thread state in order for that to be possible. + { + py::gil_scoped_release nogil; + t.join(); + } REQUIRE(!sub); From 0057e4945d3a77b7027c5970148e5addaf65ef1b Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Thu, 25 Dec 2025 15:33:02 +0800 Subject: [PATCH 15/30] Add per-interpreter storage for `gil_safe_call_once_and_store` (#5933) * Add new argument to `gil_safe_call_once_and_store::call_once_and_store_result` * Add per-interpreter storage for `gil_safe_call_once_and_store` * Make `~gil_safe_call_once_and_store` a no-op * Fix C++11 compatibility * Improve thread-safety and add default finalizer * Try fix thread-safety * Try fix thread-safety * Add a warning comment * Simplify `PYBIND11_INTERNALS_VERSION >= 12` * Try fix thread-safety * Try fix thread-safety * Revert get_pp() * Update comments * Move call-once storage out of internals * Revert internal version bump * Cleanup outdated comments * Move atomic_bool alias into pybind11::detail namespace The `using atomic_bool = ...` declaration was at global scope, polluting the global namespace. Move it into pybind11::detail to avoid potential conflicts with user code. * Add explicit #include for subinterpreter support The subinterpreter branch uses std::unordered_map but relied on transitive includes. Add an explicit include for robustness. * Remove extraneous semicolon after destructor definition Style fix: remove trailing semicolon after ~call_once_storage() destructor body. * Add comment explaining unused finalize parameter Clarify why the finalize callback parameter is intentionally ignored when subinterpreter support is disabled: the storage is process-global and leaked to avoid destructor calls after interpreter finalization. * Add comment explaining error_scope usage Clarify why error_scope is used: to preserve any existing Python error state that might be cleared or modified by dict_getitemstringref. * Improve exception safety in get_or_create_call_once_storage_map() Use std::unique_ptr to hold the newly allocated storage map until the capsule is successfully created. This prevents a memory leak if capsule creation throws an exception. * Add timeout-minutes: 3 to cpptest workflow steps Add a 3-minute timeout to all C++ test (cpptest) steps across all platforms to detect hangs early. This uses GitHub Actions' built-in timeout-minutes property which works on Linux, macOS, and Windows. * Add progress reporter for test_with_catch Catch2 runner Add a custom Catch2 streaming reporter that prints one line per test case as it starts and ends, with immediate flushing to keep CI logs current. This makes it easy to see where the embedded/interpreter tests are spending time and to pinpoint which test case is stuck when builds hang (e.g., free-threading issues). The reporter: - Prints "[ RUN ]" when each test starts - Prints "[ OK ]" or "[ FAILED ]" when each test ends - Prints the Python version once at the start via Py_GetVersion() - Uses StreamingReporterBase for immediate output (not buffered) - Is set as the default reporter via CATCH_CONFIG_DEFAULT_REPORTER This approach gives visibility into all tests without changing their behavior, turning otherwise opaque 90-minute CI timeouts into locatable issues in the Catch output. * clang-format auto-fix (overlooked before) * Disable "Move Subinterpreter" test on free-threaded Python 3.14+ This test hangs in Py_EndInterpreter() when the subinterpreter is destroyed from a different thread than it was created on. The hang was observed: - Intermittently on macOS with Python 3.14.0t - Predictably on macOS, Ubuntu, and Windows with Python 3.14.1t and 3.14.2t Root cause analysis points to an interaction between pybind11's subinterpreter creation code and CPython's free-threaded runtime, specifically around PyThreadState_Swap() after PyThreadState_DeleteCurrent(). See detailed analysis: https://github.com/pybind/pybind11/pull/5933 * style: pre-commit fixes * Add test for gil_safe_call_once_and_store per-interpreter isolation This test verifies that gil_safe_call_once_and_store provides separate storage for each interpreter when subinterpreter support is enabled. The test caches the interpreter ID in the main interpreter, then creates a subinterpreter and verifies it gets its own cached value (not the main interpreter's). Without per-interpreter storage, the subinterpreter would incorrectly see the main interpreter's cached object. * Add STARTING/DONE timestamps to test_with_catch output Print UTC timestamps at the beginning and end of the test run to make it immediately clear when tests started and whether they ran to completion. The DONE message includes the Catch session result value. Example output: [ STARTING ] 2025-12-21 03:23:20.497Z [ PYTHON ] 3.14.2 ... [ RUN ] Threads [ OK ] Threads [ DONE ] 2025-12-21 03:23:20.512Z (result 0) * Disable stdout buffering in test_with_catch Ensure test output appears immediately in CI logs by disabling stdout buffering. Without this, output may be lost if the process is killed by a timeout, making it difficult to diagnose which test was hanging. * EXPERIMENT: Re-enable hanging test to verify CI log buffering fix This is a temporary commit to verify that the unbuffered stdout fix makes the hanging test visible in CI logs. REVERT THIS COMMIT after confirming the output appears. * Revert "Disable stdout buffering in test_with_catch" This reverts commit 0f8f32a92ac4f6c6e233e17104302aa61281aee5. * Use USES_TERMINAL for cpptest to show output immediately Ninja buffers subprocess output until completion. When a test hangs, the output is never shown, making it impossible to diagnose which test is hanging. USES_TERMINAL gives the command direct terminal access, bypassing ninja's buffering. This explains why Windows CI showed test progress but Linux/macOS did not - Windows uses MSBuild which doesn't buffer the same way. * Fix clang-tidy performance-avoid-endl warning Use '\n' instead of std::endl since USES_TERMINAL now handles output buffering at the CMake level. * Add SIGTERM handler to show when test is killed by timeout When a test hangs and is killed by `timeout`, Catch2 marks it as failed but the process exits before printing [ DONE ]. This made it unclear whether the test failed normally or was terminated. The signal handler prints a clear message when SIGTERM is received, making timeout-related failures obvious in CI logs. * Fix typo: atleast -> at_least * Fix GCC warn_unused_result error for write() in signal handler Assign the return value to a variable to satisfy GCC's warn_unused_result attribute, then cast to void to suppress unused variable warning. * Add USES_TERMINAL to other C++ test targets Apply the same ninja output buffering fix to test_cross_module_rtti and test_pure_cpp targets. Also add explanatory comments to all USES_TERMINAL usages. * Revert "EXPERIMENT: Re-enable hanging test to verify CI log buffering fix" This reverts commit a3abdeea8976f0a11beb6dc3ed9c1f52a9f5f45a. * Update comment to reference PR #5940 for Move Subinterpreter fix * Add alias `interpid_t = std::int64_t` * Add isolation and gc test for `gil_safe_call_once_and_store` * Add thread local cache for gil_safe_call_once_and_store * Revert "Add thread local cache for gil_safe_call_once_and_store" This reverts commit 5d6681956d2d326fe74c7bf80e845c8e8ddb2a7c. * Revert changes according to code review * Relocate multiple-interpreters tests * Add more tests for multiple interpreters * Remove copy constructor * Apply suggestions from code review * Refactor to use per-storage capsule instead * Update comments * Update singleton tests * Use interpreter id type for `get_num_interpreters_seen()` * Suppress unused variable warning * HACKING * Revert "HACKING" This reverts commit 534235ea559247f7a4984174f57f809b90a93af4. * Try fix concurrency * Test even harder * Reorg code to avoid duplicates * Fix unique_ptr::reset -> unique_ptr::release * Extract reusable functions * Fix indentation * Appease warnings for MSVC * Appease warnings for MSVC * Appease warnings for MSVC * Try fix concurrency by not using `get_num_interpreters_seen() > 1` * Try fix tests * Make Python path handling more robust * Update comments and assertion messages * Revert changes according to code review * Disable flaky tests * Use `@pytest.mark.xfail` rather than `pytest.skip` * Retrigger CI * Retrigger CI * Revert file moves * Refactor atomic_get_or_create_in_state_dict: improve API and fix on_fetch_ bug Three improvements to atomic_get_or_create_in_state_dict: 1. Return std::pair instead of just Payload* - The bool indicates whether storage was newly created (true) or already existed (false), following std::map::insert convention. - This fixes a bug where on_fetch_ was called even for newly created internals, when it should only run for fetched (existing) ones. (Identified by @b-pass in code review) 2. Change LeakOnInterpreterShutdown from template param to runtime arg - Renamed to `clear_destructor` to describe what it does locally, rather than embedding assumptions about why it's used. - Reduces template instantiations (header-only library benefits). - The check is in the slow path (create) anyway, so negligible cost. 3. Remove unnecessary braces around the fast-path lookup - The braces created a nested scope but declared no local variables that would benefit from scoping. * Remove unused PYBIND11_MULTIPLE_INTERPRETERS_TEST_FILES variable This variable was defined but never used. --------- Co-authored-by: Ralf W. Grosse-Kunstleve Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/detail/internals.h | 123 ++++++-- include/pybind11/gil_safe_call_once.h | 191 ++++++++++- include/pybind11/pytypes.h | 44 ++- tests/CMakeLists.txt | 40 ++- ...mod_per_interpreter_gil_with_singleton.cpp | 141 +++++++++ tests/test_multiple_interpreters.py | 297 ++++++++++++++++-- tests/test_with_catch/test_subinterpreter.cpp | 122 +++++++ 7 files changed, 890 insertions(+), 68 deletions(-) create mode 100644 tests/mod_per_interpreter_gil_with_singleton.cpp diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 858de6752..95c1f2dfc 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -420,8 +420,8 @@ inline PyThreadState *get_thread_state_unchecked() { /// We use this counter to figure out if there are or have been multiple subinterpreters active at /// any point. This must never decrease while any interpreter may be running in any thread! -inline std::atomic &get_num_interpreters_seen() { - static std::atomic counter(0); +inline std::atomic &get_num_interpreters_seen() { + static std::atomic counter(0); return counter; } @@ -550,6 +550,91 @@ inline object get_python_state_dict() { return state_dict; } +// Get or create per-storage capsule in the current interpreter's state dict. +// - The storage is interpreter-dependent: different interpreters will have different storage. +// This is important when using multiple-interpreters, to avoid sharing unshareable objects +// between interpreters. +// - There is one storage per `key` in an interpreter and it is accessible between all extensions +// in the same interpreter. +// - The life span of the storage is tied to the interpreter: it will be kept alive until the +// interpreter shuts down. +// +// Use test-and-set pattern with `PyDict_SetDefault` for thread-safe concurrent access. +// WARNING: There can be multiple threads creating the storage at the same time, while only one +// will succeed in inserting its capsule into the dict. Therefore, the deleter will be +// used to clean up the storage of the unused capsules. +// +// Returns: pair of (pointer to storage, bool indicating if newly created). +// The bool follows std::map::insert convention: true = created, false = existed. +template +std::pair atomic_get_or_create_in_state_dict(const char *key, + bool clear_destructor = false) { + error_scope err_scope; // preserve any existing Python error states + + auto state_dict = reinterpret_borrow(get_python_state_dict()); + PyObject *capsule_obj = nullptr; + bool created = false; + + // Try to get existing storage (fast path). + capsule_obj = dict_getitemstring(state_dict.ptr(), key); + if (capsule_obj == nullptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } + // Storage doesn't exist yet, create a new one. + // Use unique_ptr for exception safety: if capsule creation throws, the storage is + // automatically deleted. + auto storage_ptr = std::unique_ptr(new Payload{}); + // Create capsule with destructor to clean up when the interpreter shuts down. + auto new_capsule = capsule( + storage_ptr.get(), + // The destructor will be called when the capsule is GC'ed. + // - If our capsule is inserted into the dict below, it will be kept alive until + // interpreter shutdown, so the destructor will be called at that time. + // - If our capsule is NOT inserted (another thread inserted first), it will be + // destructed when going out of scope here, so the destructor will be called + // immediately, which will also free the storage. + /*destructor=*/[](void *ptr) -> void { delete static_cast(ptr); }); + // At this point, the capsule object is created successfully. + // Release the unique_ptr and let the capsule object own the storage to avoid double-free. + (void) storage_ptr.release(); + + // Use `PyDict_SetDefault` for atomic test-and-set: + // - If key doesn't exist, inserts our capsule and returns it. + // - If key exists (another thread inserted first), returns the existing value. + // This is thread-safe because `PyDict_SetDefault` will hold a lock on the dict. + // + // NOTE: Here we use `PyDict_SetDefault` instead of `PyDict_SetDefaultRef` because the + // capsule is kept alive until interpreter shutdown, so we do not need to handle + // incref and decref here. + capsule_obj = dict_setdefaultstring(state_dict.ptr(), key, new_capsule.ptr()); + if (capsule_obj == nullptr) { + throw error_already_set(); + } + created = (capsule_obj == new_capsule.ptr()); + if (clear_destructor && created) { + // Our capsule was inserted. + // Remove the destructor to leak the storage on interpreter shutdown. + if (PyCapsule_SetDestructor(capsule_obj, nullptr) < 0) { + throw error_already_set(); + } + } + // - If key already existed, our `new_capsule` is not inserted, it will be destructed when + // going out of scope here, which will also free the storage. + // - Otherwise, our `new_capsule` is now in the dict, and it owns the storage and the state + // dict will incref it. + } + + // Get the storage pointer from the capsule. + void *raw_ptr = PyCapsule_GetPointer(capsule_obj, /*name=*/nullptr); + if (!raw_ptr) { + raise_from(PyExc_SystemError, + "pybind11::detail::atomic_get_or_create_in_state_dict() FAILED"); + throw error_already_set(); + } + return std::pair(static_cast(raw_ptr), created); +} + template class internals_pp_manager { public: @@ -622,26 +707,18 @@ private: : holder_id_(id), on_fetch_(on_fetch) {} std::unique_ptr *get_or_create_pp_in_state_dict() { - error_scope err_scope; - dict state_dict = get_python_state_dict(); - auto internals_obj - = reinterpret_steal(dict_getitemstringref(state_dict.ptr(), holder_id_)); - std::unique_ptr *pp = nullptr; - if (internals_obj) { - void *raw_ptr = PyCapsule_GetPointer(internals_obj.ptr(), /*name=*/nullptr); - if (!raw_ptr) { - raise_from(PyExc_SystemError, - "pybind11::detail::internals_pp_manager::get_pp_from_dict() FAILED"); - throw error_already_set(); - } - pp = reinterpret_cast *>(raw_ptr); - if (on_fetch_ && pp) { - on_fetch_(pp->get()); - } - } else { - pp = new std::unique_ptr; - // NOLINTNEXTLINE(bugprone-casting-through-void) - state_dict[holder_id_] = capsule(reinterpret_cast(pp)); + // The `unique_ptr` output is leaked on interpreter shutdown. Once an + // instance is created, it will never be deleted until the process exits (compare to + // interpreter shutdown in multiple-interpreter scenarios). + // Because we cannot guarantee the order of destruction of capsules in the interpreter + // state dict, leaking avoids potential use-after-free issues during interpreter shutdown. + auto result = atomic_get_or_create_in_state_dict>( + holder_id_, /*clear_destructor=*/true); + auto *pp = result.first; + bool created = result.second; + // Only call on_fetch_ when fetching existing internals, not when creating new ones. + if (!created && on_fetch_ && pp) { + on_fetch_(pp->get()); } return pp; } @@ -660,6 +737,8 @@ private: char const *holder_id_ = nullptr; on_fetch_function *on_fetch_ = nullptr; + // Pointer-to-pointer to the singleton internals for the first seen interpreter (may not be the + // main interpreter) std::unique_ptr *internals_singleton_pp_; }; diff --git a/include/pybind11/gil_safe_call_once.h b/include/pybind11/gil_safe_call_once.h index 2abd8fc32..3c8ed84df 100644 --- a/include/pybind11/gil_safe_call_once.h +++ b/include/pybind11/gil_safe_call_once.h @@ -3,17 +3,31 @@ #pragma once #include "detail/common.h" +#include "detail/internals.h" #include "gil.h" #include #include -#ifdef Py_GIL_DISABLED +#if defined(Py_GIL_DISABLED) || defined(PYBIND11_HAS_SUBINTERPRETER_SUPPORT) # include #endif +#ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT +# include +# include +# include +#endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) +#if defined(Py_GIL_DISABLED) || defined(PYBIND11_HAS_SUBINTERPRETER_SUPPORT) +using atomic_bool = std::atomic_bool; +#else +using atomic_bool = bool; +#endif +PYBIND11_NAMESPACE_END(detail) + // Use the `gil_safe_call_once_and_store` class below instead of the naive // // static auto imported_obj = py::module_::import("module_name"); // BAD, DO NOT USE! @@ -48,12 +62,23 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) // functions, which is usually the case. // // For in-depth background, see docs/advanced/deadlock.md +#ifndef PYBIND11_HAS_SUBINTERPRETER_SUPPORT +// Subinterpreter support is disabled. +// In this case, we can store the result globally, because there is only a single interpreter. +// +// The life span of the stored result is the entire process lifetime. It is leaked on process +// termination to avoid destructor calls after the Python interpreter was finalized. template class gil_safe_call_once_and_store { public: // PRECONDITION: The GIL must be held when `call_once_and_store_result()` is called. + // + // NOTE: The second parameter (finalize callback) is intentionally unused when subinterpreter + // support is disabled. In that case, storage is process-global and intentionally leaked to + // avoid calling destructors after the Python interpreter has been finalized. template - gil_safe_call_once_and_store &call_once_and_store_result(Callable &&fn) { + gil_safe_call_once_and_store &call_once_and_store_result(Callable &&fn, + void (*)(T &) /*unused*/ = nullptr) { if (!is_initialized_) { // This read is guarded by the GIL. // Multiple threads may enter here, because the GIL is released in the next line and // CPython API calls in the `fn()` call below may release and reacquire the GIL. @@ -74,29 +99,175 @@ public: T &get_stored() { assert(is_initialized_); PYBIND11_WARNING_PUSH -#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5 +# if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5 // Needed for gcc 4.8.5 PYBIND11_WARNING_DISABLE_GCC("-Wstrict-aliasing") -#endif +# endif return *reinterpret_cast(storage_); PYBIND11_WARNING_POP } constexpr gil_safe_call_once_and_store() = default; + // The instance is a global static, so its destructor runs when the process + // is terminating. Therefore, do nothing here because the Python interpreter + // may have been finalized already. PYBIND11_DTOR_CONSTEXPR ~gil_safe_call_once_and_store() = default; + // Disable copy and move operations. + gil_safe_call_once_and_store(const gil_safe_call_once_and_store &) = delete; + gil_safe_call_once_and_store(gil_safe_call_once_and_store &&) = delete; + gil_safe_call_once_and_store &operator=(const gil_safe_call_once_and_store &) = delete; + gil_safe_call_once_and_store &operator=(gil_safe_call_once_and_store &&) = delete; + private: + // The global static storage (per-process) when subinterpreter support is disabled. alignas(T) char storage_[sizeof(T)] = {}; std::once_flag once_flag_; -#ifdef Py_GIL_DISABLED - std::atomic_bool -#else - bool -#endif - is_initialized_{false}; + // The `is_initialized_`-`storage_` pair is very similar to `std::optional`, // but the latter does not have the triviality properties of former, // therefore `std::optional` is not a viable alternative here. + detail::atomic_bool is_initialized_{false}; +}; +#else +// Subinterpreter support is enabled. +// In this case, we should store the result per-interpreter instead of globally, because each +// subinterpreter has its own separate state. The cached result may not shareable across +// interpreters (e.g., imported modules and their members). + +PYBIND11_NAMESPACE_BEGIN(detail) + +template +struct call_once_storage { + alignas(T) char storage[sizeof(T)] = {}; + std::once_flag once_flag; + void (*finalize)(T &) = nullptr; + std::atomic_bool is_initialized{false}; + + call_once_storage() = default; + ~call_once_storage() { + if (is_initialized) { + if (finalize != nullptr) { + finalize(*reinterpret_cast(storage)); + } else { + reinterpret_cast(storage)->~T(); + } + } + } + call_once_storage(const call_once_storage &) = delete; + call_once_storage(call_once_storage &&) = delete; + call_once_storage &operator=(const call_once_storage &) = delete; + call_once_storage &operator=(call_once_storage &&) = delete; }; +PYBIND11_NAMESPACE_END(detail) + +// Prefix for storage keys in the interpreter state dict. +# define PYBIND11_CALL_ONCE_STORAGE_KEY_PREFIX PYBIND11_INTERNALS_ID "_call_once_storage__" + +// The life span of the stored result is the entire interpreter lifetime. An additional +// `finalize_fn` can be provided to clean up the stored result when the interpreter is destroyed. +template +class gil_safe_call_once_and_store { +public: + // PRECONDITION: The GIL must be held when `call_once_and_store_result()` is called. + template + gil_safe_call_once_and_store &call_once_and_store_result(Callable &&fn, + void (*finalize_fn)(T &) = nullptr) { + if (!is_last_storage_valid()) { + // Multiple threads may enter here, because the GIL is released in the next line and + // CPython API calls in the `fn()` call below may release and reacquire the GIL. + gil_scoped_release gil_rel; // Needed to establish lock ordering. + // There can be multiple threads going through here. + storage_type *value = nullptr; + { + gil_scoped_acquire gil_acq; // Restore lock ordering. + // This function is thread-safe under free-threading. + value = get_or_create_storage_in_state_dict(); + } + assert(value != nullptr); + std::call_once(value->once_flag, [&] { + // Only one thread will ever enter here. + gil_scoped_acquire gil_acq; + // fn may release, but will reacquire, the GIL. + ::new (value->storage) T(fn()); + value->finalize = finalize_fn; + value->is_initialized = true; + last_storage_ptr_ = reinterpret_cast(value->storage); + is_initialized_by_at_least_one_interpreter_ = true; + }); + // All threads will observe `is_initialized_by_at_least_one_interpreter_` as true here. + } + // Intentionally not returning `T &` to ensure the calling code is self-documenting. + return *this; + } + + // This must only be called after `call_once_and_store_result()` was called. + T &get_stored() { + T *result = last_storage_ptr_; + if (!is_last_storage_valid()) { + gil_scoped_acquire gil_acq; + auto *value = get_or_create_storage_in_state_dict(); + result = last_storage_ptr_ = reinterpret_cast(value->storage); + } + assert(result != nullptr); + return *result; + } + + constexpr gil_safe_call_once_and_store() = default; + // The instance is a global static, so its destructor runs when the process + // is terminating. Therefore, do nothing here because the Python interpreter + // may have been finalized already. + PYBIND11_DTOR_CONSTEXPR ~gil_safe_call_once_and_store() = default; + + // Disable copy and move operations because the memory address is used as key. + gil_safe_call_once_and_store(const gil_safe_call_once_and_store &) = delete; + gil_safe_call_once_and_store(gil_safe_call_once_and_store &&) = delete; + gil_safe_call_once_and_store &operator=(const gil_safe_call_once_and_store &) = delete; + gil_safe_call_once_and_store &operator=(gil_safe_call_once_and_store &&) = delete; + +private: + using storage_type = detail::call_once_storage; + + // Indicator of fast path for single-interpreter case. + bool is_last_storage_valid() const { + return is_initialized_by_at_least_one_interpreter_ + && detail::get_num_interpreters_seen() == 1; + } + + // Get the unique key for this storage instance in the interpreter's state dict. + // The return type should not be `py::str` because PyObject is interpreter-dependent. + std::string get_storage_key() const { + // The instance is expected to be global static, so using its address as unique identifier. + // The typical usage is like: + // + // PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; + // + return PYBIND11_CALL_ONCE_STORAGE_KEY_PREFIX + + std::to_string(reinterpret_cast(this)); + } + + // Get or create per-storage capsule in the current interpreter's state dict. + // The storage is interpreter-dependent and will not be shared across interpreters. + storage_type *get_or_create_storage_in_state_dict() { + return detail::atomic_get_or_create_in_state_dict(get_storage_key().c_str()) + .first; + } + + // No storage needed when subinterpreter support is enabled. + // The actual storage is stored in the per-interpreter state dict via + // `get_or_create_storage_in_state_dict()`. + + // Fast local cache to avoid repeated lookups when there are no multiple interpreters. + // This is only valid if there is a single interpreter. Otherwise, it is not used. + // WARNING: We cannot use thread local cache similar to `internals_pp_manager::internals_p_tls` + // because the thread local storage cannot be explicitly invalidated when interpreters + // are destroyed (unlike `internals_pp_manager` which has explicit hooks for that). + T *last_storage_ptr_ = nullptr; + // This flag is true if the value has been initialized by any interpreter (may not be the + // current one). + detail::atomic_bool is_initialized_by_at_least_one_interpreter_{false}; +}; +#endif + PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 0ab0b73e1..eb02946a8 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -997,8 +997,10 @@ inline PyObject *dict_getitem(PyObject *v, PyObject *key) { return rv; } +// PyDict_GetItemStringRef was added in Python 3.13.0a1. +// See also: https://github.com/python/pythoncapi-compat/blob/main/pythoncapi_compat.h inline PyObject *dict_getitemstringref(PyObject *v, const char *key) { -#if PY_VERSION_HEX >= 0x030D0000 +#if PY_VERSION_HEX >= 0x030D00A1 PyObject *rv = nullptr; if (PyDict_GetItemStringRef(v, key, &rv) < 0) { throw error_already_set(); @@ -1014,6 +1016,46 @@ inline PyObject *dict_getitemstringref(PyObject *v, const char *key) { #endif } +inline PyObject *dict_setdefaultstring(PyObject *v, const char *key, PyObject *defaultobj) { + PyObject *kv = PyUnicode_FromString(key); + if (kv == nullptr) { + throw error_already_set(); + } + + PyObject *rv = PyDict_SetDefault(v, kv, defaultobj); + Py_DECREF(kv); + if (rv == nullptr) { + throw error_already_set(); + } + return rv; +} + +// PyDict_SetDefaultRef was added in Python 3.13.0a4. +// See also: https://github.com/python/pythoncapi-compat/blob/main/pythoncapi_compat.h +inline PyObject *dict_setdefaultstringref(PyObject *v, const char *key, PyObject *defaultobj) { +#if PY_VERSION_HEX >= 0x030D00A4 + PyObject *kv = PyUnicode_FromString(key); + if (kv == nullptr) { + throw error_already_set(); + } + + PyObject *rv = nullptr; + if (PyDict_SetDefaultRef(v, kv, defaultobj, &rv) < 0) { + Py_DECREF(kv); + throw error_already_set(); + } + Py_DECREF(kv); + return rv; +#else + PyObject *rv = dict_setdefaultstring(v, key, defaultobj); + if (rv == nullptr || PyErr_Occurred()) { + throw error_already_set(); + } + Py_XINCREF(rv); + return rv; +#endif +} + // Helper aliases/functions to support implicit casting of values given to python // accessors/methods. When given a pyobject, this simply returns the pyobject as-is; for other C++ // type, the value goes through pybind11::cast(obj) to convert it to an `object`. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 47ba4aa86..3d618662f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -578,24 +578,32 @@ add_custom_target( USES_TERMINAL) if(NOT PYBIND11_CUDA_TESTS) - # This module doesn't get mixed with other test modules because those aren't subinterpreter safe. - pybind11_add_module(mod_per_interpreter_gil THIN_LTO mod_per_interpreter_gil.cpp) - pybind11_add_module(mod_shared_interpreter_gil THIN_LTO mod_shared_interpreter_gil.cpp) - set_target_properties(mod_per_interpreter_gil PROPERTIES LIBRARY_OUTPUT_DIRECTORY - "$<1:${CMAKE_CURRENT_BINARY_DIR}>") - set_target_properties(mod_shared_interpreter_gil PROPERTIES LIBRARY_OUTPUT_DIRECTORY - "$<1:${CMAKE_CURRENT_BINARY_DIR}>") + set(PYBIND11_MULTIPLE_INTERPRETERS_TEST_MODULES + mod_per_interpreter_gil mod_shared_interpreter_gil mod_per_interpreter_gil_with_singleton) + + # These modules don't get mixed with other test modules because those aren't subinterpreter safe. + foreach(mod IN LISTS PYBIND11_MULTIPLE_INTERPRETERS_TEST_MODULES) + pybind11_add_module("${mod}" THIN_LTO "${mod}.cpp") + pybind11_enable_warnings("${mod}") + endforeach() + + # Put the built modules next to `pybind11_tests.so` so that the test scripts can find them. + get_target_property(pybind11_tests_output_directory pybind11_tests LIBRARY_OUTPUT_DIRECTORY) + foreach(mod IN LISTS PYBIND11_MULTIPLE_INTERPRETERS_TEST_MODULES) + set_target_properties("${mod}" PROPERTIES LIBRARY_OUTPUT_DIRECTORY + "${pybind11_tests_output_directory}") + endforeach() + if(PYBIND11_TEST_SMART_HOLDER) - target_compile_definitions( - mod_per_interpreter_gil - PUBLIC -DPYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE - ) - target_compile_definitions( - mod_shared_interpreter_gil - PUBLIC -DPYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE - ) + foreach(mod IN LISTS PYBIND11_MULTIPLE_INTERPRETERS_TEST_MODULES) + target_compile_definitions( + "${mod}" + PUBLIC + -DPYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE) + endforeach() endif() - add_dependencies(pytest mod_per_interpreter_gil mod_shared_interpreter_gil) + + add_dependencies(pytest ${PYBIND11_MULTIPLE_INTERPRETERS_TEST_MODULES}) endif() if(PYBIND11_TEST_OVERRIDE) diff --git a/tests/mod_per_interpreter_gil_with_singleton.cpp b/tests/mod_per_interpreter_gil_with_singleton.cpp new file mode 100644 index 000000000..874b93c77 --- /dev/null +++ b/tests/mod_per_interpreter_gil_with_singleton.cpp @@ -0,0 +1,141 @@ +#include +#include + +#include + +namespace py = pybind11; + +#ifdef PYBIND11_HAS_NATIVE_ENUM +# include +#endif + +// A singleton class that holds references to certain Python objects +// This singleton is per-interpreter using gil_safe_call_once_and_store +class MySingleton { +public: + MySingleton() = default; + ~MySingleton() = default; + MySingleton(const MySingleton &) = delete; + MySingleton &operator=(const MySingleton &) = delete; + MySingleton(MySingleton &&) = default; + MySingleton &operator=(MySingleton &&) = default; + + static MySingleton &get_instance() { + PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store storage; + return storage + .call_once_and_store_result([]() -> MySingleton { + MySingleton instance{}; + + auto emplace = [&instance](const py::handle &obj) -> void { + obj.inc_ref(); // Ensure the object is not GC'd while interpreter is alive + instance.objects.emplace_back(obj); + }; + + // Example objects to store in the singleton + emplace(py::type::handle_of(py::none())); // static type + emplace(py::type::handle_of(py::tuple())); // static type + emplace(py::type::handle_of(py::list())); // static type + emplace(py::type::handle_of(py::dict())); // static type + emplace(py::module_::import("collections").attr("OrderedDict")); // static type + emplace(py::module_::import("collections").attr("defaultdict")); // heap type + emplace(py::module_::import("collections").attr("deque")); // heap type + + assert(instance.objects.size() == 7); + return instance; + }) + .get_stored(); + } + + std::vector &get_objects() { return objects; } + + static void init() { + // Ensure the singleton is created + auto &instance = get_instance(); + (void) instance; // suppress unused variable warning + assert(instance.objects.size() == 7); + // Register cleanup at interpreter exit + py::module_::import("atexit").attr("register")(py::cpp_function(&MySingleton::clear)); + } + + static void clear() { + auto &instance = get_instance(); + (void) instance; // suppress unused variable warning + assert(instance.objects.size() == 7); + for (const auto &obj : instance.objects) { + obj.dec_ref(); + } + instance.objects.clear(); + } + +private: + std::vector objects; +}; + +class MyClass { +public: + explicit MyClass(py::ssize_t v) : value(v) {} + py::ssize_t get_value() const { return value; } + +private: + py::ssize_t value; +}; + +class MyGlobalError : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +class MyLocalError : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +enum class MyEnum : int { + ONE = 1, + TWO = 2, + THREE = 3, +}; + +PYBIND11_MODULE(mod_per_interpreter_gil_with_singleton, + m, + py::mod_gil_not_used(), + py::multiple_interpreters::per_interpreter_gil()) { +#ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT + m.attr("defined_PYBIND11_HAS_SUBINTERPRETER_SUPPORT") = true; +#else + m.attr("defined_PYBIND11_HAS_SUBINTERPRETER_SUPPORT") = false; +#endif + + MySingleton::init(); + + // Ensure py::multiple_interpreters::per_interpreter_gil() works with singletons using + // py::gil_safe_call_once_and_store + m.def( + "get_objects_in_singleton", + []() -> std::vector { return MySingleton::get_instance().get_objects(); }, + "Get the list of objects stored in the singleton"); + + // Ensure py::multiple_interpreters::per_interpreter_gil() works with class bindings + py::class_(m, "MyClass") + .def(py::init()) + .def("get_value", &MyClass::get_value); + + // Ensure py::multiple_interpreters::per_interpreter_gil() works with global exceptions + py::register_exception(m, "MyGlobalError"); + // Ensure py::multiple_interpreters::per_interpreter_gil() works with local exceptions + py::register_local_exception(m, "MyLocalError"); + +#ifdef PYBIND11_HAS_NATIVE_ENUM + // Ensure py::multiple_interpreters::per_interpreter_gil() works with native_enum + py::native_enum(m, "MyEnum", "enum.IntEnum") + .value("ONE", MyEnum::ONE) + .value("TWO", MyEnum::TWO) + .value("THREE", MyEnum::THREE) + .finalize(); +#else + py::enum_(m, "MyEnum") + .value("ONE", MyEnum::ONE) + .value("TWO", MyEnum::TWO) + .value("THREE", MyEnum::THREE); +#endif +} diff --git a/tests/test_multiple_interpreters.py b/tests/test_multiple_interpreters.py index 627ccc591..f706e9282 100644 --- a/tests/test_multiple_interpreters.py +++ b/tests/test_multiple_interpreters.py @@ -3,10 +3,14 @@ from __future__ import annotations import contextlib import os import pickle +import subprocess import sys +import textwrap import pytest +import pybind11_tests + # 3.14.0b3+, though sys.implementation.supports_isolated_interpreters is being added in b4 # Can be simplified when we drop support for the first three betas CONCURRENT_INTERPRETERS_SUPPORT = ( @@ -82,7 +86,7 @@ def get_interpreters(*, modern: bool): def test_independent_subinterpreters(): """Makes sure the internals object differs across independent subinterpreters""" - sys.path.append(".") + sys.path.insert(0, os.path.dirname(pybind11_tests.__file__)) run_string, create = get_interpreters(modern=True) @@ -91,12 +95,14 @@ def test_independent_subinterpreters(): if not m.defined_PYBIND11_HAS_SUBINTERPRETER_SUPPORT: pytest.skip("Does not have subinterpreter support compiled in") - code = """ -import mod_per_interpreter_gil as m -import pickle -with open(pipeo, 'wb') as f: - pickle.dump(m.internals_at(), f) -""" + code = textwrap.dedent( + """ + import mod_per_interpreter_gil as m + import pickle + with open(pipeo, 'wb') as f: + pickle.dump(m.internals_at(), f) + """ + ).strip() with create() as interp1, create() as interp2: try: @@ -131,7 +137,7 @@ with open(pipeo, 'wb') as f: def test_independent_subinterpreters_modern(): """Makes sure the internals object differs across independent subinterpreters. Modern (3.14+) syntax.""" - sys.path.append(".") + sys.path.insert(0, os.path.dirname(pybind11_tests.__file__)) m = pytest.importorskip("mod_per_interpreter_gil") @@ -140,11 +146,13 @@ def test_independent_subinterpreters_modern(): from concurrent import interpreters - code = """ -import mod_per_interpreter_gil as m + code = textwrap.dedent( + """ + import mod_per_interpreter_gil as m -values.put_nowait(m.internals_at()) -""" + values.put_nowait(m.internals_at()) + """ + ).strip() with contextlib.closing(interpreters.create()) as interp1, contextlib.closing( interpreters.create() @@ -175,7 +183,7 @@ values.put_nowait(m.internals_at()) def test_dependent_subinterpreters(): """Makes sure the internals object differs across subinterpreters""" - sys.path.append(".") + sys.path.insert(0, os.path.dirname(pybind11_tests.__file__)) run_string, create = get_interpreters(modern=False) @@ -184,12 +192,14 @@ def test_dependent_subinterpreters(): if not m.defined_PYBIND11_HAS_SUBINTERPRETER_SUPPORT: pytest.skip("Does not have subinterpreter support compiled in") - code = """ -import mod_shared_interpreter_gil as m -import pickle -with open(pipeo, 'wb') as f: - pickle.dump(m.internals_at(), f) -""" + code = textwrap.dedent( + """ + import mod_shared_interpreter_gil as m + import pickle + with open(pipeo, 'wb') as f: + pickle.dump(m.internals_at(), f) + """ + ).strip() with create("legacy") as interp1: pipei, pipeo = os.pipe() @@ -198,3 +208,252 @@ with open(pipeo, 'wb') as f: res1 = pickle.load(f) assert res1 != m.internals_at(), "internals should differ from main interpreter" + + +PREAMBLE_CODE = textwrap.dedent( + f""" + def test(): + import sys + + sys.path.insert(0, {os.path.dirname(pybind11_tests.__file__)!r}) + + import collections + import mod_per_interpreter_gil_with_singleton as m + + objects = m.get_objects_in_singleton() + expected = [ + type(None), + tuple, + list, + dict, + collections.OrderedDict, + collections.defaultdict, + collections.deque, + ] + assert objects == expected, f"Expected {{expected!r}}, got {{objects!r}}." + + assert hasattr(m, 'MyClass'), "Module missing MyClass" + assert hasattr(m, 'MyGlobalError'), "Module missing MyGlobalError" + assert hasattr(m, 'MyLocalError'), "Module missing MyLocalError" + assert hasattr(m, 'MyEnum'), "Module missing MyEnum" + """ +).lstrip() + + +@pytest.mark.xfail( + reason="Duplicate C++ type registration under multiple-interpreters, needs investigation.", + # raises=interpreters.ExecutionFailed, # need to import the module + strict=False, +) +@pytest.mark.skipif( + sys.platform.startswith("emscripten"), reason="Requires loadable modules" +) +@pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+") +def test_import_module_with_singleton_per_interpreter(): + """Tests that a singleton storing Python objects works correctly per-interpreter""" + from concurrent import interpreters + + code = f"{PREAMBLE_CODE.strip()}\n\ntest()\n" + with contextlib.closing(interpreters.create()) as interp: + interp.exec(code) + + +def check_script_success_in_subprocess(code: str, *, rerun: int = 8) -> None: + """Runs the given code in a subprocess.""" + code = textwrap.dedent(code).strip() + try: + for _ in range(rerun): # run flakily failing test multiple times + subprocess.check_output( + [sys.executable, "-c", code], + cwd=os.getcwd(), + stderr=subprocess.STDOUT, + text=True, + ) + except subprocess.CalledProcessError as ex: + raise RuntimeError( + f"Subprocess failed with exit code {ex.returncode}.\n\n" + f"Code:\n" + f"```python\n" + f"{code}\n" + f"```\n\n" + f"Output:\n" + f"{ex.output}" + ) from ex + + +@pytest.mark.xfail( + reason="Duplicate C++ type registration under multiple-interpreters, needs investigation.", + raises=RuntimeError, + strict=False, +) +@pytest.mark.skipif( + sys.platform.startswith("emscripten"), reason="Requires loadable modules" +) +@pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+") +def test_import_in_subinterpreter_after_main(): + """Tests that importing a module in a subinterpreter after the main interpreter works correctly""" + check_script_success_in_subprocess( + PREAMBLE_CODE + + textwrap.dedent( + """ + import contextlib + import gc + from concurrent import interpreters + + test() + + interp = None + with contextlib.closing(interpreters.create()) as interp: + interp.call(test) + + del interp + for _ in range(5): + gc.collect() + """ + ) + ) + + check_script_success_in_subprocess( + PREAMBLE_CODE + + textwrap.dedent( + """ + import contextlib + import gc + import random + from concurrent import interpreters + + test() + + interps = interp = None + with contextlib.ExitStack() as stack: + interps = [ + stack.enter_context(contextlib.closing(interpreters.create())) + for _ in range(8) + ] + random.shuffle(interps) + for interp in interps: + interp.call(test) + + del interps, interp, stack + for _ in range(5): + gc.collect() + """ + ) + ) + + +@pytest.mark.xfail( + reason="Duplicate C++ type registration under multiple-interpreters, needs investigation.", + raises=RuntimeError, + strict=False, +) +@pytest.mark.skipif( + sys.platform.startswith("emscripten"), reason="Requires loadable modules" +) +@pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+") +def test_import_in_subinterpreter_before_main(): + """Tests that importing a module in a subinterpreter before the main interpreter works correctly""" + check_script_success_in_subprocess( + PREAMBLE_CODE + + textwrap.dedent( + """ + import contextlib + import gc + from concurrent import interpreters + + interp = None + with contextlib.closing(interpreters.create()) as interp: + interp.call(test) + + test() + + del interp + for _ in range(5): + gc.collect() + """ + ) + ) + + check_script_success_in_subprocess( + PREAMBLE_CODE + + textwrap.dedent( + """ + import contextlib + import gc + from concurrent import interpreters + + interps = interp = None + with contextlib.ExitStack() as stack: + interps = [ + stack.enter_context(contextlib.closing(interpreters.create())) + for _ in range(8) + ] + for interp in interps: + interp.call(test) + + test() + + del interps, interp, stack + for _ in range(5): + gc.collect() + """ + ) + ) + + check_script_success_in_subprocess( + PREAMBLE_CODE + + textwrap.dedent( + """ + import contextlib + import gc + from concurrent import interpreters + + interps = interp = None + with contextlib.ExitStack() as stack: + interps = [ + stack.enter_context(contextlib.closing(interpreters.create())) + for _ in range(8) + ] + for interp in interps: + interp.call(test) + + test() + + del interps, interp, stack + for _ in range(5): + gc.collect() + """ + ) + ) + + +@pytest.mark.xfail( + reason="Duplicate C++ type registration under multiple-interpreters, needs investigation.", + raises=RuntimeError, + strict=False, +) +@pytest.mark.skipif( + sys.platform.startswith("emscripten"), reason="Requires loadable modules" +) +@pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+") +def test_import_in_subinterpreter_concurrently(): + """Tests that importing a module in multiple subinterpreters concurrently works correctly""" + check_script_success_in_subprocess( + PREAMBLE_CODE + + textwrap.dedent( + """ + import gc + from concurrent.futures import InterpreterPoolExecutor, as_completed + + futures = future = None + with InterpreterPoolExecutor(max_workers=16) as executor: + futures = [executor.submit(test) for _ in range(32)] + for future in as_completed(futures): + future.result() + del futures, future, executor + + for _ in range(5): + gc.collect() + """ + ) + ) diff --git a/tests/test_with_catch/test_subinterpreter.cpp b/tests/test_with_catch/test_subinterpreter.cpp index 278313388..b17b6fbc3 100644 --- a/tests/test_with_catch/test_subinterpreter.cpp +++ b/tests/test_with_catch/test_subinterpreter.cpp @@ -1,5 +1,6 @@ #include #ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT +# include # include // Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to @@ -41,6 +42,30 @@ void unsafe_reset_internals_for_single_interpreter() { py::detail::get_local_internals(); } +py::object &get_dict_type_object() { + PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store storage; + return storage + .call_once_and_store_result( + []() -> py::object { return py::module_::import("builtins").attr("dict"); }) + .get_stored(); +} + +py::object &get_ordered_dict_type_object() { + PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store storage; + return storage + .call_once_and_store_result( + []() -> py::object { return py::module_::import("collections").attr("OrderedDict"); }) + .get_stored(); +} + +py::object &get_default_dict_type_object() { + PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store storage; + return storage + .call_once_and_store_result( + []() -> py::object { return py::module_::import("collections").attr("defaultdict"); }) + .get_stored(); +} + TEST_CASE("Single Subinterpreter") { unsafe_reset_internals_for_single_interpreter(); @@ -308,6 +333,103 @@ TEST_CASE("Multiple Subinterpreters") { unsafe_reset_internals_for_single_interpreter(); } +// Test that gil_safe_call_once_and_store provides per-interpreter storage. +// Without the per-interpreter storage fix, the subinterpreter would see the value +// cached by the main interpreter, which is invalid (different interpreter's object). +TEST_CASE("gil_safe_call_once_and_store per-interpreter isolation") { + unsafe_reset_internals_for_single_interpreter(); + + // This static simulates a typical usage pattern where a module caches + // an imported object using gil_safe_call_once_and_store. + PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store storage; + + // Get the interpreter ID in the main interpreter + auto main_interp_id = PyInterpreterState_GetID(PyInterpreterState_Get()); + + // Store a value in the main interpreter - we'll store the interpreter ID as a Python int + auto &main_value = storage + .call_once_and_store_result([]() { + return py::int_(PyInterpreterState_GetID(PyInterpreterState_Get())); + }) + .get_stored(); + REQUIRE(main_value.cast() == main_interp_id); + + py::object dict_type = get_dict_type_object(); + py::object ordered_dict_type = get_ordered_dict_type_object(); + py::object default_dict_type = get_default_dict_type_object(); + + int64_t sub_interp_id = -1; + int64_t sub_cached_value = -1; + + bool sub_default_dict_type_destroyed = false; + + // Create a subinterpreter and check that it gets its own storage + { + py::scoped_subinterpreter ssi; + + sub_interp_id = PyInterpreterState_GetID(PyInterpreterState_Get()); + REQUIRE(sub_interp_id != main_interp_id); + + // Access the same static storage from the subinterpreter. + // With per-interpreter storage, this should call the lambda again + // and cache a NEW value for this interpreter. + // Without per-interpreter storage, this would return main's cached value. + auto &sub_value + = storage + .call_once_and_store_result([]() { + return py::int_(PyInterpreterState_GetID(PyInterpreterState_Get())); + }) + .get_stored(); + + sub_cached_value = sub_value.cast(); + + // The cached value should be the SUBINTERPRETER's ID, not the main interpreter's. + // This would fail without per-interpreter storage. + REQUIRE(sub_cached_value == sub_interp_id); + REQUIRE(sub_cached_value != main_interp_id); + + py::object sub_dict_type = get_dict_type_object(); + py::object sub_ordered_dict_type = get_ordered_dict_type_object(); + py::object sub_default_dict_type = get_default_dict_type_object(); + + // Verify that the subinterpreter has its own cached type objects. + // For static types, they should be the same object across interpreters. + // See also: https://docs.python.org/3/c-api/typeobj.html#static-types + REQUIRE(sub_dict_type.is(dict_type)); // dict is a static type + REQUIRE(sub_ordered_dict_type.is(ordered_dict_type)); // OrderedDict is a static type + // For heap types, they are dynamically created per-interpreter. + // See also: https://docs.python.org/3/c-api/typeobj.html#heap-types + REQUIRE_FALSE(sub_default_dict_type.is(default_dict_type)); // defaultdict is a heap type + + // Set up a weakref callback to detect when the subinterpreter's cached default_dict_type + // is destroyed so the gil_safe_call_once_and_store storage is not leaked when the + // subinterpreter is shutdown. + (void) py::weakref(sub_default_dict_type, + py::cpp_function([&](py::handle weakref) -> void { + sub_default_dict_type_destroyed = true; + weakref.dec_ref(); + })) + .release(); + } + + // Back in main interpreter, verify main's value is unchanged + auto &main_value_after = storage.get_stored(); + REQUIRE(main_value_after.cast() == main_interp_id); + + // Verify that the types cached in main are unchanged + py::object dict_type_after = get_dict_type_object(); + py::object ordered_dict_type_after = get_ordered_dict_type_object(); + py::object default_dict_type_after = get_default_dict_type_object(); + REQUIRE(dict_type_after.is(dict_type)); + REQUIRE(ordered_dict_type_after.is(ordered_dict_type)); + REQUIRE(default_dict_type_after.is(default_dict_type)); + + // Verify that the subinterpreter's cached default_dict_type was destroyed + REQUIRE(sub_default_dict_type_destroyed); + + unsafe_reset_internals_for_single_interpreter(); +} + # ifdef Py_MOD_PER_INTERPRETER_GIL_SUPPORTED TEST_CASE("Per-Subinterpreter GIL") { auto main_int From fee2527dfa05dc9157e71a6d3a9d57bc42efcde4 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sat, 27 Dec 2025 02:59:11 +0800 Subject: [PATCH 16/30] Fix concurrency consistency for `internals_pp_manager` under multiple-interpreters (#5947) * Add per-interpreter storage for `gil_safe_call_once_and_store` * Disable thread local cache for `internals_pp_manager` * Disable thread local cache for `internals_pp_manager` for multi-interpreter only * Use anonymous namespace to separate these type_ids from other tests with the same class names. * style: pre-commit fixes * Revert internals_pp_manager changes * This is the crux of fix for the subinterpreter_before_main failure. The pre_init needs to check if it is in a subinterpreter or not. But in 3.13+ this static initializer runs in the main interpreter. So we need to check this later, during the exec phase. * Continue to do the ensure in both places, there might be a reason it was where it was... Should not hurt anything to do it extra times here. * Change get_num_interpreters_seen to a boolean flag instead. The count was not used, it was just checked for > 1, we now accomplish this by setting the flag. * Spelling typo * Work around older python versions, only need this check for newish versions * Add more comments for test case * Add more comments for test case * Stop traceback propagation * Re-enable subinterpreter support on ubuntu 3.14 builds Was disabled in e4873e8 * As suggested, don't use an anonymous namespace. * Typo in test assert format string * Use a more appropriate function name * Fix mod_per_interpreter_gil* output directory on Windows/MSVC On Windows with MSVC (multi-configuration generators), CMake uses config-specific properties like LIBRARY_OUTPUT_DIRECTORY_DEBUG when set, otherwise falls back to LIBRARY_OUTPUT_DIRECTORY//. The main test modules (pybind11_tests, etc.) correctly set both LIBRARY_OUTPUT_DIRECTORY and the config-specific variants (lines 517-528), so they output directly to tests/. However, the mod_per_interpreter_gil* modules only copied the base LIBRARY_OUTPUT_DIRECTORY property, causing them to be placed in tests/Debug/ instead of tests/. This mismatch caused test_import_in_subinterpreter_concurrently and related tests to fail with ModuleNotFoundError on Windows Python 3.14, because the test code sets sys.path based on pybind11_tests.__file__ (which is in tests/) but tries to import mod_per_interpreter_gil_with_singleton (which ended up in tests/Debug/). This bug was previously masked by @pytest.mark.xfail decorators on these tests. Now that the underlying "Duplicate C++ type registration" issue is fixed and the xfails are removed, this path issue surfaced. The fix mirrors the same pattern used for main test targets: also set LIBRARY_OUTPUT_DIRECTORY_ for each configuration type. * Remove unneeded `pytest.importorskip` * Remove comment --------- Co-authored-by: b-pass Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- .github/workflows/ci.yml | 2 +- include/pybind11/detail/common.h | 9 ++- include/pybind11/detail/internals.h | 27 ++++++--- include/pybind11/embed.h | 14 ++--- include/pybind11/gil_safe_call_once.h | 2 +- include/pybind11/subinterpreter.h | 2 +- tests/CMakeLists.txt | 8 +++ ...mod_per_interpreter_gil_with_singleton.cpp | 6 ++ tests/test_multiple_interpreters.py | 56 ++++++++----------- tests/test_with_catch/test_subinterpreter.cpp | 2 +- 10 files changed, 72 insertions(+), 56 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5569e3a9..22c1f489c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,7 +88,7 @@ jobs: cmake-args: -DCMAKE_CXX_STANDARD=20 -DPYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION=ON - runs-on: ubuntu-latest python-version: '3.14' - cmake-args: -DCMAKE_CXX_STANDARD=14 -DCMAKE_CXX_FLAGS="-DPYBIND11_HAS_SUBINTERPRETER_SUPPORT=0" + cmake-args: -DCMAKE_CXX_STANDARD=14 - runs-on: ubuntu-latest python-version: 'pypy-3.10' cmake-args: -DCMAKE_CXX_STANDARD=14 diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index ce9014a7e..4406adf5b 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -441,12 +441,11 @@ Note that this is run once for each (sub-)interpreter the module is imported int possibly concurrently. The PyModuleDef is allowed to be static, but the PyObject* resulting from PyModuleDef_Init should be treated like any other PyObject (so not shared across interpreters). */ -#define PYBIND11_MODULE_PYINIT(name, pre_init, ...) \ +#define PYBIND11_MODULE_PYINIT(name, ...) \ static int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject *); \ PYBIND11_PLUGIN_IMPL(name) { \ PYBIND11_CHECK_PYTHON_VERSION \ - pre_init; \ - PYBIND11_ENSURE_INTERNALS_READY \ + pybind11::detail::ensure_internals(); \ static ::pybind11::detail::slots_array mod_def_slots = ::pybind11::detail::init_slots( \ &PYBIND11_CONCAT(pybind11_exec_, name), ##__VA_ARGS__); \ static PyModuleDef def{/* m_base */ PyModuleDef_HEAD_INIT, \ @@ -465,6 +464,7 @@ PyModuleDef_Init should be treated like any other PyObject (so not shared across static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &); \ int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject * pm) { \ try { \ + pybind11::detail::ensure_internals(); \ auto m = pybind11::reinterpret_borrow<::pybind11::module_>(pm); \ if (!pybind11::detail::get_cached_module(m.attr("__spec__").attr("name"))) { \ PYBIND11_CONCAT(pybind11_init_, name)(m); \ @@ -518,8 +518,7 @@ PyModuleDef_Init should be treated like any other PyObject (so not shared across \endrst */ #define PYBIND11_MODULE(name, variable, ...) \ - PYBIND11_MODULE_PYINIT( \ - name, (pybind11::detail::get_num_interpreters_seen() += 1), ##__VA_ARGS__) \ + PYBIND11_MODULE_PYINIT(name, ##__VA_ARGS__) \ PYBIND11_MODULE_EXEC(name, variable) // pop gnu-zero-variadic-macro-arguments diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 95c1f2dfc..a92f196b1 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -418,11 +418,12 @@ inline PyThreadState *get_thread_state_unchecked() { #endif } -/// We use this counter to figure out if there are or have been multiple subinterpreters active at -/// any point. This must never decrease while any interpreter may be running in any thread! -inline std::atomic &get_num_interpreters_seen() { - static std::atomic counter(0); - return counter; +/// We use this to figure out if there are or have been multiple subinterpreters active at any +/// point. This must never go from true to false while any interpreter may be running in any +/// thread! +inline std::atomic_bool &has_seen_non_main_interpreter() { + static std::atomic_bool multi(false); + return multi; } template *get_pp() { #ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT - if (get_num_interpreters_seen() > 1) { + if (has_seen_non_main_interpreter()) { // Whenever the interpreter changes on the current thread we need to invalidate the // internals_pp so that it can be pulled from the interpreter's state dict. That is // slow, so we use the current PyThreadState to check if it is necessary. @@ -675,7 +676,7 @@ public: /// Drop all the references we're currently holding. void unref() { #ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT - if (get_num_interpreters_seen() > 1) { + if (has_seen_non_main_interpreter()) { last_istate_tls() = nullptr; internals_p_tls() = nullptr; return; @@ -686,7 +687,7 @@ public: void destroy() { #ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT - if (get_num_interpreters_seen() > 1) { + if (has_seen_non_main_interpreter()) { auto *tstate = get_thread_state_unchecked(); // this could be called without an active interpreter, just use what was cached if (!tstate || tstate->interp == last_istate_tls()) { @@ -791,6 +792,16 @@ PYBIND11_NOINLINE internals &get_internals() { return *internals_ptr; } +inline void ensure_internals() { + pybind11::detail::get_internals_pp_manager().unref(); +#ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT + if (PyInterpreterState_Get() != PyInterpreterState_Main()) { + has_seen_non_main_interpreter() = true; + } +#endif + pybind11::detail::get_internals(); +} + inline internals_pp_manager &get_local_internals_pp_manager() { // Use the address of this static itself as part of the key, so that the value is uniquely tied // to where the module is loaded in memory diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index a820bfbfc..c05887c33 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -58,7 +58,7 @@ PYBIND11_WARNING_PUSH PYBIND11_WARNING_DISABLE_CLANG("-Wgnu-zero-variadic-macro-arguments") #define PYBIND11_EMBEDDED_MODULE(name, variable, ...) \ - PYBIND11_MODULE_PYINIT(name, {}, ##__VA_ARGS__) \ + PYBIND11_MODULE_PYINIT(name, ##__VA_ARGS__) \ ::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name)( \ PYBIND11_TOSTRING(name), PYBIND11_CONCAT(PyInit_, name)); \ PYBIND11_MODULE_EXEC(name, variable) @@ -202,7 +202,7 @@ inline void initialize_interpreter(bool init_signal_handlers = true, #endif // There is exactly one interpreter alive currently. - detail::get_num_interpreters_seen() = 1; + detail::has_seen_non_main_interpreter() = false; } /** \rst @@ -242,12 +242,12 @@ inline void initialize_interpreter(bool init_signal_handlers = true, \endrst */ inline void finalize_interpreter() { // get rid of any thread-local interpreter cache that currently exists - if (detail::get_num_interpreters_seen() > 1) { + if (detail::has_seen_non_main_interpreter()) { detail::get_internals_pp_manager().unref(); detail::get_local_internals_pp_manager().unref(); - // We know there can be no other interpreter alive now, so we can lower the count - detail::get_num_interpreters_seen() = 1; + // We know there can be no other interpreter alive now + detail::has_seen_non_main_interpreter() = false; } // Re-fetch the internals pointer-to-pointer (but not the internals itself, which might not @@ -265,8 +265,8 @@ inline void finalize_interpreter() { // avoid undefined behaviors when initializing another interpreter detail::get_local_internals_pp_manager().destroy(); - // We know there is no interpreter alive now, so we can reset the count - detail::get_num_interpreters_seen() = 0; + // We know there is no interpreter alive now, so we can reset the multi-flag + detail::has_seen_non_main_interpreter() = false; } /** \rst diff --git a/include/pybind11/gil_safe_call_once.h b/include/pybind11/gil_safe_call_once.h index 3c8ed84df..770ed4999 100644 --- a/include/pybind11/gil_safe_call_once.h +++ b/include/pybind11/gil_safe_call_once.h @@ -232,7 +232,7 @@ private: // Indicator of fast path for single-interpreter case. bool is_last_storage_valid() const { return is_initialized_by_at_least_one_interpreter_ - && detail::get_num_interpreters_seen() == 1; + && !detail::has_seen_non_main_interpreter(); } // Get the unique key for this storage instance in the interpreter's state dict. diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index aaf520457..c47787b6e 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -109,7 +109,7 @@ public: // upon success, the new interpreter is activated in this thread result.istate_ = result.creation_tstate_->interp; - detail::get_num_interpreters_seen() += 1; // there are now many interpreters + detail::has_seen_non_main_interpreter() = true; detail::get_internals(); // initialize internals.tstate, amongst other things... // In 3.13+ this state should be deleted right away, and the memory will be reused for diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3d618662f..15e18705b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -592,6 +592,14 @@ if(NOT PYBIND11_CUDA_TESTS) foreach(mod IN LISTS PYBIND11_MULTIPLE_INTERPRETERS_TEST_MODULES) set_target_properties("${mod}" PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${pybind11_tests_output_directory}") + # Also set config-specific output directories for multi-configuration generators (MSVC) + if(DEFINED CMAKE_CONFIGURATION_TYPES) + foreach(config ${CMAKE_CONFIGURATION_TYPES}) + string(TOUPPER ${config} config) + set_target_properties("${mod}" PROPERTIES LIBRARY_OUTPUT_DIRECTORY_${config} + "${pybind11_tests_output_directory}") + endforeach() + endif() endforeach() if(PYBIND11_TEST_SMART_HOLDER) diff --git a/tests/mod_per_interpreter_gil_with_singleton.cpp b/tests/mod_per_interpreter_gil_with_singleton.cpp index 874b93c77..90e2ec838 100644 --- a/tests/mod_per_interpreter_gil_with_singleton.cpp +++ b/tests/mod_per_interpreter_gil_with_singleton.cpp @@ -9,6 +9,8 @@ namespace py = pybind11; # include #endif +namespace pybind11_tests { +namespace mod_per_interpreter_gil_with_singleton { // A singleton class that holds references to certain Python objects // This singleton is per-interpreter using gil_safe_call_once_and_store class MySingleton { @@ -95,11 +97,15 @@ enum class MyEnum : int { TWO = 2, THREE = 3, }; +} // namespace mod_per_interpreter_gil_with_singleton +} // namespace pybind11_tests PYBIND11_MODULE(mod_per_interpreter_gil_with_singleton, m, py::mod_gil_not_used(), py::multiple_interpreters::per_interpreter_gil()) { + using namespace pybind11_tests::mod_per_interpreter_gil_with_singleton; + #ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT m.attr("defined_PYBIND11_HAS_SUBINTERPRETER_SUPPORT") = true; #else diff --git a/tests/test_multiple_interpreters.py b/tests/test_multiple_interpreters.py index f706e9282..aadf3d96b 100644 --- a/tests/test_multiple_interpreters.py +++ b/tests/test_multiple_interpreters.py @@ -90,7 +90,7 @@ def test_independent_subinterpreters(): run_string, create = get_interpreters(modern=True) - m = pytest.importorskip("mod_per_interpreter_gil") + import mod_per_interpreter_gil as m if not m.defined_PYBIND11_HAS_SUBINTERPRETER_SUPPORT: pytest.skip("Does not have subinterpreter support compiled in") @@ -139,7 +139,7 @@ def test_independent_subinterpreters_modern(): sys.path.insert(0, os.path.dirname(pybind11_tests.__file__)) - m = pytest.importorskip("mod_per_interpreter_gil") + import mod_per_interpreter_gil as m if not m.defined_PYBIND11_HAS_SUBINTERPRETER_SUPPORT: pytest.skip("Does not have subinterpreter support compiled in") @@ -187,7 +187,7 @@ def test_dependent_subinterpreters(): run_string, create = get_interpreters(modern=False) - m = pytest.importorskip("mod_shared_interpreter_gil") + import mod_shared_interpreter_gil as m if not m.defined_PYBIND11_HAS_SUBINTERPRETER_SUPPORT: pytest.skip("Does not have subinterpreter support compiled in") @@ -222,15 +222,27 @@ PREAMBLE_CODE = textwrap.dedent( objects = m.get_objects_in_singleton() expected = [ - type(None), - tuple, - list, - dict, - collections.OrderedDict, - collections.defaultdict, - collections.deque, + type(None), # static type: shared between interpreters + tuple, # static type: shared between interpreters + list, # static type: shared between interpreters + dict, # static type: shared between interpreters + collections.OrderedDict, # static type: shared between interpreters + collections.defaultdict, # heap type: dynamically created per interpreter + collections.deque, # heap type: dynamically created per interpreter ] - assert objects == expected, f"Expected {{expected!r}}, got {{objects!r}}." + # Check that we have the expected objects. Avoid IndexError by checking lengths first. + assert len(objects) == len(expected), ( + f"Expected {{expected!r}} ({{len(expected)}}), got {{objects!r}} ({{len(objects)}})." + ) + # The first ones are static types shared between interpreters. + assert objects[:-2] == expected[:-2], ( + f"Expected static objects {{expected[:-2]!r}}, got {{objects[:-2]!r}}." + ) + # The last two are heap types created per-interpreter. + # The expected objects are dynamically imported from `collections`. + assert objects[-2:] == expected[-2:], ( + f"Expected heap objects {{expected[-2:]!r}}, got {{objects[-2:]!r}}." + ) assert hasattr(m, 'MyClass'), "Module missing MyClass" assert hasattr(m, 'MyGlobalError'), "Module missing MyGlobalError" @@ -240,11 +252,6 @@ PREAMBLE_CODE = textwrap.dedent( ).lstrip() -@pytest.mark.xfail( - reason="Duplicate C++ type registration under multiple-interpreters, needs investigation.", - # raises=interpreters.ExecutionFailed, # need to import the module - strict=False, -) @pytest.mark.skipif( sys.platform.startswith("emscripten"), reason="Requires loadable modules" ) @@ -278,14 +285,9 @@ def check_script_success_in_subprocess(code: str, *, rerun: int = 8) -> None: f"```\n\n" f"Output:\n" f"{ex.output}" - ) from ex + ) from None -@pytest.mark.xfail( - reason="Duplicate C++ type registration under multiple-interpreters, needs investigation.", - raises=RuntimeError, - strict=False, -) @pytest.mark.skipif( sys.platform.startswith("emscripten"), reason="Requires loadable modules" ) @@ -342,11 +344,6 @@ def test_import_in_subinterpreter_after_main(): ) -@pytest.mark.xfail( - reason="Duplicate C++ type registration under multiple-interpreters, needs investigation.", - raises=RuntimeError, - strict=False, -) @pytest.mark.skipif( sys.platform.startswith("emscripten"), reason="Requires loadable modules" ) @@ -427,11 +424,6 @@ def test_import_in_subinterpreter_before_main(): ) -@pytest.mark.xfail( - reason="Duplicate C++ type registration under multiple-interpreters, needs investigation.", - raises=RuntimeError, - strict=False, -) @pytest.mark.skipif( sys.platform.startswith("emscripten"), reason="Requires loadable modules" ) diff --git a/tests/test_with_catch/test_subinterpreter.cpp b/tests/test_with_catch/test_subinterpreter.cpp index b17b6fbc3..e322e0fe9 100644 --- a/tests/test_with_catch/test_subinterpreter.cpp +++ b/tests/test_with_catch/test_subinterpreter.cpp @@ -31,7 +31,7 @@ void unsafe_reset_internals_for_single_interpreter() { py::detail::get_local_internals_pp_manager().unref(); // we know there are no other interpreters, so we can lower this. SUPER DANGEROUS - py::detail::get_num_interpreters_seen() = 1; + py::detail::has_seen_non_main_interpreter() = false; // now we unref the static global singleton internals py::detail::get_internals_pp_manager().unref(); From b93c0f7ed84f02f29f4905dce00bb048f52ae8fe Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 30 Dec 2025 19:54:44 +0700 Subject: [PATCH 17/30] Fix ambiguous `str(handle)` constructor for object-derived types (#5949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix ambiguous `str(handle)` constructor for object-derived types Templatize `str(handle h)` with SFINAE to exclude types derived from `object`, resolving ambiguity with `str(const object&)` when calling `py::str()` on types like `kwargs`, `dict`, etc. The template now only accepts `handle` or `PyObject*`, while all `object`-derived types use the `str(const object&)` overload. * fix(tests): CIBW test fixes from b-pass→vectorcall branch - Install multiple-interpreter test modules into wheel (CMakeLists.txt) The mod_per_interpreter_gil, mod_shared_interpreter_gil, and mod_per_interpreter_gil_with_singleton modules were being built but not installed into the wheel when using scikit-build-core. - Pin numpy 2.4.0 for Python 3.14 CI tests (requirements.txt) NumPy 2.4.0 is the first version with official Python 3.14 wheels. - Add IOS platform constant to tests/env.py - Skip subinterpreter tests on iOS (test_multiple_interpreters.py) Subinterpreters are not supported in the iOS simulator environment. - Enable pytest timeout of 120s for CIBW tests (pyproject.toml) Provides a safety net to catch hanging tests before CI job timeout. - Disable pytest-timeout for Pyodide (no signal.setitimer) Pyodide runs in WebAssembly without POSIX signals. - Add -v flag for verbose pytest output in CIBW tests --- include/pybind11/pytypes.h | 8 +++++++- tests/CMakeLists.txt | 6 ++++++ tests/env.py | 1 + tests/pyproject.toml | 4 +++- tests/requirements.txt | 1 + tests/test_multiple_interpreters.py | 4 ++++ tests/test_pytypes.cpp | 2 ++ tests/test_pytypes.py | 5 +++++ 8 files changed, 29 insertions(+), 2 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index eb02946a8..30eae090f 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1690,7 +1690,13 @@ public: Return a string representation of the object. This is analogous to the ``str()`` function in Python. \endrst */ - explicit str(handle h) : object(raw_str(h.ptr()), stolen_t{}) { + // Templatized to avoid ambiguity with str(const object&) for object-derived types. + template >::value + && std::is_constructible::value, + int> + = 0> + explicit str(T &&h) : object(raw_str(handle(std::forward(h)).ptr()), stolen_t{}) { if (!m_ptr) { throw error_already_set(); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 15e18705b..fdf33628a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -602,6 +602,12 @@ if(NOT PYBIND11_CUDA_TESTS) endif() endforeach() + if(SKBUILD) + foreach(mod IN LISTS PYBIND11_MULTIPLE_INTERPRETERS_TEST_MODULES) + install(TARGETS "${mod}" LIBRARY DESTINATION .) + endforeach() + endif() + if(PYBIND11_TEST_SMART_HOLDER) foreach(mod IN LISTS PYBIND11_MULTIPLE_INTERPRETERS_TEST_MODULES) target_compile_definitions( diff --git a/tests/env.py b/tests/env.py index 4b48e9193..ee932ad77 100644 --- a/tests/env.py +++ b/tests/env.py @@ -5,6 +5,7 @@ import sys import sysconfig ANDROID = sys.platform.startswith("android") +IOS = sys.platform.startswith("ios") LINUX = sys.platform.startswith("linux") MACOS = sys.platform.startswith("darwin") WIN = sys.platform.startswith("win32") or sys.platform.startswith("cygwin") diff --git a/tests/pyproject.toml b/tests/pyproject.toml index fa478122d..8821ea3f3 100644 --- a/tests/pyproject.toml +++ b/tests/pyproject.toml @@ -27,7 +27,9 @@ PYBIND11_FINDPYTHON = true [tool.cibuildwheel] test-sources = ["tests", "pyproject.toml"] -test-command = "python -m pytest -o timeout=0 -p no:cacheprovider tests" +test-command = "python -m pytest -v -o timeout=120 -p no:cacheprovider tests" +# Pyodide doesn't have signal.setitimer, so pytest-timeout can't work with timeout > 0 +pyodide.test-command = "python -m pytest -v -o timeout=0 -p no:cacheprovider tests" environment.PIP_ONLY_BINARY = "numpy" environment.PIP_PREFER_BINARY = "1" diff --git a/tests/requirements.txt b/tests/requirements.txt index 41fc9f143..3d6166923 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -10,6 +10,7 @@ numpy~=1.22.2; platform_python_implementation=="CPython" and python_version=="3. numpy~=1.26.0; platform_python_implementation=="CPython" and python_version>="3.11" and python_version<"3.13" and platform_machine!="ARM64" numpy~=2.3.0; platform_python_implementation=="CPython" and python_version>="3.11" and platform_machine=="ARM64" numpy~=2.2.0; platform_python_implementation=="CPython" and python_version=="3.13" and platform_machine!="ARM64" +numpy==2.4.0; platform_python_implementation=="CPython" and python_version>="3.14" pytest>=6 pytest-timeout scipy~=1.5.4; platform_python_implementation=="CPython" and python_version<"3.10" diff --git a/tests/test_multiple_interpreters.py b/tests/test_multiple_interpreters.py index aadf3d96b..44877e772 100644 --- a/tests/test_multiple_interpreters.py +++ b/tests/test_multiple_interpreters.py @@ -9,8 +9,12 @@ import textwrap import pytest +import env import pybind11_tests +if env.IOS: + pytest.skip("Subinterpreters not supported on iOS", allow_module_level=True) + # 3.14.0b3+, though sys.implementation.supports_isolated_interpreters is being added in b4 # Can be simplified when we drop support for the first three betas CONCURRENT_INTERPRETERS_SUPPORT = ( diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 7d5423e54..e21435001 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -1211,4 +1211,6 @@ TEST_SUBMODULE(pytypes, m) { m.def("check_type_is", [](const py::object &x) -> py::typing::TypeIs { return py::isinstance(x); }); + + m.def("const_kwargs_ref_to_str", [](const py::kwargs &kwargs) { return py::str(kwargs); }); } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 09fc5f37e..580371f02 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1367,3 +1367,8 @@ def test_arg_return_type_hints(doc, backport_typehints): backport_typehints(doc(m.check_type_guard)) == "check_type_guard(arg0: list[object]) -> typing.TypeGuard[list[float]]" ) + + +def test_const_kwargs_ref_to_str(): + assert m.const_kwargs_ref_to_str() == "{}" + assert m.const_kwargs_ref_to_str(a=1) == "{'a': 1}" From 2c9191e9dc5a3e60c18e2ac40b194ea7ba25d0ec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 31 Dec 2025 21:17:09 -0800 Subject: [PATCH 18/30] chore(deps): bump egor-tensin/setup-clang in the actions group (#5950) Bumps the actions group with 1 update: [egor-tensin/setup-clang](https://github.com/egor-tensin/setup-clang). Updates `egor-tensin/setup-clang` from 1 to 2 - [Release notes](https://github.com/egor-tensin/setup-clang/releases) - [Commits](https://github.com/egor-tensin/setup-clang/compare/v1...v2) --- updated-dependencies: - dependency-name: egor-tensin/setup-clang dependency-version: '2' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22c1f489c..8dcf48928 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1166,7 +1166,7 @@ jobs: uses: actions/checkout@v6 - name: Set up Clang - uses: egor-tensin/setup-clang@v1 + uses: egor-tensin/setup-clang@v2 - name: Setup Python ${{ matrix.python }} uses: actions/setup-python@v6 From c761608a22b956e51218f4360661b0135dca3b7f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 23:02:23 -0800 Subject: [PATCH 19/30] chore(deps): update pre-commit hooks (#5951) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update pre-commit hooks updates: - [github.com/pre-commit/mirrors-clang-format: v21.1.6 → v21.1.8](https://github.com/pre-commit/mirrors-clang-format/compare/v21.1.6...v21.1.8) - [github.com/astral-sh/ruff-pre-commit: v0.14.7 → v0.14.10](https://github.com/astral-sh/ruff-pre-commit/compare/v0.14.7...v0.14.10) - [github.com/pre-commit/mirrors-mypy: v1.19.0 → v1.19.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.19.0...v1.19.1) - [github.com/crate-ci/typos: v1.40.0 → v1](https://github.com/crate-ci/typos/compare/v1.40.0...v1) - [github.com/python-jsonschema/check-jsonschema: 0.35.0 → 0.36.0](https://github.com/python-jsonschema/check-jsonschema/compare/0.35.0...0.36.0) * fix(tests): allow numpy>=2.3.0 for ARM64 to fix Windows ARM CI MSYS2 upgraded from numpy 2.3.x to 2.4.0, which no longer satisfies the ~=2.3.0 constraint. Relaxing to >=2.3.0 allows the MSYS2-provided numpy 2.4.0 to be used. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- .pre-commit-config.yaml | 10 +++++----- tests/requirements.txt | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa7b58cc4..9cb3f22df 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,14 +25,14 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v21.1.6" + rev: "v21.1.8" hooks: - id: clang-format types_or: [c++, c, cuda] # Ruff, the Python auto-correcting linter/formatter written in Rust - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.7 + rev: v0.14.10 hooks: - id: ruff-check args: ["--fix", "--show-fixes"] @@ -40,7 +40,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.19.0" + rev: "v1.19.1" hooks: - id: mypy args: [] @@ -120,7 +120,7 @@ repos: # Also check spelling - repo: https://github.com/crate-ci/typos - rev: v1.40.0 + rev: v1 hooks: - id: typos args: [] @@ -149,7 +149,7 @@ repos: # Check schemas on some of our YAML files - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.35.0 + rev: 0.36.0 hooks: - id: check-readthedocs - id: check-github-workflows diff --git a/tests/requirements.txt b/tests/requirements.txt index 3d6166923..50dd10381 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -8,7 +8,7 @@ numpy~=1.26.0; platform_python_implementation=="GraalVM" and sys_platform=="linu numpy~=1.21.5; platform_python_implementation=="CPython" and python_version>="3.8" and python_version<"3.10" numpy~=1.22.2; platform_python_implementation=="CPython" and python_version=="3.10" numpy~=1.26.0; platform_python_implementation=="CPython" and python_version>="3.11" and python_version<"3.13" and platform_machine!="ARM64" -numpy~=2.3.0; platform_python_implementation=="CPython" and python_version>="3.11" and platform_machine=="ARM64" +numpy>=2.3.0; platform_python_implementation=="CPython" and python_version>="3.11" and platform_machine=="ARM64" numpy~=2.2.0; platform_python_implementation=="CPython" and python_version=="3.13" and platform_machine!="ARM64" numpy==2.4.0; platform_python_implementation=="CPython" and python_version>="3.14" pytest>=6 From 10f8708979fb424163bc333eaa6f96f5f40a58cd Mon Sep 17 00:00:00 2001 From: b-pass Date: Tue, 6 Jan 2026 16:32:57 -0500 Subject: [PATCH 20/30] Change function calls to use vectorcall (#5948) * Make argument_vector re-usable for other types. * Attempt to collect args into array for vectorcall * Revert "Attempt to collect args into array for vectorcall" This reverts commit 418a034195c5c8b517e7404686555c097efe7a4b. * Implement vectorcall args collector * pre-commit fixes * Checkpoint in moving to METH_FASTCALL * pre-commit fixes * Use the names tuple directly, cleaner code and less reference counting * Fix unit test, the code now holds more references It cannot re-use the incoming tuple as before, because it is no longer a tuple at all. So a new tuple must be created, which then holds references for each member. * Make clangtidy happy * Oops, _v is C++14 * style: pre-commit fixes * Minor code cleanup * Fix signed conversions * Fix args expansion This would be easier with `if constexpr` * style: pre-commit fixes * Code cleanup * fix(tests): Install multiple-interpreter test modules into wheel The `mod_per_interpreter_gil`, `mod_shared_interpreter_gil`, and `mod_per_interpreter_gil_with_singleton` modules were being built but not installed into the wheel when using scikit-build-core (SKBUILD=true). This caused iOS (and potentially Android) CIBW tests to fail with ModuleNotFoundError. Root cause analysis: - The main test targets have install() commands (line 531) - The PYBIND11_MULTIPLE_INTERPRETERS_TEST_MODULES were missing equivalent install() commands - For regular CMake builds, this wasn't a problem because LIBRARY_OUTPUT_DIRECTORY places the modules next to pybind11_tests - For wheel builds, only targets with explicit install() commands are included in the wheel This issue was latent until commit fee2527d changed the test imports from `pytest.importorskip()` (graceful skip) to direct `import` statements (hard failure), which exposed the missing modules. Failing tests: - test_multiple_interpreters.py::test_independent_subinterpreters - test_multiple_interpreters.py::test_dependent_subinterpreters Error: ModuleNotFoundError: No module named 'mod_per_interpreter_gil' * tests: Pin numpy 2.4.0 for Python 3.14 CI tests Add numpy==2.4.0 requirement for Python 3.14 (both default and free-threaded builds). NumPy 2.4.0 is the first version to provide official PyPI wheels for Python 3.14: - numpy-2.4.0-cp314-cp314-manylinux_2_27_x86_64...whl (default) - numpy-2.4.0-cp314-cp314t-manylinux_2_27_x86_64...whl (free-threaded) Previously, CI was skipping all numpy-dependent tests for Python 3.14 because PIP_ONLY_BINARY was set and no wheels were available: SKIPPED [...] test_numpy_array.py:8: could not import 'numpy': No module named 'numpy' With this change, the full numpy test suite will run on Python 3.14, providing better test coverage for the newest Python version. Note: Using exact pin (==2.4.0) rather than compatible release (~=2.4.0) to ensure reproducible CI results with the first known-working version. * tests: Add verbose flag to CIBW pytest command Add `-v` to the pytest command in tests/pyproject.toml to help diagnose hanging tests in CIBW jobs (particularly iOS). This will show each test name as it runs, making it easier to identify which specific test is hanging. * tests: Skip subinterpreter tests on iOS, add pytest timeout - Add `IOS` platform constant to `tests/env.py` for consistency with existing `ANDROID`, `LINUX`, `MACOS`, `WIN`, `FREEBSD` constants. - Skip `test_multiple_interpreters.py` module on iOS. Subinterpreters are not supported in the iOS simulator environment. These tests were previously skipped implicitly because the modules weren't installed in the wheel; now that they are (commit 6ed6d5a8), we need an explicit skip. - Change pytest timeout from 0 (disabled) to 120 seconds. This provides a safety net to catch hanging tests before the CI job times out after hours. Normal test runs complete in 33-55 seconds total (~1100 tests), so 120 seconds per test is very generous. - Add `-v` flag for verbose output to help diagnose any future issues. * More cleanups in argument vector, per comments. * Per Cursor, move all versions to Vectorcall since it has been supported since 3.8. This means getting rid of simple_collector, we can do the same with a constexpr if in the unpacking_collector. * Switch to a bool vec for the used_kwargs flag... This makes more sense and saves a sort, and the small_vector implementation means it will actually take less space than a vector of size_t elements. The most common case is that all kwargs are used. * Fix signedness for clang * Another signedness issue * tests: Disable pytest-timeout for Pyodide (no signal.setitimer) Pyodide runs in a WebAssembly sandbox without POSIX signals, so `signal.setitimer` is not available. This causes pytest-timeout to crash with `AttributeError: module 'signal' has no attribute 'setitimer'` when timeout > 0. Override the test-command for Pyodide to keep timeout=0 (disabled). * Combine temp storage and args into one vector It's a good bit faster at the cost of this one scary reinterpret_cast. * Phrasing * Delete incorrect comment At 6, the struct is 144 bytes (not 128 bytes as the comment said). * Fix push_back * Update push_back in argument_vector.h Co-authored-by: Aaron Gokaslan * style: pre-commit fixes * Use real types for these instead of object They can be null if you "steal" a null handle. * refactor: Replace small_vector with ref_small_vector for explicit ownership Introduce `ref_small_vector` to manage PyObject* references in `unpacking_collector`, replacing the previous `small_vector` approach. Primary goals: 1. **Maintainability**: The previous implementation relied on `sizeof(object) == sizeof(PyObject*)` and used a reinterpret_cast to pass the object array to vectorcall. This coupling to py::object's internal layout could break if someone adds a debug field or other member to py::handle/py::object in the future. 2. **Readability**: The new `push_back_steal()` vs `push_back_borrow()` API makes reference counting intent explicit at each call site, rather than relying on implicit py::object semantics. 3. **Intuitive code**: Storing `PyObject*` directly and passing it to `_PyObject_Vectorcall` without casts is straightforward and matches what the C API expects. No "scary" reinterpret_cast needed. Additional benefits: - `PyObject*` is trivially copyable, simplifying vector operations - Batch decref in destructor (tight loop vs N individual object destructors) - Self-documenting ownership semantics Design consideration: We considered folding the ref-counting functionality directly into `small_vector` via template specialization for `PyObject*`. We decided against this because: - It would give `small_vector` a different interface than the generic `small_vector` (steal/borrow vs push_back) - Someone might want a non-ref-counting `small_vector` - The specialization behavior could surprise users expecting uniform semantics A separate `ref_small_vector` type makes the ref-counting behavior explicit and self-documenting, while keeping `small_vector` generic and predictable. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve Co-authored-by: Aaron Gokaslan --- include/pybind11/cast.h | 197 ++++++++++++---------- include/pybind11/detail/argument_vector.h | 195 +++++++++++++++------ include/pybind11/pybind11.h | 176 ++++++++++--------- tests/test_kwargs_and_defaults.py | 3 +- 4 files changed, 347 insertions(+), 224 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 310b77b34..f5a94da20 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -2078,8 +2078,7 @@ using is_pos_only = std::is_same, pos_only>; // forward declaration (definition in attr.h) struct function_record; -/// (Inline size chosen mostly arbitrarily; 6 should pad function_call out to two cache lines -/// (16 pointers) in size.) +/// Inline size chosen mostly arbitrarily. constexpr std::size_t arg_vector_small_size = 6; /// Internal data associated with a single function call @@ -2191,86 +2190,121 @@ private: std::tuple...> argcasters; }; -/// Helper class which collects only positional arguments for a Python function call. -/// A fancier version below can collect any argument, but this one is optimal for simple calls. -template -class simple_collector { -public: - template - explicit simple_collector(Ts &&...values) - : m_args(pybind11::make_tuple(std::forward(values)...)) {} - - const tuple &args() const & { return m_args; } - dict kwargs() const { return {}; } - - tuple args() && { return std::move(m_args); } - - /// Call a Python function and pass the collected arguments - object call(PyObject *ptr) const { - PyObject *result = PyObject_CallObject(ptr, m_args.ptr()); - if (!result) { - throw error_already_set(); - } - return reinterpret_steal(result); - } - -private: - tuple m_args; -}; +// [workaround(intel)] Separate function required here +// We need to put this into a separate function because the Intel compiler +// fails to compile enable_if_t...>::value> +// (tested with ICC 2021.1 Beta 20200827). +template +constexpr bool args_has_keyword_or_ds() { + return any_of...>::value; +} /// Helper class which collects positional, keyword, * and ** arguments for a Python function call template class unpacking_collector { public: template - explicit unpacking_collector(Ts &&...values) { - // Tuples aren't (easily) resizable so a list is needed for collection, - // but the actual function call strictly requires a tuple. - auto args_list = list(); - using expander = int[]; - (void) expander{0, (process(args_list, std::forward(values)), 0)...}; + explicit unpacking_collector(Ts &&...values) + : m_names(reinterpret_steal( + handle())) // initialize to null to avoid useless allocation of 0-length tuple + { + /* + Python can sometimes utilize an extra space before the arguments to prepend `self`. + This is important enough that there is a special flag for it: + PY_VECTORCALL_ARGUMENTS_OFFSET. + All we have to do is allocate an extra space at the beginning of this array, and set the + flag. Note that the extra space is not passed directly in to vectorcall. + */ + m_args.reserve(sizeof...(values) + 1); + m_args.push_back_null(); - m_args = std::move(args_list); + if (args_has_keyword_or_ds()) { + list names_list; + + // collect_arguments guarantees this can't be constructed with kwargs before the last + // positional so we don't need to worry about Ts... being in anything but normal python + // order. + using expander = int[]; + (void) expander{0, (process(names_list, std::forward(values)), 0)...}; + + m_names = reinterpret_steal(PyList_AsTuple(names_list.ptr())); + } else { + auto not_used + = reinterpret_steal(handle()); // initialize as null (to avoid an allocation) + + using expander = int[]; + (void) expander{0, (process(not_used, std::forward(values)), 0)...}; + } } - const tuple &args() const & { return m_args; } - const dict &kwargs() const & { return m_kwargs; } - - tuple args() && { return std::move(m_args); } - dict kwargs() && { return std::move(m_kwargs); } - /// Call a Python function and pass the collected arguments object call(PyObject *ptr) const { - PyObject *result = PyObject_Call(ptr, m_args.ptr(), m_kwargs.ptr()); + size_t nargs = m_args.size() - 1; // -1 for PY_VECTORCALL_ARGUMENTS_OFFSET (see ctor) + if (m_names) { + nargs -= m_names.size(); + } + PyObject *result = _PyObject_Vectorcall( + ptr, m_args.data() + 1, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, m_names.ptr()); if (!result) { throw error_already_set(); } return reinterpret_steal(result); } + tuple args() const { + size_t nargs = m_args.size() - 1; // -1 for PY_VECTORCALL_ARGUMENTS_OFFSET (see ctor) + if (m_names) { + nargs -= m_names.size(); + } + tuple val(nargs); + for (size_t i = 0; i < nargs; ++i) { + // +1 for PY_VECTORCALL_ARGUMENTS_OFFSET (see ctor) + val[i] = reinterpret_borrow(m_args[i + 1]); + } + return val; + } + + dict kwargs() const { + dict val; + if (m_names) { + size_t offset = m_args.size() - m_names.size(); + for (size_t i = 0; i < m_names.size(); ++i, ++offset) { + val[m_names[i]] = reinterpret_borrow(m_args[offset]); + } + } + return val; + } + private: + // normal argument, possibly needing conversion template - void process(list &args_list, T &&x) { - auto o = reinterpret_steal( - detail::make_caster::cast(std::forward(x), policy, {})); - if (!o) { + void process(list & /*names_list*/, T &&x) { + handle h = detail::make_caster::cast(std::forward(x), policy, {}); + if (!h) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size())); + throw cast_error_unable_to_convert_call_arg(std::to_string(m_args.size() - 1)); #else - throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size()), + throw cast_error_unable_to_convert_call_arg(std::to_string(m_args.size() - 1), type_id()); #endif } - args_list.append(std::move(o)); + m_args.push_back_steal(h.ptr()); // cast returns a new reference } - void process(list &args_list, detail::args_proxy ap) { + // * unpacking + void process(list & /*names_list*/, detail::args_proxy ap) { + if (!ap) { + return; + } for (auto a : ap) { - args_list.append(a); + m_args.push_back_borrow(a.ptr()); } } - void process(list & /*args_list*/, arg_v a) { + // named argument + // NOLINTNEXTLINE(performance-unnecessary-value-param) + void process(list &names_list, arg_v a) { + assert(names_list); if (!a.name) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) nameless_argument_error(); @@ -2278,7 +2312,8 @@ private: nameless_argument_error(a.type); #endif } - if (m_kwargs.contains(a.name)) { + auto name = str(a.name); + if (names_list.contains(name)) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) multiple_values_error(); #else @@ -2292,22 +2327,27 @@ private: throw cast_error_unable_to_convert_call_arg(a.name, a.type); #endif } - m_kwargs[a.name] = std::move(a.value); + names_list.append(std::move(name)); + m_args.push_back_borrow(a.value.ptr()); } - void process(list & /*args_list*/, detail::kwargs_proxy kp) { + // ** unpacking + void process(list &names_list, detail::kwargs_proxy kp) { if (!kp) { return; } - for (auto k : reinterpret_borrow(kp)) { - if (m_kwargs.contains(k.first)) { + assert(names_list); + for (auto &&k : reinterpret_borrow(kp)) { + auto name = str(k.first); + if (names_list.contains(name)) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) multiple_values_error(); #else - multiple_values_error(str(k.first)); + multiple_values_error(name); #endif } - m_kwargs[k.first] = k.second; + names_list.append(std::move(name)); + m_args.push_back_borrow(k.second.ptr()); } } @@ -2333,39 +2373,20 @@ private: } private: - tuple m_args; - dict m_kwargs; + ref_small_vector m_args; + tuple m_names; }; -// [workaround(intel)] Separate function required here -// We need to put this into a separate function because the Intel compiler -// fails to compile enable_if_t...>::value> -// (tested with ICC 2021.1 Beta 20200827). -template -constexpr bool args_are_all_positional() { - return all_of...>::value; -} - -/// Collect only positional arguments for a Python function call -template ()>> -simple_collector collect_arguments(Args &&...args) { - return simple_collector(std::forward(args)...); -} - -/// Collect all arguments, including keywords and unpacking (only instantiated when needed) -template ()>> +/// Collect all arguments, including keywords and unpacking +template unpacking_collector collect_arguments(Args &&...args) { // Following argument order rules for generalized unpacking according to PEP 448 - static_assert(constexpr_last() - < constexpr_first() - && constexpr_last() - < constexpr_first(), - "Invalid function call: positional args must precede keywords and ** unpacking; " - "* unpacking must precede ** unpacking"); + static_assert( + constexpr_last() < constexpr_first(), + "Invalid function call: positional args must precede keywords and */** unpacking;"); + static_assert(constexpr_last() + < constexpr_first(), + "Invalid function call: * unpacking must precede ** unpacking"); return unpacking_collector(std::forward(args)...); } diff --git a/include/pybind11/detail/argument_vector.h b/include/pybind11/detail/argument_vector.h index e15a3cfab..6e2c2ec48 100644 --- a/include/pybind11/detail/argument_vector.h +++ b/include/pybind11/detail/argument_vector.h @@ -66,24 +66,23 @@ union inline_array_or_vector { inline_array iarray; heap_vector hvector; - static_assert(std::is_trivially_move_constructible::value, - "ArrayT must be trivially move constructible"); - static_assert(std::is_trivially_destructible::value, - "ArrayT must be trivially destructible"); - inline_array_or_vector() : iarray() {} + ~inline_array_or_vector() { - if (!is_inline()) { + if (is_inline()) { + iarray.~inline_array(); + } else { hvector.~heap_vector(); } } + // Disable copy ctor and assignment. inline_array_or_vector(const inline_array_or_vector &) = delete; inline_array_or_vector &operator=(const inline_array_or_vector &) = delete; inline_array_or_vector(inline_array_or_vector &&rhs) noexcept { if (rhs.is_inline()) { - std::memcpy(&iarray, &rhs.iarray, sizeof(iarray)); + new (&iarray) inline_array(std::move(rhs.iarray)); } else { new (&hvector) heap_vector(std::move(rhs.hvector)); } @@ -95,17 +94,16 @@ union inline_array_or_vector { return *this; } - if (rhs.is_inline()) { - if (!is_inline()) { - hvector.~heap_vector(); - } - std::memcpy(&iarray, &rhs.iarray, sizeof(iarray)); + if (is_inline()) { + iarray.~inline_array(); } else { - if (is_inline()) { - new (&hvector) heap_vector(std::move(rhs.hvector)); - } else { - hvector = std::move(rhs.hvector); - } + hvector.~heap_vector(); + } + + if (rhs.is_inline()) { + new (&iarray) inline_array(std::move(rhs.iarray)); + } else { + new (&hvector) heap_vector(std::move(rhs.hvector)); } return *this; } @@ -126,18 +124,16 @@ union inline_array_or_vector { } }; -// small_vector-like container to avoid heap allocation for N or fewer -// arguments. -template -struct argument_vector { +template +struct small_vector { public: - argument_vector() = default; + small_vector() = default; // Disable copy ctor and assignment. - argument_vector(const argument_vector &) = delete; - argument_vector &operator=(const argument_vector &) = delete; - argument_vector(argument_vector &&) noexcept = default; - argument_vector &operator=(argument_vector &&) noexcept = default; + small_vector(const small_vector &) = delete; + small_vector &operator=(const small_vector &) = delete; + small_vector(small_vector &&) noexcept = default; + small_vector &operator=(small_vector &&) noexcept = default; std::size_t size() const { if (is_inline()) { @@ -146,7 +142,14 @@ public: return m_repr.hvector.vec.size(); } - handle &operator[](std::size_t idx) { + T const *data() const { + if (is_inline()) { + return m_repr.iarray.arr.data(); + } + return m_repr.hvector.vec.data(); + } + + T &operator[](std::size_t idx) { assert(idx < size()); if (is_inline()) { return m_repr.iarray.arr[idx]; @@ -154,7 +157,7 @@ public: return m_repr.hvector.vec[idx]; } - handle operator[](std::size_t idx) const { + T const &operator[](std::size_t idx) const { assert(idx < size()); if (is_inline()) { return m_repr.iarray.arr[idx]; @@ -162,28 +165,28 @@ public: return m_repr.hvector.vec[idx]; } - void push_back(handle x) { + void push_back(const T &x) { emplace_back(x); } + + void push_back(T &&x) { emplace_back(std::move(x)); } + + template + void emplace_back(Args &&...x) { if (is_inline()) { auto &ha = m_repr.iarray; - if (ha.size == N) { - move_to_heap_vector_with_reserved_size(N + 1); - push_back_slow_path(x); + if (ha.size == InlineSize) { + move_to_heap_vector_with_reserved_size(InlineSize + 1); + m_repr.hvector.vec.emplace_back(std::forward(x)...); } else { - ha.arr[ha.size++] = x; + ha.arr[ha.size++] = T(std::forward(x)...); } } else { - push_back_slow_path(x); + m_repr.hvector.vec.emplace_back(std::forward(x)...); } } - template - void emplace_back(Arg &&x) { - push_back(handle(x)); - } - void reserve(std::size_t sz) { if (is_inline()) { - if (sz > N) { + if (sz > InlineSize) { move_to_heap_vector_with_reserved_size(sz); } } else { @@ -192,7 +195,7 @@ public: } private: - using repr_type = inline_array_or_vector; + using repr_type = inline_array_or_vector; repr_type m_repr; PYBIND11_NOINLINE void move_to_heap_vector_with_reserved_size(std::size_t reserved_size) { @@ -201,32 +204,33 @@ private: using heap_vector = typename repr_type::heap_vector; heap_vector hv; hv.vec.reserve(reserved_size); - std::copy(ha.arr.begin(), ha.arr.begin() + ha.size, std::back_inserter(hv.vec)); + static_assert(std::is_nothrow_move_constructible::value, + "this conversion is not exception safe"); + static_assert(std::is_nothrow_move_constructible::value, + "this conversion is not exception safe"); + std::move(ha.arr.begin(), ha.arr.begin() + ha.size, std::back_inserter(hv.vec)); new (&m_repr.hvector) heap_vector(std::move(hv)); } - PYBIND11_NOINLINE void push_back_slow_path(handle x) { m_repr.hvector.vec.push_back(x); } - PYBIND11_NOINLINE void reserve_slow_path(std::size_t sz) { m_repr.hvector.vec.reserve(sz); } bool is_inline() const { return m_repr.is_inline(); } }; -// small_vector-like container to avoid heap allocation for N or fewer -// arguments. +// Container to avoid heap allocation for kRequestedInlineSize or fewer booleans. template -struct args_convert_vector { +struct small_vector { private: public: - args_convert_vector() = default; + small_vector() = default; // Disable copy ctor and assignment. - args_convert_vector(const args_convert_vector &) = delete; - args_convert_vector &operator=(const args_convert_vector &) = delete; - args_convert_vector(args_convert_vector &&) noexcept = default; - args_convert_vector &operator=(args_convert_vector &&) noexcept = default; + small_vector(const small_vector &) = delete; + small_vector &operator=(const small_vector &) = delete; + small_vector(small_vector &&) noexcept = default; + small_vector &operator=(small_vector &&) noexcept = default; - args_convert_vector(std::size_t count, bool value) { + small_vector(std::size_t count, bool value) { if (count > kInlineSize) { new (&m_repr.hvector) typename repr_type::heap_vector(count, value); } else { @@ -284,7 +288,24 @@ public: } } - void swap(args_convert_vector &rhs) noexcept { std::swap(m_repr, rhs.m_repr); } + void set(std::size_t idx, bool value = true) { + if (is_inline()) { + auto &ha = m_repr.iarray; + assert(ha.size < kInlineSize); + const auto wbi = word_and_bit_index(idx); + assert(wbi.word < kWords); + assert(wbi.bit < kBitsPerWord); + if (value) { + ha.arr[wbi.word] |= (static_cast(1) << wbi.bit); + } else { + ha.arr[wbi.word] &= ~(static_cast(1) << wbi.bit); + } + } else { + m_repr.hvector.vec[idx] = value; + } + } + + void swap(small_vector &rhs) noexcept { std::swap(m_repr, rhs.m_repr); } private: struct WordAndBitIndex { @@ -326,5 +347,71 @@ private: bool is_inline() const { return m_repr.is_inline(); } }; +// Container to avoid heap allocation for N or fewer arguments. +template +using argument_vector = small_vector; + +// Container to avoid heap allocation for N or fewer booleans. +template +using args_convert_vector = small_vector; + +/// A small_vector of PyObject* that holds references and releases them on destruction. +/// This provides explicit ownership semantics without relying on py::object's +/// destructor, and avoids the need for reinterpret_cast when passing to vectorcall. +template +class ref_small_vector { +public: + ref_small_vector() = default; + + ~ref_small_vector() { + for (std::size_t i = 0; i < m_ptrs.size(); ++i) { + Py_XDECREF(m_ptrs[i]); + } + } + + // Disable copy (prevent accidental double-decref) + ref_small_vector(const ref_small_vector &) = delete; + ref_small_vector &operator=(const ref_small_vector &) = delete; + + // Move is allowed + ref_small_vector(ref_small_vector &&other) noexcept : m_ptrs(std::move(other.m_ptrs)) { + // other.m_ptrs is now empty, so its destructor won't decref anything + } + + ref_small_vector &operator=(ref_small_vector &&other) noexcept { + if (this != &other) { + // Decref our current contents + for (std::size_t i = 0; i < m_ptrs.size(); ++i) { + Py_XDECREF(m_ptrs[i]); + } + m_ptrs = std::move(other.m_ptrs); + } + return *this; + } + + /// Add a pointer, taking ownership (no incref, will decref on destruction) + void push_back_steal(PyObject *p) { m_ptrs.push_back(p); } + + /// Add a pointer, borrowing (increfs now, will decref on destruction) + void push_back_borrow(PyObject *p) { + Py_XINCREF(p); + m_ptrs.push_back(p); + } + + /// Add a null pointer (for PY_VECTORCALL_ARGUMENTS_OFFSET slot) + void push_back_null() { m_ptrs.push_back(nullptr); } + + void reserve(std::size_t sz) { m_ptrs.reserve(sz); } + + std::size_t size() const { return m_ptrs.size(); } + + PyObject *operator[](std::size_t idx) const { return m_ptrs[idx]; } + + PyObject *const *data() const { return m_ptrs.data(); } + +private: + small_vector m_ptrs; +}; + PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index f93272628..45e8e46f8 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -684,7 +684,7 @@ protected: rec->def->ml_name = rec->name; rec->def->ml_meth = reinterpret_cast(reinterpret_cast(dispatcher)); - rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS; + rec->def->ml_flags = METH_FASTCALL | METH_KEYWORDS; object py_func_rec = detail::function_record_PyObject_New(); (reinterpret_cast(py_func_rec.ptr()))->cpp_func_rec @@ -847,7 +847,8 @@ protected: } /// Main dispatch logic for calls to functions bound using pybind11 - static PyObject *dispatcher(PyObject *self, PyObject *args_in, PyObject *kwargs_in) { + static PyObject * + dispatcher(PyObject *self, PyObject *const *args_in_arr, size_t nargsf, PyObject *kwnames_in) { using namespace detail; const function_record *overloads = function_record_ptr_from_PyObject(self); assert(overloads != nullptr); @@ -857,9 +858,9 @@ protected: /* Need to know how many arguments + keyword arguments there are to pick the right overload */ - const auto n_args_in = static_cast(PyTuple_GET_SIZE(args_in)); + const auto n_args_in = static_cast(PyVectorcall_NARGS(nargsf)); - handle parent = n_args_in > 0 ? PyTuple_GET_ITEM(args_in, 0) : nullptr, + handle parent = n_args_in > 0 ? args_in_arr[0] : nullptr, result = PYBIND11_TRY_NEXT_OVERLOAD; auto self_value_and_holder = value_and_holder(); @@ -948,7 +949,7 @@ protected: self_value_and_holder.type->dealloc(self_value_and_holder); } - call.init_self = PyTuple_GET_ITEM(args_in, 0); + call.init_self = args_in_arr[0]; call.args.emplace_back(reinterpret_cast(&self_value_and_holder)); call.args_convert.push_back(false); ++args_copied; @@ -959,17 +960,24 @@ protected: for (; args_copied < args_to_copy; ++args_copied) { const argument_record *arg_rec = args_copied < func.args.size() ? &func.args[args_copied] : nullptr; - if (kwargs_in && arg_rec && arg_rec->name - && dict_getitemstring(kwargs_in, arg_rec->name)) { + + /* if the argument is listed in the call site's kwargs, but the argument is + also fulfilled positionally, then the call can't match this overload. for + example, the call site is: foo(0, key=1) but our overload is foo(key:int) then + this call can't be for us, because it would be invalid. + */ + if (kwnames_in && arg_rec && arg_rec->name + && keyword_index(kwnames_in, arg_rec->name) >= 0) { bad_arg = true; break; } - handle arg(PyTuple_GET_ITEM(args_in, args_copied)); + handle arg(args_in_arr[args_copied]); if (arg_rec && !arg_rec->none && arg.is_none()) { bad_arg = true; break; } + call.args.push_back(arg); call.args_convert.push_back(arg_rec ? arg_rec->convert : true); } @@ -981,20 +989,12 @@ protected: // to copy the rest into a py::args argument. size_t positional_args_copied = args_copied; - // We'll need to copy this if we steal some kwargs for defaults - dict kwargs = reinterpret_borrow(kwargs_in); - // 1.5. Fill in any missing pos_only args from defaults if they exist if (args_copied < func.nargs_pos_only) { for (; args_copied < func.nargs_pos_only; ++args_copied) { const auto &arg_rec = func.args[args_copied]; - handle value; - if (arg_rec.value) { - value = arg_rec.value; - } - if (value) { - call.args.push_back(value); + call.args.push_back(arg_rec.value); call.args_convert.push_back(arg_rec.convert); } else { break; @@ -1007,46 +1007,42 @@ protected: } // 2. Check kwargs and, failing that, defaults that may help complete the list + small_vector used_kwargs( + kwnames_in ? static_cast(PyTuple_GET_SIZE(kwnames_in)) : 0, false); + size_t used_kwargs_count = 0; if (args_copied < num_args) { - bool copied_kwargs = false; - for (; args_copied < num_args; ++args_copied) { const auto &arg_rec = func.args[args_copied]; handle value; - if (kwargs_in && arg_rec.name) { - value = dict_getitemstring(kwargs.ptr(), arg_rec.name); + if (kwnames_in && arg_rec.name) { + ssize_t i = keyword_index(kwnames_in, arg_rec.name); + if (i >= 0) { + value = args_in_arr[n_args_in + static_cast(i)]; + used_kwargs.set(static_cast(i), true); + used_kwargs_count++; + } } - if (value) { - // Consume a kwargs value - if (!copied_kwargs) { - kwargs = reinterpret_steal(PyDict_Copy(kwargs.ptr())); - copied_kwargs = true; - } - if (PyDict_DelItemString(kwargs.ptr(), arg_rec.name) == -1) { - throw error_already_set(); - } - } else if (arg_rec.value) { + if (!value) { value = arg_rec.value; + if (!value) { + break; + } } if (!arg_rec.none && value.is_none()) { break; } - if (value) { - // If we're at the py::args index then first insert a stub for it to be - // replaced later - if (func.has_args && call.args.size() == func.nargs_pos) { - call.args.push_back(none()); - } - - call.args.push_back(value); - call.args_convert.push_back(arg_rec.convert); - } else { - break; + // If we're at the py::args index then first insert a stub for it to be + // replaced later + if (func.has_args && call.args.size() == func.nargs_pos) { + call.args.push_back(none()); } + + call.args.push_back(value); + call.args_convert.push_back(arg_rec.convert); } if (args_copied < num_args) { @@ -1056,47 +1052,46 @@ protected: } // 3. Check everything was consumed (unless we have a kwargs arg) - if (kwargs && !kwargs.empty() && !func.has_kwargs) { + if (!func.has_kwargs && used_kwargs_count < used_kwargs.size()) { continue; // Unconsumed kwargs, but no py::kwargs argument to accept them } // 4a. If we have a py::args argument, create a new tuple with leftovers if (func.has_args) { - tuple extra_args; - if (args_to_copy == 0) { - // We didn't copy out any position arguments from the args_in tuple, so we - // can reuse it directly without copying: - extra_args = reinterpret_borrow(args_in); - } else if (positional_args_copied >= n_args_in) { - extra_args = tuple(0); + if (positional_args_copied >= n_args_in) { + call.args_ref = tuple(0); } else { size_t args_size = n_args_in - positional_args_copied; - extra_args = tuple(args_size); + tuple extra_args(args_size); for (size_t i = 0; i < args_size; ++i) { - extra_args[i] = PyTuple_GET_ITEM(args_in, positional_args_copied + i); + extra_args[i] = args_in_arr[positional_args_copied + i]; } + call.args_ref = std::move(extra_args); } if (call.args.size() <= func.nargs_pos) { - call.args.push_back(extra_args); + call.args.push_back(call.args_ref); } else { - call.args[func.nargs_pos] = extra_args; + call.args[func.nargs_pos] = call.args_ref; } call.args_convert.push_back(false); - call.args_ref = std::move(extra_args); } // 4b. If we have a py::kwargs, pass on any remaining kwargs if (func.has_kwargs) { - if (!kwargs.ptr()) { - kwargs = dict(); // If we didn't get one, send an empty one + dict kwargs; + for (size_t i = 0; i < used_kwargs.size(); ++i) { + if (!used_kwargs[i]) { + kwargs[PyTuple_GET_ITEM(kwnames_in, i)] = args_in_arr[n_args_in + i]; + } } call.args.push_back(kwargs); call.args_convert.push_back(false); call.kwargs_ref = std::move(kwargs); } -// 5. Put everything in a vector. Not technically step 5, we've been building it -// in `call.args` all along. + // 5. Put everything in a vector. Not technically step 5, we've been building it + // in `call.args` all along. + #if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (call.args.size() != func.nargs || call.args_convert.size() != func.nargs) { pybind11_fail("Internal error: function call dispatcher inserted wrong number " @@ -1227,40 +1222,37 @@ protected: msg += '\n'; } msg += "\nInvoked with: "; - auto args_ = reinterpret_borrow(args_in); bool some_args = false; - for (size_t ti = overloads->is_constructor ? 1 : 0; ti < args_.size(); ++ti) { + for (size_t ti = overloads->is_constructor ? 1 : 0; ti < n_args_in; ++ti) { if (!some_args) { some_args = true; } else { msg += ", "; } try { - msg += pybind11::repr(args_[ti]); + msg += pybind11::repr(args_in_arr[ti]); } catch (const error_already_set &) { msg += ""; } } - if (kwargs_in) { - auto kwargs = reinterpret_borrow(kwargs_in); - if (!kwargs.empty()) { - if (some_args) { - msg += "; "; + if (kwnames_in && PyTuple_GET_SIZE(kwnames_in) > 0) { + if (some_args) { + msg += "; "; + } + msg += "kwargs: "; + bool first = true; + for (size_t i = 0; i < static_cast(PyTuple_GET_SIZE(kwnames_in)); ++i) { + if (first) { + first = false; + } else { + msg += ", "; } - msg += "kwargs: "; - bool first = true; - for (const auto &kwarg : kwargs) { - if (first) { - first = false; - } else { - msg += ", "; - } - msg += pybind11::str("{}=").format(kwarg.first); - try { - msg += pybind11::repr(kwarg.second); - } catch (const error_already_set &) { - msg += ""; - } + msg += reinterpret_borrow(PyTuple_GET_ITEM(kwnames_in, i)); + msg += '='; + try { + msg += pybind11::repr(args_in_arr[n_args_in + i]); + } catch (const error_already_set &) { + msg += ""; } } } @@ -1295,6 +1287,28 @@ protected: } return result.ptr(); } + + static ssize_t keyword_index(PyObject *haystack, char const *needle) { + /* kwargs is usually very small (<= 5 entries). The arg strings are typically interned. + * CPython itself implements the search this way, first comparing all pointers ... which is + * cheap and will work if the strings are interned. If it fails, then it falls back to a + * second lexicographic check. This is wildly expensive for huge argument lists, but those + * are incredibly rare so we optimize for the vastly common case of just a couple of args. + */ + auto n = PyTuple_GET_SIZE(haystack); + auto s = reinterpret_steal(PyUnicode_InternFromString(needle)); + for (ssize_t i = 0; i < n; ++i) { + if (PyTuple_GET_ITEM(haystack, i) == s.ptr()) { + return i; + } + } + for (ssize_t i = 0; i < n; ++i) { + if (PyUnicode_Compare(PyTuple_GET_ITEM(haystack, i), s.ptr()) == 0) { + return i; + } + } + return -1; + } }; PYBIND11_NAMESPACE_BEGIN(detail) diff --git a/tests/test_kwargs_and_defaults.py b/tests/test_kwargs_and_defaults.py index d41e50558..a7745d1ec 100644 --- a/tests/test_kwargs_and_defaults.py +++ b/tests/test_kwargs_and_defaults.py @@ -458,7 +458,8 @@ def test_args_refcount(): assert refcount(myval) == expected exp3 = refcount(myval, myval, myval) - assert m.args_refcount(myval, myval, myval) == (exp3, exp3, exp3) + # if we have to create a new tuple internally, then it will hold an extra reference for each item in it. + assert m.args_refcount(myval, myval, myval) == (exp3 + 3, exp3 + 3, exp3 + 3) assert refcount(myval) == expected # This function takes the first arg as a `py::object` and the rest as a `py::args`. Unlike the From 745e5688e7dc1909c674ccd30104a83de479df58 Mon Sep 17 00:00:00 2001 From: "T.Yamada" Date: Wed, 7 Jan 2026 16:04:17 +0900 Subject: [PATCH 21/30] Fix longdouble complex docstring to clongdouble (#5952) --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 236d90bd3..6fa6c772b 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -416,7 +416,7 @@ struct npy_format_descriptor_name::value>> { || std::is_same::value > (const_name("numpy.complex") + const_name(), - const_name("numpy.longcomplex")); + const_name("numpy.clongdouble")); }; template From d36f5dd5a4ee012e7969617ff67c4fd816bdc927 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Thu, 8 Jan 2026 08:28:38 +0800 Subject: [PATCH 22/30] Appease MSVC Warning C4866: compiler may not enforce left-to-right evaluation order (#5953) * Appease MSVC Warning C4866: compiler may not enforce left-to-right evaluation order * Remove const qualifier * Reword comment to be self-explanatory without PR context The previous comment referenced the MSVC warning but didn't explain why the code is structured as two statements. The revised comment clarifies the intent: fetching the value first ensures well-defined evaluation order. * chore(deps): switch typos to mirror repo Switch from crate-ci/typos to adhtruong/mirrors-typos because pre-commit autoupdate confuses tags in the upstream repo, selecting the mutable `v1` tag instead of pinned versions like `v1.41.0`. See https://github.com/crate-ci/typos/issues/390 --------- Co-authored-by: Ralf W. Grosse-Kunstleve --- .pre-commit-config.yaml | 6 ++++-- include/pybind11/pybind11.h | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9cb3f22df..1271c2afe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -119,8 +119,10 @@ repos: args: ["-x.codespell-ignore-lines", "-Lccompiler,intstruct"] # Also check spelling -- repo: https://github.com/crate-ci/typos - rev: v1 +# Use mirror because pre-commit autoupdate confuses tags in the upstream repo. +# See https://github.com/crate-ci/typos/issues/390 +- repo: https://github.com/adhtruong/mirrors-typos + rev: "v1.41.0" hooks: - id: typos args: [] diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 45e8e46f8..c457e149c 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1081,7 +1081,10 @@ protected: dict kwargs; for (size_t i = 0; i < used_kwargs.size(); ++i) { if (!used_kwargs[i]) { - kwargs[PyTuple_GET_ITEM(kwnames_in, i)] = args_in_arr[n_args_in + i]; + // Fetch value before indexing into kwargs to ensure well-defined + // evaluation order (MSVC C4866). + PyObject *const arg_in_arr = args_in_arr[n_args_in + i]; + kwargs[PyTuple_GET_ITEM(kwnames_in, i)] = arg_in_arr; } } call.args.push_back(kwargs); From cc551ada331feaf89102626217cd77f4b0cf1abb Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sun, 11 Jan 2026 02:46:01 +0800 Subject: [PATCH 23/30] Appease MSVC Warning C4866: compiler may not enforce left-to-right evaluation order (#5955) --- include/pybind11/pybind11.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index c457e149c..02d2e72c2 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1081,10 +1081,11 @@ protected: dict kwargs; for (size_t i = 0; i < used_kwargs.size(); ++i) { if (!used_kwargs[i]) { - // Fetch value before indexing into kwargs to ensure well-defined - // evaluation order (MSVC C4866). - PyObject *const arg_in_arr = args_in_arr[n_args_in + i]; - kwargs[PyTuple_GET_ITEM(kwnames_in, i)] = arg_in_arr; + // Cast values into handles before indexing into kwargs to ensure + // well-defined evaluation order (MSVC C4866). + handle arg_in_arr = args_in_arr[n_args_in + i], + kwname = PyTuple_GET_ITEM(kwnames_in, i); + kwargs[kwname] = arg_in_arr; } } call.args.push_back(kwargs); From ca1d996461d75ec01dbfa37f01b8c17aac957151 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Jan 2026 11:44:19 -0800 Subject: [PATCH 24/30] chore(deps): bump urllib3 from 2.6.0 to 2.6.3 in /docs (#5954) Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.6.0 to 2.6.3. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/2.6.0...2.6.3) --- updated-dependencies: - dependency-name: urllib3 dependency-version: 2.6.3 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 9d2ea4f85..1b6bcf0c2 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -83,5 +83,5 @@ sphinxcontrib-serializinghtml==1.1.10 # via sphinx sphinxcontrib-svg2pdfconverter==1.2.2 # via -r requirements.in -urllib3==2.6.0 +urllib3==2.6.3 # via requests From e44aae2268d4bc32719f35fddd2b33cee558ed85 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 15 Jan 2026 12:38:45 -0500 Subject: [PATCH 25/30] chore: bump CMake max policy to 4.2 (#5944) * chore: bupm CMake to 4.1 Signed-off-by: Henry Schreiner * ci: use newest version of CMake in a few places Signed-off-by: Henry Schreiner * chore: bupm CMake to 4.2 Signed-off-by: Henry Schreiner --------- Signed-off-by: Henry Schreiner --- .github/workflows/configure.yml | 4 ++-- CMakeLists.txt | 2 +- docs/advanced/embedding.rst | 2 +- docs/compiling.rst | 10 +++++----- tests/CMakeLists.txt | 2 +- tests/test_cmake_build/installed_embed/CMakeLists.txt | 2 +- .../test_cmake_build/installed_function/CMakeLists.txt | 2 +- tests/test_cmake_build/installed_target/CMakeLists.txt | 2 +- .../test_cmake_build/subdirectory_embed/CMakeLists.txt | 2 +- .../subdirectory_function/CMakeLists.txt | 2 +- .../subdirectory_target/CMakeLists.txt | 2 +- tools/pybind11GuessPythonExtSuffix.cmake | 2 +- tools/test-pybind11GuessPythonExtSuffix.cmake | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index cd034c883..226e3f718 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -39,10 +39,10 @@ jobs: cmake: "3.15" - runs-on: macos-14 - cmake: "4.0" + cmake: "4.2" - runs-on: windows-latest - cmake: "4.0" + cmake: "4.2" name: 🐍 3.11 • CMake ${{ matrix.cmake }} • ${{ matrix.runs-on }} runs-on: ${{ matrix.runs-on }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 806330393..097b4eba2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ if(NOT CMAKE_VERSION VERSION_LESS "3.27") cmake_policy(GET CMP0148 _pybind11_cmp0148) endif() -cmake_minimum_required(VERSION 3.15...4.0) +cmake_minimum_required(VERSION 3.15...4.2) if(_pybind11_cmp0148) cmake_policy(SET CMP0148 ${_pybind11_cmp0148}) diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index 145471b7f..c41aec152 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -18,7 +18,7 @@ information, see :doc:`/compiling`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.15...4.0) + cmake_minimum_required(VERSION 3.15...4.2) project(example) find_package(pybind11 REQUIRED) # or `add_subdirectory(pybind11)` diff --git a/docs/compiling.rst b/docs/compiling.rst index e74e3b203..b693bd587 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -18,7 +18,7 @@ A Python extension module can be created with just a few lines of code: .. code-block:: cmake - cmake_minimum_required(VERSION 3.15...4.0) + cmake_minimum_required(VERSION 3.15...4.2) project(example LANGUAGES CXX) set(PYBIND11_FINDPYTHON ON) @@ -447,7 +447,7 @@ See the `Config file`_ docstring for details of relevant CMake variables. .. code-block:: cmake - cmake_minimum_required(VERSION 3.15...4.0) + cmake_minimum_required(VERSION 3.15...4.2) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) @@ -492,7 +492,7 @@ FindPython, pybind11 will detect this and use the existing targets instead: .. code-block:: cmake - cmake_minimum_required(VERSION 3.15...4.0) + cmake_minimum_required(VERSION 3.15...4.2) project(example LANGUAGES CXX) find_package(Python 3.8 COMPONENTS Interpreter Development REQUIRED) @@ -570,7 +570,7 @@ You can use these targets to build complex applications. For example, the .. code-block:: cmake - cmake_minimum_required(VERSION 3.15...4.0) + cmake_minimum_required(VERSION 3.15...4.2) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) @@ -628,7 +628,7 @@ information about usage in C++, see :doc:`/advanced/embedding`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.15...4.0) + cmake_minimum_required(VERSION 3.15...4.2) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fdf33628a..275031c5d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,7 +5,7 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.15...4.0) +cmake_minimum_required(VERSION 3.15...4.2) # Filter out items; print an optional message if any items filtered. This ignores extensions. # diff --git a/tests/test_cmake_build/installed_embed/CMakeLists.txt b/tests/test_cmake_build/installed_embed/CMakeLists.txt index 8561ef438..a36496655 100644 --- a/tests/test_cmake_build/installed_embed/CMakeLists.txt +++ b/tests/test_cmake_build/installed_embed/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15...4.0) +cmake_minimum_required(VERSION 3.15...4.2) project(test_installed_embed CXX) diff --git a/tests/test_cmake_build/installed_function/CMakeLists.txt b/tests/test_cmake_build/installed_function/CMakeLists.txt index 8e75f86e6..d27741123 100644 --- a/tests/test_cmake_build/installed_function/CMakeLists.txt +++ b/tests/test_cmake_build/installed_function/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15...4.0) +cmake_minimum_required(VERSION 3.15...4.2) project(test_installed_function CXX) diff --git a/tests/test_cmake_build/installed_target/CMakeLists.txt b/tests/test_cmake_build/installed_target/CMakeLists.txt index d8af4391b..6ee01693d 100644 --- a/tests/test_cmake_build/installed_target/CMakeLists.txt +++ b/tests/test_cmake_build/installed_target/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15...4.0) +cmake_minimum_required(VERSION 3.15...4.2) project(test_installed_target CXX) diff --git a/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt b/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt index bb83a6e6e..81b445813 100644 --- a/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15...4.0) +cmake_minimum_required(VERSION 3.15...4.2) project(test_subdirectory_embed CXX) diff --git a/tests/test_cmake_build/subdirectory_function/CMakeLists.txt b/tests/test_cmake_build/subdirectory_function/CMakeLists.txt index f35549288..10b283dee 100644 --- a/tests/test_cmake_build/subdirectory_function/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_function/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15...4.0) +cmake_minimum_required(VERSION 3.15...4.2) project(test_subdirectory_function CXX) diff --git a/tests/test_cmake_build/subdirectory_target/CMakeLists.txt b/tests/test_cmake_build/subdirectory_target/CMakeLists.txt index aadc8c0d8..88d73f604 100644 --- a/tests/test_cmake_build/subdirectory_target/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_target/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15...4.0) +cmake_minimum_required(VERSION 3.15...4.2) project(test_subdirectory_target CXX) diff --git a/tools/pybind11GuessPythonExtSuffix.cmake b/tools/pybind11GuessPythonExtSuffix.cmake index b8c351ef0..5c2a229f8 100644 --- a/tools/pybind11GuessPythonExtSuffix.cmake +++ b/tools/pybind11GuessPythonExtSuffix.cmake @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15...4.0) +cmake_minimum_required(VERSION 3.15...4.2) function(pybind11_guess_python_module_extension python) diff --git a/tools/test-pybind11GuessPythonExtSuffix.cmake b/tools/test-pybind11GuessPythonExtSuffix.cmake index a66282db2..a9ecc29f1 100644 --- a/tools/test-pybind11GuessPythonExtSuffix.cmake +++ b/tools/test-pybind11GuessPythonExtSuffix.cmake @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15...4.0) +cmake_minimum_required(VERSION 3.15...4.2) # Tests for pybind11_guess_python_module_extension # Run using `cmake -P tools/test-pybind11GuessPythonExtSuffix.cmake` From da6e0710846caa341abfe99e2ead10136c74857e Mon Sep 17 00:00:00 2001 From: b-pass Date: Sun, 18 Jan 2026 13:24:34 -0500 Subject: [PATCH 26/30] Destruct internals during interpreter finalization (#5958) * Add a shutdown method to internals. shutdown can safely DECREF Python objects owned by the internals. * Actually free internals during interpreter shutdown (instead of after) * Make sure python is alive before DECREFing If something triggers internals to be created during finalization, it might end up being destroyed after finalization and we don't want to do the DECREF at that point, we need the leaky behavior. * make clang-tidy happy * Check IsFinalizing and use Py_CLEAR, make capsule creation safe if the capsule already exists. * oops, put TLS destructor back how it was. * Oops, proper spelling of unstable _Py_IsFinalizing * Add cleanup step to CI workflow Added a step to clean out unused files to save space during CI. * Accept suggested comment * Avoid recreating internals during type deallocation at shutdown. --------- Co-authored-by: Henry Schreiner --- .github/workflows/ci.yml | 12 +++- include/pybind11/detail/class.h | 2 +- include/pybind11/detail/internals.h | 102 ++++++++++++++++++++-------- 3 files changed, 85 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8dcf48928..45f867588 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -485,6 +485,11 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Clean out unused stuff to save space + run: | + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL + sudo apt-get clean + - name: Add NVHPC Repo run: | echo 'deb [trusted=yes] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | \ @@ -492,10 +497,11 @@ jobs: - name: Install 🐍 3 & NVHPC run: | - sudo apt-get update -y && \ - sudo apt-get install -y cmake environment-modules git python3-dev python3-pip python3-numpy && \ - sudo apt-get install -y --no-install-recommends nvhpc-25-11 && \ + sudo apt-get update -y + sudo apt-get install -y cmake environment-modules git python3-dev python3-pip python3-numpy + sudo apt-get install -y --no-install-recommends nvhpc-25-11 sudo rm -rf /var/lib/apt/lists/* + apt-cache depends nvhpc-25-11 python3 -m pip install --upgrade pip python3 -m pip install --upgrade pytest diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 21e966cfe..480c369aa 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -207,7 +207,7 @@ extern "C" inline PyObject *pybind11_meta_call(PyObject *type, PyObject *args, P /// Cleanup the type-info for a pybind11-registered type. extern "C" inline void pybind11_meta_dealloc(PyObject *obj) { - with_internals([obj](internals &internals) { + with_internals_if_internals([obj](internals &internals) { auto *type = (PyTypeObject *) obj; // A pybind11-registered type will: diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index a92f196b1..d66cf72cc 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -103,7 +103,7 @@ public: // However, in GraalPy (as of v24.2 or older), TSS is implemented by Java and this call // requires a living Python interpreter. #ifdef GRAALVM_PYTHON - if (!Py_IsInitialized() || _Py_IsFinalizing()) { + if (Py_IsInitialized() == 0 || _Py_IsFinalizing() != 0) { return; } #endif @@ -195,6 +195,14 @@ struct override_hash { using instance_map = std::unordered_multimap; +inline bool is_interpreter_alive() { +#if PY_VERSION_HEX < 0x030D0000 + return Py_IsInitialized() != 0 || _Py_IsFinalizing() != 0; +#else + return Py_IsInitialized() != 0 || Py_IsFinalizing() != 0; +#endif +} + #ifdef Py_GIL_DISABLED // Wrapper around PyMutex to provide BasicLockable semantics class pymutex { @@ -308,7 +316,17 @@ struct internals { internals(internals &&other) = delete; internals &operator=(const internals &other) = delete; internals &operator=(internals &&other) = delete; - ~internals() = default; + ~internals() { + // Normally this destructor runs during interpreter finalization and it may DECREF things. + // In odd finalization scenarios it might end up running after the interpreter has + // completely shut down, In that case, we should not decref these objects because pymalloc + // is gone. + if (is_interpreter_alive()) { + Py_CLEAR(instance_base); + Py_CLEAR(default_metaclass); + Py_CLEAR(static_property_type); + } + } }; // the internals struct (above) is shared between all the modules. local_internals are only @@ -325,6 +343,16 @@ struct local_internals { std::forward_list registered_exception_translators; PyTypeObject *function_record_py_type = nullptr; + + ~local_internals() { + // Normally this destructor runs during interpreter finalization and it may DECREF things. + // In odd finalization scenarios it might end up running after the interpreter has + // completely shut down, In that case, we should not decref these objects because pymalloc + // is gone. + if (is_interpreter_alive()) { + Py_CLEAR(function_record_py_type); + } + } }; enum class holder_enum_t : uint8_t { @@ -569,7 +597,7 @@ inline object get_python_state_dict() { // The bool follows std::map::insert convention: true = created, false = existed. template std::pair atomic_get_or_create_in_state_dict(const char *key, - bool clear_destructor = false) { + void (*dtor)(PyObject *) = nullptr) { error_scope err_scope; // preserve any existing Python error states auto state_dict = reinterpret_borrow(get_python_state_dict()); @@ -586,16 +614,13 @@ std::pair atomic_get_or_create_in_state_dict(const char *key, // Use unique_ptr for exception safety: if capsule creation throws, the storage is // automatically deleted. auto storage_ptr = std::unique_ptr(new Payload{}); - // Create capsule with destructor to clean up when the interpreter shuts down. - auto new_capsule = capsule( - storage_ptr.get(), - // The destructor will be called when the capsule is GC'ed. - // - If our capsule is inserted into the dict below, it will be kept alive until - // interpreter shutdown, so the destructor will be called at that time. - // - If our capsule is NOT inserted (another thread inserted first), it will be - // destructed when going out of scope here, so the destructor will be called - // immediately, which will also free the storage. - /*destructor=*/[](void *ptr) -> void { delete static_cast(ptr); }); + auto new_capsule + = capsule(storage_ptr.get(), + // The destructor will be called when the capsule is GC'ed. + // If the insert below fails (entry already in the dict), then this + // destructor will be called on the newly created capsule at the end of this + // function, and we want to just release this memory. + /*destructor=*/[](void *v) { delete static_cast(v); }); // At this point, the capsule object is created successfully. // Release the unique_ptr and let the capsule object own the storage to avoid double-free. (void) storage_ptr.release(); @@ -613,17 +638,16 @@ std::pair atomic_get_or_create_in_state_dict(const char *key, throw error_already_set(); } created = (capsule_obj == new_capsule.ptr()); - if (clear_destructor && created) { - // Our capsule was inserted. - // Remove the destructor to leak the storage on interpreter shutdown. - if (PyCapsule_SetDestructor(capsule_obj, nullptr) < 0) { + // - If key already existed, our `new_capsule` is not inserted, it will be destructed when + // going out of scope here, and will call the destructor set above. + // - Otherwise, our `new_capsule` is now in the dict, and it owns the storage and the state + // dict will incref it. We need to set the caller's destructor on it, which will be + // called when the interpreter shuts down. + if (created && dtor) { + if (PyCapsule_SetDestructor(capsule_obj, dtor) < 0) { throw error_already_set(); } } - // - If key already existed, our `new_capsule` is not inserted, it will be destructed when - // going out of scope here, which will also free the storage. - // - Otherwise, our `new_capsule` is now in the dict, and it owns the storage and the state - // dict will incref it. } // Get the storage pointer from the capsule. @@ -707,14 +731,27 @@ private: internals_pp_manager(char const *id, on_fetch_function *on_fetch) : holder_id_(id), on_fetch_(on_fetch) {} + static void internals_shutdown(PyObject *capsule) { + auto *pp = static_cast *>( + PyCapsule_GetPointer(capsule, nullptr)); + if (pp) { + pp->reset(); + } + // We reset the unique_ptr's contents but cannot delete the unique_ptr itself here. + // The pp_manager in this module (and possibly other modules sharing internals) holds + // a raw pointer to this unique_ptr, and that pointer would dangle if we deleted it now. + // + // For pybind11-owned interpreters (via embed.h or subinterpreter.h), destroy() is + // called after Py_Finalize/Py_EndInterpreter completes, which safely deletes the + // unique_ptr. For interpreters not owned by pybind11 (e.g., a pybind11 extension + // loaded into an external interpreter), destroy() is never called and the unique_ptr + // shell (8 bytes, not its contents) is leaked. + // (See PR #5958 for ideas to eliminate this leak.) + } + std::unique_ptr *get_or_create_pp_in_state_dict() { - // The `unique_ptr` output is leaked on interpreter shutdown. Once an - // instance is created, it will never be deleted until the process exits (compare to - // interpreter shutdown in multiple-interpreter scenarios). - // Because we cannot guarantee the order of destruction of capsules in the interpreter - // state dict, leaking avoids potential use-after-free issues during interpreter shutdown. auto result = atomic_get_or_create_in_state_dict>( - holder_id_, /*clear_destructor=*/true); + holder_id_, &internals_shutdown); auto *pp = result.first; bool created = result.second; // Only call on_fetch_ when fetching existing internals, not when creating new ones. @@ -834,6 +871,17 @@ inline auto with_internals(const F &cb) -> decltype(cb(get_internals())) { return cb(internals); } +template +inline void with_internals_if_internals(const F &cb) { + auto &ppmgr = get_internals_pp_manager(); + auto &internals_ptr = *ppmgr.get_pp(); + if (internals_ptr) { + auto &internals = *internals_ptr; + PYBIND11_LOCK_INTERNALS(internals); + cb(internals); + } +} + template inline auto with_exception_translators(const F &cb) -> decltype(cb(get_internals().registered_exception_translators, From a8e223d0cd20619195288eb864a99351c5950101 Mon Sep 17 00:00:00 2001 From: b-pass Date: Tue, 20 Jan 2026 06:51:25 -0500 Subject: [PATCH 27/30] Directly check if/which interpreter is active before doing CLEAR in destructor (#5965) * Directly check if/which interpreter is active before doing CLEAR in destructors. Py_IsFinalizing only applies to the main interpreter. * Backward compatibility fixes * Make clang-tidy happy * Add nullptr checks to istate as Cursor suggested --- include/pybind11/detail/internals.h | 91 ++++++++++++++--------------- include/pybind11/subinterpreter.h | 9 --- 2 files changed, 44 insertions(+), 56 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index d66cf72cc..9b3e69f4d 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -143,6 +143,38 @@ inline PyTypeObject *make_default_metaclass(); inline PyObject *make_object_base_type(PyTypeObject *metaclass); inline void translate_exception(std::exception_ptr p); +inline PyThreadState *get_thread_state_unchecked() { +#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) + return PyThreadState_GET(); +#elif PY_VERSION_HEX < 0x030D0000 + return _PyThreadState_UncheckedGet(); +#else + return PyThreadState_GetUnchecked(); +#endif +} + +inline PyInterpreterState *get_interpreter_state_unchecked() { + auto *tstate = get_thread_state_unchecked(); + return tstate ? tstate->interp : nullptr; +} + +inline object get_python_state_dict() { + object state_dict; +#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) + state_dict = reinterpret_borrow(PyEval_GetBuiltins()); +#else + auto *istate = get_interpreter_state_unchecked(); + if (istate) { + state_dict = reinterpret_borrow(PyInterpreterState_GetDict(istate)); + } +#endif + if (!state_dict) { + raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED"); + throw error_already_set(); + } + return state_dict; +} + // Python loads modules by default with dlopen with the RTLD_LOCAL flag; under libc++ and possibly // other STLs, this means `typeid(A)` from one module won't equal `typeid(A)` from another module // even when `A` is the same, non-hidden-visibility type (e.g. from a common include). Under @@ -195,14 +227,6 @@ struct override_hash { using instance_map = std::unordered_multimap; -inline bool is_interpreter_alive() { -#if PY_VERSION_HEX < 0x030D0000 - return Py_IsInitialized() != 0 || _Py_IsFinalizing() != 0; -#else - return Py_IsInitialized() != 0 || Py_IsFinalizing() != 0; -#endif -} - #ifdef Py_GIL_DISABLED // Wrapper around PyMutex to provide BasicLockable semantics class pymutex { @@ -293,11 +317,8 @@ struct internals { internals() : static_property_type(make_static_property_type()), - default_metaclass(make_default_metaclass()) { + default_metaclass(make_default_metaclass()), istate(get_interpreter_state_unchecked()) { tstate.set(nullptr); // See PR #5870 - PyThreadState *cur_tstate = PyThreadState_Get(); - - istate = cur_tstate->interp; registered_exception_translators.push_front(&translate_exception); #ifdef Py_GIL_DISABLED // Scale proportional to the number of cores. 2x is a heuristic to reduce contention. @@ -320,8 +341,10 @@ struct internals { // Normally this destructor runs during interpreter finalization and it may DECREF things. // In odd finalization scenarios it might end up running after the interpreter has // completely shut down, In that case, we should not decref these objects because pymalloc - // is gone. - if (is_interpreter_alive()) { + // is gone. This also applies across sub-interpreters, we should only DECREF when the + // original owning interpreter is active. + auto *cur_istate = get_interpreter_state_unchecked(); + if (cur_istate && cur_istate == istate) { Py_CLEAR(instance_base); Py_CLEAR(default_metaclass); Py_CLEAR(static_property_type); @@ -336,6 +359,8 @@ struct internals { // impact any other modules, because the only things accessing the local internals is the // module that contains them. struct local_internals { + local_internals() : istate(get_interpreter_state_unchecked()) {} + // It should be safe to use fast_type_map here because this entire // data structure is scoped to our single module, and thus a single // DSO and single instance of type_info for any particular type. @@ -343,13 +368,16 @@ struct local_internals { std::forward_list registered_exception_translators; PyTypeObject *function_record_py_type = nullptr; + PyInterpreterState *istate = nullptr; ~local_internals() { // Normally this destructor runs during interpreter finalization and it may DECREF things. // In odd finalization scenarios it might end up running after the interpreter has // completely shut down, In that case, we should not decref these objects because pymalloc - // is gone. - if (is_interpreter_alive()) { + // is gone. This also applies across sub-interpreters, we should only DECREF when the + // original owning interpreter is active. + auto *cur_istate = get_interpreter_state_unchecked(); + if (cur_istate && cur_istate == istate) { Py_CLEAR(function_record_py_type); } } @@ -436,16 +464,6 @@ struct native_enum_record { "__pybind11_module_local_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID "__" -inline PyThreadState *get_thread_state_unchecked() { -#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) - return PyThreadState_GET(); -#elif PY_VERSION_HEX < 0x030D0000 - return _PyThreadState_UncheckedGet(); -#else - return PyThreadState_GetUnchecked(); -#endif -} - /// We use this to figure out if there are or have been multiple subinterpreters active at any /// point. This must never go from true to false while any interpreter may be running in any /// thread! @@ -558,27 +576,6 @@ inline void translate_local_exception(std::exception_ptr p) { } #endif -inline object get_python_state_dict() { - object state_dict; -#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) - state_dict = reinterpret_borrow(PyEval_GetBuiltins()); -#else -# if PY_VERSION_HEX < 0x03090000 - PyInterpreterState *istate = _PyInterpreterState_Get(); -# else - PyInterpreterState *istate = PyInterpreterState_Get(); -# endif - if (istate) { - state_dict = reinterpret_borrow(PyInterpreterState_GetDict(istate)); - } -#endif - if (!state_dict) { - raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED"); - throw error_already_set(); - } - return state_dict; -} - // Get or create per-storage capsule in the current interpreter's state dict. // - The storage is interpreter-dependent: different interpreters will have different storage. // This is important when using multiple-interpreters, to avoid sharing unshareable objects diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index c47787b6e..547545263 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -20,15 +20,6 @@ #endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) -PYBIND11_NAMESPACE_BEGIN(detail) -inline PyInterpreterState *get_interpreter_state_unchecked() { - auto *cur_tstate = get_thread_state_unchecked(); - if (cur_tstate) { - return cur_tstate->interp; - } - return nullptr; -} -PYBIND11_NAMESPACE_END(detail) class subinterpreter; From 53ccc32f84f94627fbb7a32ef217dfb7b6a4c27f Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 20 Jan 2026 22:35:16 +0000 Subject: [PATCH 28/30] fix: cross-compilation updates (#5829) * Android updates * Revert removal of Interpreter --- tests/CMakeLists.txt | 4 ++- tools/pybind11Common.cmake | 13 +++++++- tools/pybind11GuessPythonExtSuffix.cmake | 30 ++++++++++++------- tools/pybind11NewTools.cmake | 26 ++++------------ tools/test-pybind11GuessPythonExtSuffix.cmake | 24 +++++++++++++++ 5 files changed, 64 insertions(+), 33 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 275031c5d..9a35052da 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -545,7 +545,9 @@ source_group( FILES ${PYBIND11_HEADERS}) # Make sure pytest is found or produce a warning -pybind11_find_import(pytest VERSION 3.1) +if(NOT CMAKE_CROSSCOMPILING) + pybind11_find_import(pytest VERSION 3.1) +endif() if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) # This is not used later in the build, so it's okay to regenerate each time. diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index 1c1d4f88a..da887a225 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -41,7 +41,18 @@ set(pybind11_INCLUDE_DIRS "${pybind11_INCLUDE_DIR}" CACHE INTERNAL "Include directory for pybind11 (Python not requested)") -if(CMAKE_CROSSCOMPILING AND PYBIND11_USE_CROSSCOMPILING) +# CMP0190 prohibits calling FindPython with both Interpreter and Development components +# when cross-compiling, unless the CMAKE_CROSSCOMPILING_EMULATOR variable is defined. +if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.1") + cmake_policy(GET CMP0190 _pybind11_cmp0190) + if(_pybind11_cmp0190 STREQUAL "NEW") + set(PYBIND11_USE_CROSSCOMPILING "ON") + endif() +endif() + +if(CMAKE_CROSSCOMPILING + AND PYBIND11_USE_CROSSCOMPILING + AND NOT DEFINED CMAKE_CROSSCOMPILING_EMULATOR) set(_PYBIND11_CROSSCOMPILING ON CACHE INTERNAL "") diff --git a/tools/pybind11GuessPythonExtSuffix.cmake b/tools/pybind11GuessPythonExtSuffix.cmake index 5c2a229f8..f14573903 100644 --- a/tools/pybind11GuessPythonExtSuffix.cmake +++ b/tools/pybind11GuessPythonExtSuffix.cmake @@ -14,15 +14,23 @@ function(pybind11_guess_python_module_extension python) STRING "Extension suffix for Python extension modules (Initialized from SETUPTOOLS_EXT_SUFFIX)") endif() + + # The final extension depends on the system + set(_PY_BUILD_EXTENSION "${CMAKE_SHARED_MODULE_SUFFIX}") + if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(_PY_BUILD_EXTENSION ".pyd") + endif() + + # If running under scikit-build-core, use the SKBUILD_SOABI variable: + if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED SKBUILD_SOABI) + message(STATUS "Determining Python extension suffix based on SKBUILD_SOABI: ${SKBUILD_SOABI}") + set(PYTHON_MODULE_EXT_SUFFIX ".${SKBUILD_SOABI}${_PY_BUILD_EXTENSION}") + endif() + # If that didn't work, use the Python_SOABI variable: if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ${python}_SOABI) message( STATUS "Determining Python extension suffix based on ${python}_SOABI: ${${python}_SOABI}") - # The final extension depends on the system - set(_PY_BUILD_EXTENSION "${CMAKE_SHARED_MODULE_SUFFIX}") - if(CMAKE_SYSTEM_NAME STREQUAL "Windows") - set(_PY_BUILD_EXTENSION ".pyd") - endif() # If the SOABI already has an extension, use it as the full suffix # (used for debug versions of Python on Windows) if(${python}_SOABI MATCHES "\\.") @@ -43,9 +51,9 @@ function(pybind11_guess_python_module_extension python) # If we could not deduce the extension suffix, unset the results: if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX) - unset(PYTHON_MODULE_DEBUG_POSTFIX PARENT_SCOPE) - unset(PYTHON_MODULE_EXTENSION PARENT_SCOPE) - unset(PYTHON_IS_DEBUG PARENT_SCOPE) + unset(PYTHON_MODULE_DEBUG_POSTFIX CACHE) + unset(PYTHON_MODULE_EXTENSION CACHE) + unset(PYTHON_IS_DEBUG CACHE) return() endif() @@ -75,12 +83,12 @@ function(pybind11_guess_python_module_extension python) # Return results set(PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_DEBUG_POSTFIX}" - PARENT_SCOPE) + CACHE INTERNAL "") set(PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXTENSION}" - PARENT_SCOPE) + CACHE INTERNAL "") set(PYTHON_IS_DEBUG "${_PYTHON_IS_DEBUG}" - PARENT_SCOPE) + CACHE INTERNAL "") endfunction() diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index e881ca7ca..5ab3142e4 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -106,18 +106,7 @@ if(PYBIND11_MASTER_PROJECT) endif() endif() -if(NOT _PYBIND11_CROSSCOMPILING) - # If a user finds Python, they may forget to include the Interpreter component - # and the following two steps require it. It is highly recommended by CMake - # when finding development libraries anyway, so we will require it. - if(NOT DEFINED ${_Python}_EXECUTABLE) - message( - FATAL_ERROR - "${_Python} was found without the Interpreter component. Pybind11 requires this component." - ) - - endif() - +if(NOT _PYBIND11_CROSSCOMPILING AND DEFINED ${_Python}_EXECUTABLE) if(DEFINED PYBIND11_PYTHON_EXECUTABLE_LAST AND NOT ${_Python}_EXECUTABLE STREQUAL PYBIND11_PYTHON_EXECUTABLE_LAST) # Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed @@ -190,15 +179,15 @@ else() include("${CMAKE_CURRENT_LIST_DIR}/pybind11GuessPythonExtSuffix.cmake") pybind11_guess_python_module_extension("${_Python}") endif() - # When cross-compiling, we cannot query the Python interpreter, so we require - # the user to set these variables explicitly. if(NOT DEFINED PYTHON_IS_DEBUG OR NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) message( FATAL_ERROR - "When cross-compiling, you should set the PYTHON_IS_DEBUG, PYTHON_MODULE_EXTENSION and PYTHON_MODULE_DEBUG_POSTFIX \ - variables appropriately before loading pybind11 (e.g. in your CMake toolchain file)") + "A Python interpreter was not found, or you are cross-compiling, and the " + "PYTHON_IS_DEBUG, PYTHON_MODULE_EXTENSION and PYTHON_MODULE_DEBUG_POSTFIX " + "variables could not be guessed. Set these variables appropriately before " + "loading pybind11 (e.g. in your CMake toolchain file)") endif() endif() @@ -248,10 +237,7 @@ if(TARGET ${_Python}::Module) # files. get_target_property(module_target_type ${_Python}::Module TYPE) if(ANDROID AND module_target_type STREQUAL INTERFACE_LIBRARY) - set_property( - TARGET ${_Python}::Module - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES "${${_Python}_LIBRARIES}") + target_link_libraries(${_Python}::Module INTERFACE ${${_Python}_LIBRARIES}) endif() set_property( diff --git a/tools/test-pybind11GuessPythonExtSuffix.cmake b/tools/test-pybind11GuessPythonExtSuffix.cmake index a9ecc29f1..1976abf26 100644 --- a/tools/test-pybind11GuessPythonExtSuffix.cmake +++ b/tools/test-pybind11GuessPythonExtSuffix.cmake @@ -87,6 +87,30 @@ unset(PYTHON_MODULE_EXT_SUFFIX) unset(PYTHON_MODULE_EXT_SUFFIX CACHE) unset(ENV{SETUPTOOLS_EXT_SUFFIX}) +# Check the priority of the possible suffix sources. +set(ENV{SETUPTOOLS_EXT_SUFFIX} ".from-setuptools.pyd") +set(SKBUILD_SOABI "from-skbuild") +set(Python3_SOABI "from-python3") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".from-setuptools.pyd") + +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) +unset(ENV{SETUPTOOLS_EXT_SUFFIX}) +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".from-skbuild.pyd") + +unset(SKBUILD_SOABI) +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".from-python3.pyd") + +set(Python3_SOABI "") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".pyd") + +unset(Python3_SOABI) +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" "") + # macOS set(CMAKE_SYSTEM_NAME "Darwin") set(CMAKE_SHARED_MODULE_SUFFIX ".so") From 5a6edc999831ed4fab7cd8ced672e6c1906dea81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 21 Jan 2026 06:49:59 +0100 Subject: [PATCH 29/30] Exclude MSVC up to 19.16 from using std::launder (#5968) * Exclude further MSVC versions from std::launder Versions 19.4, 19.5 and 19.6 now also excluded. Error seen with 19.6, error triggered by this commit: https://github.com/pybind/pybind11/commit/57b9a0af815d19b236b74be06a172bc5c9956618 _deps\fetchedpybind11-src\include\pybind11\pybind11.h(3008): fatal error C1001: An internal error has occurred in the compiler. [C:\projects\openpmd-api\build\openPMD.py.vcxproj] (compiler file 'd:\agent\_work\8\s\src\vctools\compiler\utc\src\p2\main.c', line 187) To work around this problem, try simplifying or changing the program near the locations listed above. Please choose the Technical Support command on the Visual C++ Help menu, or open the Technical Support help file for more information * Add minimal comment // See PR #5968 --------- Co-authored-by: Ralf W. Grosse-Kunstleve --- include/pybind11/detail/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 4406adf5b..19ebc8532 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -87,7 +87,7 @@ # endif #endif -#if defined(__cpp_lib_launder) && !(defined(_MSC_VER) && (_MSC_VER < 1914)) +#if defined(__cpp_lib_launder) && !(defined(_MSC_VER) && (_MSC_VER < 1920)) // See PR #5968 # define PYBIND11_STD_LAUNDER std::launder # define PYBIND11_HAS_STD_LAUNDER 1 #else From 6c836071ad7725d3f0923a3e2b6928701ebf0100 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 21 Jan 2026 13:51:18 +0700 Subject: [PATCH 30/30] Changelog updates (#5967) * docs: seed 3.0.2 changelog from needs-changelog PRs Collect suggested entries early to streamline release prep. * Misc trivial manual fixes. * Shorten changelog entry for PR 5862 * Remove mention of a minor doc formatting fix. * Cursor-generated "all past-tense" style * Restore the meaning of the 5958 entry using the "... now ..." trick, and restore a couple other entries that also use the "now" trick. * Replace ... now ... style with ... updated to ... style * [skip ci] docs: group 3.0.2 entries under Internal heading Align changelog categories with recent releases for review. * Update changelog with CMake policy compatibility fix Fix compatibility with CMake policy CMP0190 for cross-compiling. * Add changelog entries for 5965 and 5968 * docs: make CMP0190 changelog entry past tense Align 3.0.2 bug-fix entry with changelog style. * [skip ci] docs: add missing 3.0.2 changelog entries Capture remaining needs-changelog PRs across categories. (These slipped through the cracks somehow.) --------- Co-authored-by: Henry Schreiner --- docs/changelog.md | 155 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 4bf23a697..2a232c86a 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -13,6 +13,161 @@ Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. +## Version 3.0.2 (release date TBD) + +Bug fixes: + +- MSVC 19.16 and earlier were blocked from using `std::launder` due to internal compiler errors. + [#5968](https://github.com/pybind/pybind11/pull/5968) + +- Internals destructors were updated to check the owning interpreter before clearing Python objects. + [#5965](https://github.com/pybind/pybind11/pull/5965) + +- pybind11 internals were updated to be deallocated during (sub-)interpreter shutdown to avoid memory leaks. + [#5958](https://github.com/pybind/pybind11/pull/5958) + +- Fixed ambiguous `str(handle)` construction for `object`-derived types like `kwargs` or `dict` by templatizing the constructor with SFINAE. + [#5949](https://github.com/pybind/pybind11/pull/5949) + +- Fixed concurrency consistency for `internals_pp_manager` under multiple-interpreters. + [#5947](https://github.com/pybind/pybind11/pull/5947) + +- Fixed MSVC LNK2001 in C++20 builds when /GL (whole program optimization) is enabled. + [#5939](https://github.com/pybind/pybind11/pull/5939) + +- Added per-interpreter storage for `gil_safe_call_once_and_store` to make it safe under multi-interpreters. + [#5933](https://github.com/pybind/pybind11/pull/5933) + +- A workaround for a GCC `-Warray-bounds` false positive in `argument_vector` was added. + [#5908](https://github.com/pybind/pybind11/pull/5908) + +- Corrected a mistake where support for `__index__` was added, but the type hints did not reflect acceptance of `SupportsIndex` objects. Also fixed a long-standing bug: the complex-caster did not accept `__index__` in `convert` mode. + [#5891](https://github.com/pybind/pybind11/pull/5891) + +- Fixed `*args/**kwargs` return types. Added type hinting to `py::make_tuple`. + [#5881](https://github.com/pybind/pybind11/pull/5881) + +- Fixed compiler error in `type_caster_generic` when casting a `T` implicitly convertible from `T*`. + [#5873](https://github.com/pybind/pybind11/pull/5873) + +- Updated `py::native_enum` bindings to unregister enum types on destruction, preventing a use-after-free when returning a destroyed enum instance. + [#5871](https://github.com/pybind/pybind11/pull/5871) + +- Fixed undefined behavior that occurred when importing pybind11 modules from non-main threads created by C API modules or embedded python interpreters. + [#5870](https://github.com/pybind/pybind11/pull/5870) + +- Fixed dangling pointer in `internals::registered_types_cpp_fast`. + [#5867](https://github.com/pybind/pybind11/pull/5867) + +- Added support for `std::shared_ptr` when loading module-local or conduit types from other modules. + [#5862](https://github.com/pybind/pybind11/pull/5862) + +- Fixed thread-safety issues if types were concurrently registered while `get_local_type_info()` was called in free threaded Python. + [#5856](https://github.com/pybind/pybind11/pull/5856) + +- Fixed py::float_ casting and py::int_ and py::float_ type hints. + [#5839](https://github.com/pybind/pybind11/pull/5839) + +- Fixed two `smart_holder` bugs in `shared_ptr` and `unique_ptr` adoption with multiple/virtual inheritance: + - `shared_ptr` to-Python caster was updated to register the correct subobject pointer (fixes #5786). + - `unique_ptr` adoption was updated to own the proper object start while aliasing subobject pointers for registration, which fixed MSVC crashes during destruction. + [#5836](https://github.com/pybind/pybind11/pull/5836) + +- Constrained `accessor::operator=` templates to avoid obscuring special members. + [#5832](https://github.com/pybind/pybind11/pull/5832) + +- Fixed crash that can occur when finalizers acquire and release the GIL. + [#5828](https://github.com/pybind/pybind11/pull/5828) + +- Fixed compiler detection in `pybind11/detail/pybind11_namespace_macros.h` for clang-cl on Windows, to address warning suppression macros. + [#5816](https://github.com/pybind/pybind11/pull/5816) + +- Fixed compatibility with CMake policy CMP0190 by not always requiring a Python interpreter when cross-compiling. + [#5829](https://github.com/pybind/pybind11/pull/5829) + +- Added a static assertion to disallow `keep_alive` and `call_guard` on properties. + [#5533](https://github.com/pybind/pybind11/pull/5533) + +Internal: + +- CMake policy limit was set to 4.1. + [#5944](https://github.com/pybind/pybind11/pull/5944) + +- Improved performance of function calls between Python and C++ by switching to the "vectorcall" calling protocol. + [#5948](https://github.com/pybind/pybind11/pull/5948) + +- Many C-style casts were replaced with C++-style casts. + [#5930](https://github.com/pybind/pybind11/pull/5930) + +- Added `cast_sources` abstraction to `type_caster_generic`. + [#5866](https://github.com/pybind/pybind11/pull/5866) + +- Improved the performance of from-Python conversions of legacy pybind11 enum objects bound by `py::enum_`. + [#5860](https://github.com/pybind/pybind11/pull/5860) + +- Reduced size overhead by deduplicating functions' readable signatures and type information. + [#5857](https://github.com/pybind/pybind11/pull/5857) + +- Used new Python 3.14 C APIs when available. + [#5854](https://github.com/pybind/pybind11/pull/5854) + +- Improved performance of function dispatch and type casting by porting two-level type info lookup strategy from nanobind. + [#5842](https://github.com/pybind/pybind11/pull/5842) + +- Updated `.gitignore` to exclude `__pycache__/` directories. + [#5838](https://github.com/pybind/pybind11/pull/5838) + +- Changed internals to use `thread_local` instead of `thread_specific_storage` for increased performance. + [#5834](https://github.com/pybind/pybind11/pull/5834) + +- Reduced function call overhead by using thread_local for loader_life_support when possible. + [#5830](https://github.com/pybind/pybind11/pull/5830) + +- Removed heap allocation for the C++ argument array when dispatching functions with 6 or fewer arguments. + [#5824](https://github.com/pybind/pybind11/pull/5824) + + +Documentation: + +- Fixed docstring for `long double` complex types to use `numpy.clongdouble` instead of the deprecated `numpy.longcomplex` (removed in NumPy 2.0). + [#5952](https://github.com/pybind/pybind11/pull/5952) + +- The "Supported compilers" and "Supported platforms" sections in the main `README.rst` were replaced with a new "Supported platforms & compilers" section that points to the CI test matrix as the living source of truth. + [#5910](https://github.com/pybind/pybind11/pull/5910) + +- Fixed documentation formatting. + [#5903](https://github.com/pybind/pybind11/pull/5903) + +- Updated upgrade notes for `py::native_enum`. + [#5885](https://github.com/pybind/pybind11/pull/5885) + +- Clarified in the docs to what extent bindings are global. + [#5859](https://github.com/pybind/pybind11/pull/5859) + + +Tests: + +- Calls to `env.deprecated_call()` were replaced with direct calls to `pytest.deprecated_call()`. + [#5893](https://github.com/pybind/pybind11/pull/5893) + +- Updated pytest configuration to use `log_level` instead of `log_cli_level`. + [#5890](https://github.com/pybind/pybind11/pull/5890) + + +CI: + +- Added CI tests for windows-11-arm with clang/MSVC (currently python 3.13), windows-11-arm with clang/mingw (currently python 3.12). + [#5932](https://github.com/pybind/pybind11/pull/5932) + +- These clang-tidy rules were added: `readability-redundant-casting`, `readability-redundant-inline-specifier`, `readability-redundant-member-init` + [#5924](https://github.com/pybind/pybind11/pull/5924) + +- Replaced deprecated macos-13 runners with macos-15-intel in CI. + [#5916](https://github.com/pybind/pybind11/pull/5916) + +- Restored `runs-on: windows-latest` in CI. + [#5835](https://github.com/pybind/pybind11/pull/5835) + ## Version 3.0.1 (August 22, 2025) Bug fixes: