Fix build failure when shared_ptr<T> points to private std::enable_shared_from_this base (#5590)

* Squashed private_esft/manuscript — 3f2b1201b830d9e431448bd8f5fe577afaa02dbf — 2025-03-30 12:38:30 -0700

[Browse private_esft/manuscript tree](3f2b1201b8)

[Browse private_esft/manuscript commits](3f2b1201b8/)

* Remove mention of ChatGPT

Co-authored-by: Henry Schreiner <HenrySchreinerIII@gmail.com>

---------

Co-authored-by: Henry Schreiner <HenrySchreinerIII@gmail.com>

include/pybind11/detail/struct_smart_holder.h

@@ -62,13 +62,23 @@ Details:
 namespace pybindit {
 namespace memory {
 
+// Default fallback.
 static constexpr bool type_has_shared_from_this(...) { return false; }
 
+// This overload uses SFINAE to skip enable_shared_from_this checks when the
+// base is inaccessible (e.g. private inheritance).
 template <typename T>
-static constexpr bool type_has_shared_from_this(const std::enable_shared_from_this<T> *) {
+static auto type_has_shared_from_this(const T *ptr)
+    -> decltype(static_cast<const std::enable_shared_from_this<T> *>(ptr), true) {
     return true;
 }
 
+// Inaccessible base → substitution failure → fallback overload selected
+template <typename T>
+static constexpr bool type_has_shared_from_this(const void *) {
+    return false;
+}
+
 struct guarded_delete {
     std::weak_ptr<void> released_ptr;    // Trick to keep the smart_holder memory footprint small.
     std::function<void(void *)> del_fun; // Rare case.

tests/pure_cpp/smart_holder_poc.h

@@ -10,6 +10,10 @@ namespace pybindit {
 namespace memory {
 namespace smart_holder_poc { // Proof-of-Concept implementations.
 
+struct PrivateESFT : private std::enable_shared_from_this<PrivateESFT> {};
+static_assert(!pybindit::memory::type_has_shared_from_this(static_cast<PrivateESFT *>(nullptr)),
+              "should detect inaccessible base");
+
 template <typename T>
 T &as_lvalue_ref(const smart_holder &hld) {
     static const char *context = "as_lvalue_ref";

tests/test_smart_ptr.cpp

@@ -473,4 +473,13 @@ TEST_SUBMODULE(smart_ptr, m) {
             }
             return list;
         });
+
+    class PrivateESFT : /* implicit private */ std::enable_shared_from_this<PrivateESFT> {};
+    struct ContainerUsingPrivateESFT {
+        std::shared_ptr<PrivateESFT> ptr;
+    };
+    py::class_<ContainerUsingPrivateESFT>(m, "ContainerUsingPrivateESFT")
+        .def(py::init<>())
+        .def_readwrite("ptr",
+                       &ContainerUsingPrivateESFT::ptr); // <- access ESFT through shared_ptr
 }

tests/test_smart_ptr.py

@@ -326,3 +326,15 @@ def test_shared_ptr_gc():
     pytest.gc_collect()
     for i, v in enumerate(el.get()):
         assert i == v.value()
+
+
+def test_private_esft_tolerance():
+    # Regression test: binding a shared_ptr<T> member where T privately inherits
+    # enable_shared_from_this<T> must not cause a C++ compile error.
+    c = m.ContainerUsingPrivateESFT()
+    # The ptr member is not actually usable in any way, but this is how the
+    # pybind11 v2 release series worked.
+    with pytest.raises(TypeError):
+        _ = c.ptr  # getattr
+    with pytest.raises(TypeError):
+        c.ptr = None  # setattr