native_enum: add capsule containing enum information and cleanup logic (#5871)

* native_enum: add capsule containing enum information and cleanup logic

* style: pre-commit fixes

* Updates from code review

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

tests/conftest.py

@@ -16,6 +16,7 @@ import sys
 import sysconfig
 import textwrap
 import traceback
+import weakref
 from typing import Callable
 
 import pytest
@@ -208,19 +209,54 @@ def pytest_assertrepr_compare(op, left, right):  # noqa: ARG001
     return None
 
 
+# Number of times we think repeatedly collecting garbage might do anything.
+# The only reason to do more than once is because finalizers executed during
+# one GC pass could create garbage that can't be collected until a future one.
+# This quickly produces diminishing returns, and GC passes can be slow, so this
+# value is a tradeoff between non-flakiness and fast tests. (It errs on the
+# side of non-flakiness; many uses of this idiom only do 3 passes.)
+num_gc_collect = 5
+
+
 def gc_collect():
-    """Run the garbage collector three times (needed when running
+    """Run the garbage collector several times (needed when running
     reference counting tests with PyPy)"""
-    gc.collect()
-    gc.collect()
-    gc.collect()
-    gc.collect()
-    gc.collect()
+    for _ in range(num_gc_collect):
+        gc.collect()
+
+
+def delattr_and_ensure_destroyed(*specs):
+    """For each of the given *specs* (a tuple of the form ``(scope, name)``),
+    perform ``delattr(scope, name)``, then do enough GC collections that the
+    deleted reference has actually caused the target to be destroyed. This is
+    typically used to test what happens when a type object is destroyed; if you
+    use it for that, you should be aware that extension types, or all types,
+    are immortal on some Python versions. See ``env.TYPES_ARE_IMMORTAL``.
+    """
+    wrs = []
+    for mod, name in specs:
+        wrs.append(weakref.ref(getattr(mod, name)))
+        delattr(mod, name)
+
+    for _ in range(num_gc_collect):
+        gc.collect()
+        if all(wr() is None for wr in wrs):
+            break
+    else:
+        # If this fires, most likely something is still holding a reference
+        # to the object you tried to destroy - for example, it's a type that
+        # still has some instances alive. Try setting a breakpoint here and
+        # examining `gc.get_referrers(wrs[0]())`. It's vaguely possible that
+        # num_gc_collect needs to be increased also.
+        pytest.fail(
+            f"Could not delete bindings such as {next(wr for wr in wrs if wr() is not None)!r}"
+        )
 
 
 def pytest_configure():
     pytest.suppress = contextlib.suppress
     pytest.gc_collect = gc_collect
+    pytest.delattr_and_ensure_destroyed = delattr_and_ensure_destroyed
 
 
 def pytest_report_header():

tests/env.py

@@ -24,6 +24,12 @@ PY_GIL_DISABLED = bool(sysconfig.get_config_var("Py_GIL_DISABLED"))
 # Runtime state (what's actually happening now)
 sys_is_gil_enabled = getattr(sys, "_is_gil_enabled", lambda: True)
 
+TYPES_ARE_IMMORTAL = (
+    PYPY
+    or GRAALPY
+    or (CPYTHON and PY_GIL_DISABLED and (3, 13) <= sys.version_info < (3, 14))
+)
+
 
 def deprecated_call():
     """

tests/test_class_cross_module_use_after_one_module_dealloc.py

@@ -1,40 +1,16 @@
 from __future__ import annotations
 
-import gc
-import sys
-import sysconfig
 import types
-import weakref
 
 import pytest
 
 import env
 from pybind11_tests import class_cross_module_use_after_one_module_dealloc as m
 
-is_python_3_13_free_threaded = (
-    env.CPYTHON
-    and sysconfig.get_config_var("Py_GIL_DISABLED")
-    and (3, 13) <= sys.version_info < (3, 14)
+
+@pytest.mark.skipif(
+    env.TYPES_ARE_IMMORTAL, reason="can't GC type objects on this platform"
 )
-
-
-def delattr_and_ensure_destroyed(*specs):
-    wrs = []
-    for mod, name in specs:
-        wrs.append(weakref.ref(getattr(mod, name)))
-        delattr(mod, name)
-
-    for _ in range(5):
-        gc.collect()
-        if all(wr() is None for wr in wrs):
-            break
-    else:
-        pytest.fail(
-            f"Could not delete bindings such as {next(wr for wr in wrs if wr() is not None)!r}"
-        )
-
-
-@pytest.mark.skipif("env.PYPY or env.GRAALPY or is_python_3_13_free_threaded")
 def test_cross_module_use_after_one_module_dealloc():
     # This is a regression test for a bug that occurred during development of
     # internals::registered_types_cpp_fast (see #5842). registered_types_cpp_fast maps
@@ -58,7 +34,7 @@ def test_cross_module_use_after_one_module_dealloc():
     cm.consume_cross_dso_class(instance)
 
     del instance
-    delattr_and_ensure_destroyed((module_scope, "CrossDSOClass"))
+    pytest.delattr_and_ensure_destroyed((module_scope, "CrossDSOClass"))
 
     # Make sure that CrossDSOClass gets allocated at a different address.
     m.register_unrelated_class(module_scope)

tests/test_native_enum.cpp

@@ -93,10 +93,14 @@ TEST_SUBMODULE(native_enum, m) {
         .value("blue", color::blue)
         .finalize();
 
-    py::native_enum<altitude>(m, "altitude", "enum.Enum")
-        .value("high", altitude::high)
-        .value("low", altitude::low)
-        .finalize();
+    m.def("bind_altitude", [](const py::module_ &mod) {
+        py::native_enum<altitude>(mod, "altitude", "enum.Enum")
+            .value("high", altitude::high)
+            .value("low", altitude::low)
+            .finalize();
+    });
+    m.def("is_high_altitude", [](altitude alt) { return alt == altitude::high; });
+    m.def("get_altitude", []() -> altitude { return altitude::high; });
 
     py::native_enum<flags_uchar>(m, "flags_uchar", "enum.Flag")
         .value("bit0", flags_uchar::bit0)

tests/test_native_enum.py

@@ -59,7 +59,6 @@ FUNC_SIG_RENDERING_MEMBERS = ()
 ENUM_TYPES_AND_MEMBERS = (
     (m.smallenum, SMALLENUM_MEMBERS),
     (m.color, COLOR_MEMBERS),
-    (m.altitude, ALTITUDE_MEMBERS),
     (m.flags_uchar, FLAGS_UCHAR_MEMBERS),
     (m.flags_uint, FLAGS_UINT_MEMBERS),
     (m.export_values, EXPORT_VALUES_MEMBERS),
@@ -320,3 +319,59 @@ def test_native_enum_missing_finalize_failure():
     if not isinstance(m.native_enum_missing_finalize_failure, str):
         m.native_enum_missing_finalize_failure()
         pytest.fail("Process termination expected.")
+
+
+def test_unregister_native_enum_when_destroyed():
+    # For stability when running tests in parallel, this test should be the
+    # only one that touches `m.altitude` or calls `m.bind_altitude`.
+
+    def test_altitude_enum():
+        # Logic copied from test_enum_type / test_enum_members.
+        # We don't test altitude there to avoid possible clashes if
+        # parallelizing against other tests in this file, and we also
+        # don't want to hold any references to the enumerators that
+        # would prevent GCing the enum type below.
+        assert isinstance(m.altitude, enum.EnumMeta)
+        assert m.altitude.__module__ == m.__name__
+        for name, value in ALTITUDE_MEMBERS:
+            assert m.altitude[name].value == value
+
+    def test_altitude_binding():
+        assert m.is_high_altitude(m.altitude.high)
+        assert not m.is_high_altitude(m.altitude.low)
+        assert m.get_altitude() is m.altitude.high
+        with pytest.raises(TypeError, match="incompatible function arguments"):
+            m.is_high_altitude("oops")
+
+    m.bind_altitude(m)
+    test_altitude_enum()
+    test_altitude_binding()
+
+    if env.TYPES_ARE_IMMORTAL:
+        pytest.skip("can't GC type objects on this platform")
+
+    # Delete the enum type. Returning an instance from Python should fail
+    # rather than accessing a deleted object.
+    pytest.delattr_and_ensure_destroyed((m, "altitude"))
+    with pytest.raises(TypeError, match="Unable to convert function return"):
+        m.get_altitude()
+    with pytest.raises(TypeError, match="incompatible function arguments"):
+        m.is_high_altitude("oops")
+
+    # Recreate the enum type; should not have any duplicate-binding error
+    m.bind_altitude(m)
+    test_altitude_enum()
+    test_altitude_binding()
+
+    # Remove the pybind11 capsule without removing the type; enum is still
+    # usable but can't be passed to/from bound functions
+    del m.altitude.__pybind11_native_enum__
+    pytest.gc_collect()
+    test_altitude_enum()  # enum itself still works
+
+    with pytest.raises(TypeError, match="Unable to convert function return"):
+        m.get_altitude()
+    with pytest.raises(TypeError, match="incompatible function arguments"):
+        m.is_high_altitude(m.altitude.high)
+
+    del m.altitude