Enable Conversions Between Native Python Enum Types and C++ Enums (#5555)

* Apply smart_holder-branch-based PR #5280 on top of master.

* Add pytest.skip("GraalPy does not raise UnicodeDecodeError")

* Add `parent_scope` as first argument to `py::native_enum` ctor.

* Replace `operator+=` API with `.finalize()` API. The error messages still need cleanup.

* Resolve clang-tidy performance-unnecessary-value-param errors

* Rename (effectively) native_enum_add_to_parent() -> finalize()

* Update error message: pybind11::native_enum<...>("Fake", ...): MISSING .finalize()

* Pass py::module_ by reference to resolve clang-tidy errors (this is entirely inconsequential otherwise for all practical purposes).

* test_native_enum_correct_use_failure -> test_native_enum_missing_finalize_failure

* Add test_native_enum_double_finalize(), test_native_enum_value_after_finalize()

* Clean up public/protected API.

* [ci skip] Update the Enumerations section in classes.rst

* Rename `py::native_enum_kind` → `py::enum_kind` as suggested by gh-henryiii:

https://github.com/pybind/pybind11/pull/5555#issuecomment-2711672335

* Experiment: StrEnum

enum.StrEnum does not map to C++ enum:

* https://chatgpt.com/share/67d5e965-ccb0-8008-95b7-0df2502309b3

```
============================= test session starts ==============================
platform linux -- Python 3.12.3, pytest-8.3.3, pluggy-1.5.0
C++ Info: 13.3.0 C++20 __pybind11_internals_v10000000_system_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_1__ PYBIND11_SIMPLE_GIL_MANAGEMENT=False PYBIND11_NUMPY_1_ONLY=False
configfile: pytest.ini
plugins: parallel-0.1.1, xdist-3.6.1
collected 40 items / 39 deselected / 1 selected

test_native_enum.py F                                                    [100%]

=================================== FAILURES ===================================
________________________ test_native_enum_StrEnum_greek ________________________

    def test_native_enum_StrEnum_greek():
        assert not hasattr(m, "greek")
>       m.native_enum_StrEnum_greek(m)

test_native_enum.py:150:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/usr/lib/python3.12/enum.py:764: in __call__
    return cls._create_(
        boundary   = None
        cls        = <enum 'StrEnum'>
        module     = None
        names      = [('Alpha', 10), ('Omega', 20)]
        qualname   = None
        start      = 1
        type       = None
        value      = 'greek'
        values     = ()
/usr/lib/python3.12/enum.py:917: in _create_
    return metacls.__new__(metacls, class_name, bases, classdict, boundary=boundary)
        _          = <class 'str'>
        bases      = (<enum 'StrEnum'>,)
        boundary   = None
        class_name = 'greek'
        classdict  = {'_generate_next_value_': <function StrEnum._generate_next_value_ at 0x701ec1711e40>, 'Alpha': 10, 'Omega': 20, '__module__': 'test_native_enum'}
        cls        = <enum 'StrEnum'>
        first_enum = <enum 'StrEnum'>
        item       = ('Omega', 20)
        member_name = 'Omega'
        member_value = 20
        metacls    = <class 'enum.EnumType'>
        module     = 'test_native_enum'
        names      = [('Alpha', 10), ('Omega', 20)]
        qualname   = None
        start      = 1
        type       = None
/usr/lib/python3.12/enum.py:606: in __new__
    raise exc.with_traceback(tb)
        __class__  = <class 'enum.EnumType'>
        __new__    = <function StrEnum.__new__ at 0x701ec1711da0>
        _gnv       = <staticmethod(<function StrEnum._generate_next_value_ at 0x701ec1711e40>)>
        _order_    = None
        _simple    = False
        bases      = (<enum 'StrEnum'>,)
        boundary   = None
        classdict  = {'Alpha': <enum._proto_member object at 0x701ebc74f9b0>, 'Omega': <enum._proto_member object at 0x701ebc74cce0>, '__module__': 'test_native_enum', '_all_bits_': 0, ...}
        cls        = 'greek'
        exc        = TypeError('10 is not a string')
        first_enum = <enum 'StrEnum'>
        ignore     = ['_ignore_']
        invalid_names = set()
        key        = '_ignore_'
        kwds       = {}
        member_names = {'Alpha': None, 'Omega': None}
        member_type = <class 'str'>
        metacls    = <class 'enum.EnumType'>
        name       = 'Omega'
        save_new   = False
        tb         = <traceback object at 0x701ebc7a6cc0>
        use_args   = True
        value      = 20
/usr/lib/python3.12/enum.py:596: in __new__
    enum_class = super().__new__(metacls, cls, bases, classdict, **kwds)
        __class__  = <class 'enum.EnumType'>
        __new__    = <function StrEnum.__new__ at 0x701ec1711da0>
        _gnv       = <staticmethod(<function StrEnum._generate_next_value_ at 0x701ec1711e40>)>
        _order_    = None
        _simple    = False
        bases      = (<enum 'StrEnum'>,)
        boundary   = None
        classdict  = {'Alpha': <enum._proto_member object at 0x701ebc74f9b0>, 'Omega': <enum._proto_member object at 0x701ebc74cce0>, '__module__': 'test_native_enum', '_all_bits_': 0, ...}
        cls        = 'greek'
        exc        = TypeError('10 is not a string')
        first_enum = <enum 'StrEnum'>
        ignore     = ['_ignore_']
        invalid_names = set()
        key        = '_ignore_'
        kwds       = {}
        member_names = {'Alpha': None, 'Omega': None}
        member_type = <class 'str'>
        metacls    = <class 'enum.EnumType'>
        name       = 'Omega'
        save_new   = False
        tb         = <traceback object at 0x701ebc7a6cc0>
        use_args   = True
        value      = 20
/usr/lib/python3.12/enum.py:271: in __set_name__
    enum_member = enum_class._new_member_(enum_class, *args)
        args       = (10,)
        enum_class = <enum 'greek'>
        member_name = 'Alpha'
        self       = <enum._proto_member object at 0x701ebc74f9b0>
        value      = 10
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

cls = <enum 'greek'>, values = (10,)

    def __new__(cls, *values):
        "values must already be of type `str`"
        if len(values) > 3:
            raise TypeError('too many arguments for str(): %r' % (values, ))
        if len(values) == 1:
            # it must be a string
            if not isinstance(values[0], str):
>               raise TypeError('%r is not a string' % (values[0], ))
E               TypeError: 10 is not a string

cls        = <enum 'greek'>
values     = (10,)

/usr/lib/python3.12/enum.py:1322: TypeError
=========================== short test summary info ============================
FAILED test_native_enum.py::test_native_enum_StrEnum_greek - TypeError: 10 is...
======================= 1 failed, 39 deselected in 0.07s =======================

ERROR: completed_process.returncode=1
```

* Remove StrEnum code.

* Make enum_kind::Enum the default kind.

* Catch redundant .export_values() calls.

* [ci skip] Add back original documentation for `py::enum_` under new advanced/deprecated.rst

* [ci skip] Add documentation for `py::enum_kind` and `py::detail::type_caster_enum_type_enabled`

* Rename `Type` to `EnumType` for readability.

* Eliminate py::enum_kind, use "enum.Enum", "enum.IntEnum" directly. This is still WIP.

* EXPERIMENTAL StrEnum code. To be removed.

* Remove experimental StrEnum code:

My judgement: Supporting StrEnum is maybe nice, but not very valuable. I don't think it is worth the extra C++ code.

A level of indirection would need to be managed, e.g.

    RED   ↔ Python "r" ↔ C++ 0
    Green ↔ Python "g" ↔ C++ 1

These mappings would need to be stored and processed.

* Add test with enum.IntFlag (no production code changes required).

* First import_or_getattr() implementation (dedicated tests are still missing).

* Fix import_or_getattr() implementation, add tests, fix clang-tidy errors.

* [ci skip] Update classes.rst: replace `py::enum_kind` with `native_type_name`

* For "constructor similar to that of enum.Enum" point to https://docs.python.org/3/howto/enum.html#functional-api, as suggested by gh-timohl (https://github.com/pybind/pybind11/pull/5555#discussion_r2009277507).

* Advertise Enum, IntEnum, Flag, IntFlags are compatible stdlib enum types in the documentation (as suggested by gh-timohl, https://github.com/pybind/pybind11/pull/5555#pullrequestreview-2708832587); add test for enum.Flag to ensure that is actually true.

tests/test_native_enum.cpp

@@ -0,0 +1,234 @@
+#include <pybind11/native_enum.h>
+
+#include "pybind11_tests.h"
+
+#include <typeindex>
+
+namespace test_native_enum {
+
+// https://en.cppreference.com/w/cpp/language/enum
+
+// enum that takes 16 bits
+enum smallenum : std::int16_t { a, b, c };
+
+// color may be red (value 0), yellow (value 1), green (value 20), or blue (value 21)
+enum color { red, yellow, green = 20, blue };
+
+// altitude may be altitude::high or altitude::low
+enum class altitude : char {
+    high = 'h',
+    low = 'l', // trailing comma only allowed after CWG518
+};
+
+enum class flags_uchar : unsigned char { bit0 = 0x1u, bit1 = 0x2u, bit2 = 0x4u };
+enum class flags_uint : unsigned int { bit0 = 0x1u, bit1 = 0x2u, bit2 = 0x4u };
+
+enum class export_values { exv0, exv1 };
+
+enum class member_doc { mem0, mem1, mem2 };
+
+struct class_with_enum {
+    enum class in_class { one, two };
+};
+
+// https://github.com/protocolbuffers/protobuf/blob/d70b5c5156858132decfdbae0a1103e6a5cb1345/src/google/protobuf/generated_enum_util.h#L52-L53
+template <typename T>
+struct is_proto_enum : std::false_type {};
+
+enum some_proto_enum : int { Zero, One };
+
+template <>
+struct is_proto_enum<some_proto_enum> : std::true_type {};
+
+} // namespace test_native_enum
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+template <typename ProtoEnumType>
+struct type_caster_enum_type_enabled<
+    ProtoEnumType,
+    detail::enable_if_t<test_native_enum::is_proto_enum<ProtoEnumType>::value>> : std::false_type {
+};
+
+// https://github.com/pybind/pybind11_protobuf/blob/a50899c2eb604fc5f25deeb8901eff6231b8b3c0/pybind11_protobuf/enum_type_caster.h#L101-L105
+template <typename ProtoEnumType>
+struct type_caster<ProtoEnumType,
+                   detail::enable_if_t<test_native_enum::is_proto_enum<ProtoEnumType>::value>> {
+    static handle
+    cast(const ProtoEnumType & /*src*/, return_value_policy /*policy*/, handle /*parent*/) {
+        return py::none();
+    }
+
+    bool load(handle /*src*/, bool /*convert*/) {
+        value = static_cast<ProtoEnumType>(0);
+        return true;
+    }
+
+    PYBIND11_TYPE_CASTER(ProtoEnumType, const_name<ProtoEnumType>());
+};
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)
+
+TEST_SUBMODULE(native_enum, m) {
+    using namespace test_native_enum;
+
+    py::native_enum<smallenum>(m, "smallenum", "enum.IntEnum")
+        .value("a", smallenum::a)
+        .value("b", smallenum::b)
+        .value("c", smallenum::c)
+        .finalize();
+
+    py::native_enum<color>(m, "color", "enum.IntEnum")
+        .value("red", color::red)
+        .value("yellow", color::yellow)
+        .value("green", color::green)
+        .value("blue", color::blue)
+        .finalize();
+
+    py::native_enum<altitude>(m, "altitude")
+        .value("high", altitude::high)
+        .value("low", altitude::low)
+        .finalize();
+
+    py::native_enum<flags_uchar>(m, "flags_uchar", "enum.Flag")
+        .value("bit0", flags_uchar::bit0)
+        .value("bit1", flags_uchar::bit1)
+        .value("bit2", flags_uchar::bit2)
+        .finalize();
+
+    py::native_enum<flags_uint>(m, "flags_uint", "enum.IntFlag")
+        .value("bit0", flags_uint::bit0)
+        .value("bit1", flags_uint::bit1)
+        .value("bit2", flags_uint::bit2)
+        .finalize();
+
+    py::native_enum<export_values>(m, "export_values", "enum.IntEnum")
+        .value("exv0", export_values::exv0)
+        .value("exv1", export_values::exv1)
+        .export_values()
+        .finalize();
+
+    py::native_enum<member_doc>(m, "member_doc", "enum.IntEnum")
+        .value("mem0", member_doc::mem0, "docA")
+        .value("mem1", member_doc::mem1)
+        .value("mem2", member_doc::mem2, "docC")
+        .finalize();
+
+    py::class_<class_with_enum> py_class_with_enum(m, "class_with_enum");
+    py::native_enum<class_with_enum::in_class>(py_class_with_enum, "in_class", "enum.IntEnum")
+        .value("one", class_with_enum::in_class::one)
+        .value("two", class_with_enum::in_class::two)
+        .finalize();
+
+    m.def("isinstance_color", [](const py::object &obj) { return py::isinstance<color>(obj); });
+
+    m.def("pass_color", [](color e) { return static_cast<int>(e); });
+    m.def("return_color", [](int i) { return static_cast<color>(i); });
+
+    m.def("pass_some_proto_enum", [](some_proto_enum) { return py::none(); });
+    m.def("return_some_proto_enum", []() { return some_proto_enum::Zero; });
+
+#if defined(__MINGW32__)
+    m.attr("obj_cast_color_ptr") = "MinGW: dangling pointer to an unnamed temporary may be used "
+                                   "[-Werror=dangling-pointer=]";
+#elif defined(NDEBUG)
+    m.attr("obj_cast_color_ptr") = "NDEBUG disables cast safety check";
+#else
+    m.def("obj_cast_color_ptr", [](const py::object &obj) { obj.cast<color *>(); });
+#endif
+
+    m.def("py_cast_color_handle", [](py::handle obj) {
+        // Exercises `if (is_enum_cast && cast_is_temporary_value_reference<T>::value)`
+        // in `T cast(const handle &handle)`
+        auto e = py::cast<color>(obj);
+        return static_cast<int>(e);
+    });
+
+    m.def("exercise_import_or_getattr", [](py::module_ &m, const char *native_type_name) {
+        enum fake { x };
+        py::native_enum<fake>(m, "fake_import_or_getattr", native_type_name)
+            .value("x", fake::x)
+            .finalize();
+    });
+
+    m.def("native_enum_data_missing_finalize_error_message",
+          [](const std::string &enum_name_encoded) {
+              return py::detail::native_enum_missing_finalize_error_message(enum_name_encoded);
+          });
+
+    m.def("native_enum_ctor_malformed_utf8", [](const char *malformed_utf8) {
+        enum fake { x };
+        py::native_enum<fake>{py::none(), malformed_utf8, "enum.IntEnum"};
+    });
+
+    m.def("native_enum_double_finalize", [](py::module_ &m) {
+        enum fake { x };
+        py::native_enum<fake> ne(m, "fake_native_enum_double_finalize", "enum.IntEnum");
+        ne.finalize();
+        ne.finalize();
+    });
+
+    m.def("native_enum_value_after_finalize", [](py::module_ &m) {
+        enum fake { x };
+        py::native_enum<fake> ne(m, "fake_native_enum_value_after_finalize", "enum.IntEnum");
+        ne.finalize();
+        ne.value("x", fake::x);
+    });
+
+    m.def("native_enum_value_malformed_utf8", [](const char *malformed_utf8) {
+        enum fake { x };
+        py::native_enum<fake>(py::none(), "fake", "enum.IntEnum").value(malformed_utf8, fake::x);
+    });
+
+    m.def("double_registration_native_enum", [](py::module_ &m) {
+        enum fake { x };
+        py::native_enum<fake>(m, "fake_double_registration_native_enum", "enum.IntEnum")
+            .value("x", fake::x)
+            .finalize();
+        py::native_enum<fake>(m, "fake_double_registration_native_enum");
+    });
+
+    m.def("native_enum_name_clash", [](py::module_ &m) {
+        enum fake { x };
+        py::native_enum<fake>(m, "fake_native_enum_name_clash", "enum.IntEnum")
+            .value("x", fake::x)
+            .finalize();
+    });
+
+    m.def("native_enum_value_name_clash", [](py::module_ &m) {
+        enum fake { x };
+        py::native_enum<fake>(m, "fake_native_enum_value_name_clash", "enum.IntEnum")
+            .value("fake_native_enum_value_name_clash_x", fake::x)
+            .export_values()
+            .finalize();
+    });
+
+    m.def("double_registration_enum_before_native_enum", [](py::module_ &m) {
+        enum fake { x };
+        py::enum_<fake>(m, "fake_enum_first").value("x", fake::x);
+        py::native_enum<fake>(m, "fake_enum_first", "enum.IntEnum").value("x", fake::x).finalize();
+    });
+
+    m.def("double_registration_native_enum_before_enum", [](py::module_ &m) {
+        enum fake { x };
+        py::native_enum<fake>(m, "fake_native_enum_first", "enum.IntEnum")
+            .value("x", fake::x)
+            .finalize();
+        py::enum_<fake>(m, "name_must_be_different_to_reach_desired_code_path");
+    });
+
+#if defined(PYBIND11_NEGATE_THIS_CONDITION_FOR_LOCAL_TESTING) && !defined(NDEBUG)
+    m.def("native_enum_missing_finalize_failure", []() {
+        enum fake { x };
+        py::native_enum<fake>(
+            py::none(), "fake_native_enum_missing_finalize_failure", "enum.IntEnum")
+            .value("x", fake::x)
+            // .finalize() missing
+            ;
+    });
+#else
+    m.attr("native_enum_missing_finalize_failure") = "For local testing only: terminates process";
+#endif
+}