fix(types): type hints from future python versions (#5693)

* fix future type hints

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* style: pre-commit fixes

* remove unused var

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* remove union_helper

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* fix speelling error

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* base case for union_concat

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* style: pre-commit fixes

* add case for one descr

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* weakref and final test

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* Add acrpss_version_type_hint_checker

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* cleanup

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* style: pre-commit fixes

* remove test.pyi

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* use new unions and add fixture

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* timohl suggested cleanup

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* style: pre-commit fixes

* add missing auto

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>

* style: pre-commit fixes

* move operator| def

---------

Signed-off-by: Michael Carlstrom <rmc@carlstrom.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

tests/conftest.py

@@ -16,6 +16,7 @@ import sys
 import sysconfig
 import textwrap
 import traceback
+from typing import Callable
 
 import pytest
 
@@ -242,3 +243,25 @@ def pytest_report_header():
         lines.append("free-threaded Python build")
 
     return lines
+
+
+@pytest.fixture
+def backport_typehints() -> Callable[[SanitizedString], SanitizedString]:
+    d = {}
+    if sys.version_info < (3, 13):
+        d["typing_extensions.TypeIs"] = "typing.TypeIs"
+        d["typing_extensions.CapsuleType"] = "types.CapsuleType"
+    if sys.version_info < (3, 12):
+        d["typing_extensions.Buffer"] = "collections.abc.Buffer"
+    if sys.version_info < (3, 11):
+        d["typing_extensions.Never"] = "typing.Never"
+    if sys.version_info < (3, 10):
+        d["typing_extensions.TypeGuard"] = "typing.TypeGuard"
+
+    def backport(sanatized_string: SanitizedString) -> SanitizedString:
+        for old, new in d.items():
+            sanatized_string.string = sanatized_string.string.replace(old, new)
+
+        return sanatized_string
+
+    return backport

tests/test_buffers.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import ctypes
 import io
 import struct
-import sys
 
 import pytest
 
@@ -228,12 +227,11 @@ def test_ctypes_from_buffer():
         assert not cinfo.readonly
 
 
-def test_buffer_docstring():
-    if sys.version_info >= (3, 12):
-        docstring = "get_buffer_info(arg0: collections.abc.Buffer) -> pybind11_tests.buffers.buffer_info"
-    else:
-        docstring = "get_buffer_info(arg0: typing_extensions.Buffer) -> pybind11_tests.buffers.buffer_info"
-    assert m.get_buffer_info.__doc__.strip() == docstring
+def test_buffer_docstring(doc, backport_typehints):
+    assert (
+        backport_typehints(doc(m.get_buffer_info))
+        == "get_buffer_info(arg0: collections.abc.Buffer) -> m.buffers.buffer_info"
+    )
 
 
 def test_buffer_exception():

tests/test_opaque_types.py

@@ -27,7 +27,7 @@ def test_string_list():
     assert m.print_opaque_list(cvp.stringList) == "Opaque list: [Element 1, Element 3]"
 
 
-def test_pointers(msg):
+def test_pointers(msg, backport_typehints):
     living_before = ConstructorStats.get(UserType).alive()
     assert m.get_void_ptr_value(m.return_void_ptr()) == 0x1234
     assert m.get_void_ptr_value(UserType())  # Should also work for other C++ types
@@ -37,14 +37,15 @@ def test_pointers(msg):
 
     with pytest.raises(TypeError) as excinfo:
         m.get_void_ptr_value([1, 2, 3])  # This should not work
-    assert (
-        msg(excinfo.value)
-        == """
-        get_void_ptr_value(): incompatible function arguments. The following argument types are supported:
-            1. (arg0: types.CapsuleType) -> int
 
-        Invoked with: [1, 2, 3]
-    """
+    assert (
+        backport_typehints(msg(excinfo.value))
+        == """
+            get_void_ptr_value(): incompatible function arguments. The following argument types are supported:
+                1. (arg0: types.CapsuleType) -> int
+
+            Invoked with: [1, 2, 3]
+        """
     )
 
     assert m.return_null_str() is None

tests/test_pytypes.cpp

@@ -155,7 +155,7 @@ namespace detail {
 
 template <>
 struct type_caster<RealNumber> {
-    PYBIND11_TYPE_CASTER(RealNumber, io_name("typing.Union[float, int]", "float"));
+    PYBIND11_TYPE_CASTER(RealNumber, io_name("float | int", "float"));
 
     static handle cast(const RealNumber &number, return_value_policy, handle) {
         return py::float_(number.value).release();

tests/test_pytypes.py

@@ -130,10 +130,7 @@ def test_set(capture, doc):
     assert m.anyset_contains({"foo"}, "foo")
 
     assert doc(m.get_set) == "get_set() -> set"
-    assert (
-        doc(m.print_anyset)
-        == "print_anyset(arg0: typing.Union[set, frozenset]) -> None"
-    )
+    assert doc(m.print_anyset) == "print_anyset(arg0: set | frozenset) -> None"
 
 
 def test_frozenset(capture, doc):
@@ -992,41 +989,37 @@ def test_type_annotation(doc):
 def test_union_annotations(doc):
     assert (
         doc(m.annotate_union)
-        == "annotate_union(arg0: list[typing.Union[str, typing.SupportsInt, object]], arg1: str, arg2: typing.SupportsInt, arg3: object) -> list[typing.Union[str, int, object]]"
+        == "annotate_union(arg0: list[str | typing.SupportsInt | object], arg1: str, arg2: typing.SupportsInt, arg3: object) -> list[str | int | object]"
     )
 
 
 def test_union_typing_only(doc):
-    assert (
-        doc(m.union_typing_only)
-        == "union_typing_only(arg0: list[typing.Union[str]]) -> list[typing.Union[int]]"
-    )
+    assert doc(m.union_typing_only) == "union_typing_only(arg0: list[str]) -> list[int]"
 
 
 def test_union_object_annotations(doc):
     assert (
         doc(m.annotate_union_to_object)
-        == "annotate_union_to_object(arg0: typing.Union[typing.SupportsInt, str]) -> object"
+        == "annotate_union_to_object(arg0: typing.SupportsInt | str) -> object"
     )
 
 
 def test_optional_annotations(doc):
     assert (
-        doc(m.annotate_optional)
-        == "annotate_optional(arg0: list) -> list[typing.Optional[str]]"
+        doc(m.annotate_optional) == "annotate_optional(arg0: list) -> list[str | None]"
     )
 
 
-def test_type_guard_annotations(doc):
+def test_type_guard_annotations(doc, backport_typehints):
     assert (
-        doc(m.annotate_type_guard)
+        backport_typehints(doc(m.annotate_type_guard))
         == "annotate_type_guard(arg0: object) -> typing.TypeGuard[str]"
     )
 
 
-def test_type_is_annotations(doc):
+def test_type_is_annotations(doc, backport_typehints):
     assert (
-        doc(m.annotate_type_is)
+        backport_typehints(doc(m.annotate_type_is))
         == "annotate_type_is(arg0: object) -> typing.TypeIs[str]"
     )
 
@@ -1035,14 +1028,16 @@ def test_no_return_annotation(doc):
     assert doc(m.annotate_no_return) == "annotate_no_return() -> typing.NoReturn"
 
 
-def test_never_annotation(doc):
-    assert doc(m.annotate_never) == "annotate_never() -> typing.Never"
+def test_never_annotation(doc, backport_typehints):
+    assert (
+        backport_typehints(doc(m.annotate_never)) == "annotate_never() -> typing.Never"
+    )
 
 
 def test_optional_object_annotations(doc):
     assert (
         doc(m.annotate_optional_to_object)
-        == "annotate_optional_to_object(arg0: typing.Optional[typing.SupportsInt]) -> object"
+        == "annotate_optional_to_object(arg0: typing.SupportsInt | None) -> object"
     )
 
 
@@ -1078,11 +1073,11 @@ def test_literal(doc):
     )
     assert (
         doc(m.identity_literal_arrow_with_io_name)
-        == 'identity_literal_arrow_with_io_name(arg0: typing.Literal["->"], arg1: typing.Union[float, int]) -> typing.Literal["->"]'
+        == 'identity_literal_arrow_with_io_name(arg0: typing.Literal["->"], arg1: float | int) -> typing.Literal["->"]'
     )
     assert (
         doc(m.identity_literal_arrow_with_callable)
-        == 'identity_literal_arrow_with_callable(arg0: collections.abc.Callable[[typing.Literal["->"], typing.Union[float, int]], float]) -> collections.abc.Callable[[typing.Literal["->"], typing.Union[float, int]], float]'
+        == 'identity_literal_arrow_with_callable(arg0: collections.abc.Callable[[typing.Literal["->"], float | int], float]) -> collections.abc.Callable[[typing.Literal["->"], float | int], float]'
     )
     assert (
         doc(m.identity_literal_all_special_chars)
@@ -1168,9 +1163,10 @@ def test_module_attribute_types() -> None:
     assert module_annotations["list_int"] == "list[typing.SupportsInt]"
     assert module_annotations["set_str"] == "set[str]"
     assert module_annotations["foo"] == "pybind11_tests.pytypes.foo"
+
     assert (
         module_annotations["foo_union"]
-        == "typing.Union[pybind11_tests.pytypes.foo, pybind11_tests.pytypes.foo2, pybind11_tests.pytypes.foo3]"
+        == "pybind11_tests.pytypes.foo | pybind11_tests.pytypes.foo2 | pybind11_tests.pytypes.foo3"
     )
 
 
@@ -1249,14 +1245,11 @@ def test_final_annotation() -> None:
     assert module_annotations["CONST_INT"] == "typing.Final[int]"
 
 
-def test_arg_return_type_hints(doc):
-    assert (
-        doc(m.half_of_number)
-        == "half_of_number(arg0: typing.Union[float, int]) -> float"
-    )
+def test_arg_return_type_hints(doc, backport_typehints):
+    assert doc(m.half_of_number) == "half_of_number(arg0: float | int) -> float"
     assert (
         doc(m.half_of_number_convert)
-        == "half_of_number_convert(x: typing.Union[float, int]) -> float"
+        == "half_of_number_convert(x: float | int) -> float"
     )
     assert (
         doc(m.half_of_number_noconvert) == "half_of_number_noconvert(x: float) -> float"
@@ -1266,55 +1259,53 @@ def test_arg_return_type_hints(doc):
     assert m.half_of_number(0) == 0
     assert isinstance(m.half_of_number(0), float)
     assert not isinstance(m.half_of_number(0), int)
+
     # std::vector<T>
     assert (
         doc(m.half_of_number_vector)
-        == "half_of_number_vector(arg0: collections.abc.Sequence[typing.Union[float, int]]) -> list[float]"
+        == "half_of_number_vector(arg0: collections.abc.Sequence[float | int]) -> list[float]"
     )
     # Tuple<T, T>
     assert (
         doc(m.half_of_number_tuple)
-        == "half_of_number_tuple(arg0: tuple[typing.Union[float, int], typing.Union[float, int]]) -> tuple[float, float]"
+        == "half_of_number_tuple(arg0: tuple[float | int, float | int]) -> tuple[float, float]"
     )
     # Tuple<T, ...>
     assert (
         doc(m.half_of_number_tuple_ellipsis)
-        == "half_of_number_tuple_ellipsis(arg0: tuple[typing.Union[float, int], ...]) -> tuple[float, ...]"
+        == "half_of_number_tuple_ellipsis(arg0: tuple[float | int, ...]) -> tuple[float, ...]"
     )
     # Dict<K, V>
     assert (
         doc(m.half_of_number_dict)
-        == "half_of_number_dict(arg0: dict[str, typing.Union[float, int]]) -> dict[str, float]"
+        == "half_of_number_dict(arg0: dict[str, float | int]) -> dict[str, float]"
     )
     # List<T>
     assert (
         doc(m.half_of_number_list)
-        == "half_of_number_list(arg0: list[typing.Union[float, int]]) -> list[float]"
+        == "half_of_number_list(arg0: list[float | int]) -> list[float]"
     )
     # List<List<T>>
     assert (
         doc(m.half_of_number_nested_list)
-        == "half_of_number_nested_list(arg0: list[list[typing.Union[float, int]]]) -> list[list[float]]"
+        == "half_of_number_nested_list(arg0: list[list[float | int]]) -> list[list[float]]"
     )
     # Set<T>
-    assert (
-        doc(m.identity_set)
-        == "identity_set(arg0: set[typing.Union[float, int]]) -> set[float]"
-    )
+    assert doc(m.identity_set) == "identity_set(arg0: set[float | int]) -> set[float]"
     # Iterable<T>
     assert (
         doc(m.identity_iterable)
-        == "identity_iterable(arg0: collections.abc.Iterable[typing.Union[float, int]]) -> collections.abc.Iterable[float]"
+        == "identity_iterable(arg0: collections.abc.Iterable[float | int]) -> collections.abc.Iterable[float]"
     )
     # Iterator<T>
     assert (
         doc(m.identity_iterator)
-        == "identity_iterator(arg0: collections.abc.Iterator[typing.Union[float, int]]) -> collections.abc.Iterator[float]"
+        == "identity_iterator(arg0: collections.abc.Iterator[float | int]) -> collections.abc.Iterator[float]"
     )
     # Callable<R(A)> identity
     assert (
         doc(m.identity_callable)
-        == "identity_callable(arg0: collections.abc.Callable[[typing.Union[float, int]], float]) -> collections.abc.Callable[[typing.Union[float, int]], float]"
+        == "identity_callable(arg0: collections.abc.Callable[[float | int], float]) -> collections.abc.Callable[[float | int], float]"
     )
     # Callable<R(...)> identity
     assert (
@@ -1324,32 +1315,35 @@ def test_arg_return_type_hints(doc):
     # Nested Callable<R(A)> identity
     assert (
         doc(m.identity_nested_callable)
-        == "identity_nested_callable(arg0: collections.abc.Callable[[collections.abc.Callable[[typing.Union[float, int]], float]], collections.abc.Callable[[typing.Union[float, int]], float]]) -> collections.abc.Callable[[collections.abc.Callable[[typing.Union[float, int]], float]], collections.abc.Callable[[typing.Union[float, int]], float]]"
+        == "identity_nested_callable(arg0: collections.abc.Callable[[collections.abc.Callable[[float | int], float]], collections.abc.Callable[[float | int], float]]) -> collections.abc.Callable[[collections.abc.Callable[[float | int], float]], collections.abc.Callable[[float | int], float]]"
     )
     # Callable<R(A)>
     assert (
         doc(m.apply_callable)
-        == "apply_callable(arg0: typing.Union[float, int], arg1: collections.abc.Callable[[typing.Union[float, int]], float]) -> float"
+        == "apply_callable(arg0: float | int, arg1: collections.abc.Callable[[float | int], float]) -> float"
     )
     # Callable<R(...)>
     assert (
         doc(m.apply_callable_ellipsis)
-        == "apply_callable_ellipsis(arg0: typing.Union[float, int], arg1: collections.abc.Callable[..., float]) -> float"
+        == "apply_callable_ellipsis(arg0: float | int, arg1: collections.abc.Callable[..., float]) -> float"
     )
     # Union<T1, T2>
     assert (
         doc(m.identity_union)
-        == "identity_union(arg0: typing.Union[typing.Union[float, int], str]) -> typing.Union[float, str]"
+        == "identity_union(arg0: float | int | str) -> float | str"
     )
     # Optional<T>
     assert (
         doc(m.identity_optional)
-        == "identity_optional(arg0: typing.Optional[typing.Union[float, int]]) -> typing.Optional[float]"
+        == "identity_optional(arg0: float | int | None) -> float | None"
+    )
+    # TypeIs<T>
+    assert (
+        backport_typehints(doc(m.check_type_is))
+        == "check_type_is(arg0: object) -> typing.TypeIs[float]"
     )
     # TypeGuard<T>
     assert (
-        doc(m.check_type_guard)
+        backport_typehints(doc(m.check_type_guard))
         == "check_type_guard(arg0: list[object]) -> typing.TypeGuard[list[float]]"
     )
-    # TypeIs<T>
-    assert doc(m.check_type_is) == "check_type_is(arg0: object) -> typing.TypeIs[float]"

tests/test_stl.py

@@ -227,11 +227,16 @@ def test_boost_optional():
     assert int(props.access_by_copy) == 42
 
 
-def test_reference_sensitive_optional():
+def test_reference_sensitive_optional(doc):
     assert m.double_or_zero_refsensitive(None) == 0
     assert m.double_or_zero_refsensitive(42) == 84
     pytest.raises(TypeError, m.double_or_zero_refsensitive, "foo")
 
+    assert (
+        doc(m.double_or_zero_refsensitive)
+        == "double_or_zero_refsensitive(arg0: typing.SupportsInt | None) -> int"
+    )
+
     assert m.half_or_none_refsensitive(0) is None
     assert m.half_or_none_refsensitive(42) == 21
     pytest.raises(TypeError, m.half_or_none_refsensitive, "foo")
@@ -257,7 +262,7 @@ def test_reference_sensitive_optional():
 
 
 @pytest.mark.skipif(not hasattr(m, "has_filesystem"), reason="no <filesystem>")
-def test_fs_path(doc):
+def test_fs_path():
     from pathlib import Path
 
     class PseudoStrPath:
@@ -274,37 +279,17 @@ def test_fs_path(doc):
     assert m.parent_path(b"foo/bar") == Path("foo")
     assert m.parent_path(PseudoStrPath()) == Path("foo")
     assert m.parent_path(PseudoBytesPath()) == Path("foo")
-    assert (
-        doc(m.parent_path)
-        == "parent_path(arg0: typing.Union[os.PathLike, str, bytes]) -> pathlib.Path"
-    )
     # std::vector
     assert m.parent_paths(["foo/bar", "foo/baz"]) == [Path("foo"), Path("foo")]
-    assert (
-        doc(m.parent_paths)
-        == "parent_paths(arg0: collections.abc.Sequence[typing.Union[os.PathLike, str, bytes]]) -> list[pathlib.Path]"
-    )
     # py::typing::List
     assert m.parent_paths_list(["foo/bar", "foo/baz"]) == [Path("foo"), Path("foo")]
-    assert (
-        doc(m.parent_paths_list)
-        == "parent_paths_list(arg0: list[typing.Union[os.PathLike, str, bytes]]) -> list[pathlib.Path]"
-    )
     # Nested py::typing::List
     assert m.parent_paths_nested_list([["foo/bar"], ["foo/baz", "foo/buzz"]]) == [
         [Path("foo")],
         [Path("foo"), Path("foo")],
     ]
-    assert (
-        doc(m.parent_paths_nested_list)
-        == "parent_paths_nested_list(arg0: list[list[typing.Union[os.PathLike, str, bytes]]]) -> list[list[pathlib.Path]]"
-    )
     # py::typing::Tuple
     assert m.parent_paths_tuple(("foo/bar", "foo/baz")) == (Path("foo"), Path("foo"))
-    assert (
-        doc(m.parent_paths_tuple)
-        == "parent_paths_tuple(arg0: tuple[typing.Union[os.PathLike, str, bytes], typing.Union[os.PathLike, str, bytes]]) -> tuple[pathlib.Path, pathlib.Path]"
-    )
     # py::typing::Dict
     assert m.parent_paths_dict(
         {
@@ -317,9 +302,39 @@ def test_fs_path(doc):
         "key2": Path("foo"),
         "key3": Path("foo"),
     }
+
+
+@pytest.mark.skipif(not hasattr(m, "has_filesystem"), reason="no <filesystem>")
+def test_path_typing(doc):
+    # Single argument
+    assert (
+        doc(m.parent_path)
+        == "parent_path(arg0: os.PathLike | str | bytes) -> pathlib.Path"
+    )
+    # std::vector
+    assert (
+        doc(m.parent_paths)
+        == "parent_paths(arg0: collections.abc.Sequence[os.PathLike | str | bytes]) -> list[pathlib.Path]"
+    )
+    # py::typing::List
+    assert (
+        doc(m.parent_paths_list)
+        == "parent_paths_list(arg0: list[os.PathLike | str | bytes]) -> list[pathlib.Path]"
+    )
+    # Nested py::typing::List
+    assert (
+        doc(m.parent_paths_nested_list)
+        == "parent_paths_nested_list(arg0: list[list[os.PathLike | str | bytes]]) -> list[list[pathlib.Path]]"
+    )
+    # py::typing::Tuple
+    assert (
+        doc(m.parent_paths_tuple)
+        == "parent_paths_tuple(arg0: tuple[os.PathLike | str | bytes, os.PathLike | str | bytes]) -> tuple[pathlib.Path, pathlib.Path]"
+    )
+    # py::typing::Dict
     assert (
         doc(m.parent_paths_dict)
-        == "parent_paths_dict(arg0: dict[str, typing.Union[os.PathLike, str, bytes]]) -> dict[str, pathlib.Path]"
+        == "parent_paths_dict(arg0: dict[str, os.PathLike | str | bytes]) -> dict[str, pathlib.Path]"
     )
 
 
@@ -337,7 +352,7 @@ def test_variant(doc):
 
     assert (
         doc(m.load_variant)
-        == "load_variant(arg0: typing.Union[typing.SupportsInt, str, typing.SupportsFloat, None]) -> str"
+        == "load_variant(arg0: typing.SupportsInt | str | typing.SupportsFloat | None) -> str"
     )
 
 
@@ -353,7 +368,7 @@ def test_variant_monostate(doc):
 
     assert (
         doc(m.load_monostate_variant)
-        == "load_monostate_variant(arg0: typing.Union[None, typing.SupportsInt, str]) -> str"
+        == "load_monostate_variant(arg0: None | typing.SupportsInt | str) -> str"
     )