Revert internals destruction and add test for internals recreation (#5972)

* Bump internals version

* Prevent internals destruction before all pybind11 types are destroyed

* Use Py_XINCREF and Py_XDECREF

* Hold GIL before decref

* Use weakrefs

* Remove unused code

* Move code location

* Move code location

* Move code location

* Try add tests

* Fix PYTHONPATH

* Fix PYTHONPATH

* Skip tests for subprocess

* Revert to leak internals

* Revert to leak internals

* Revert "Revert to leak internals"

This reverts commit c5ec1cf886.
This reverts commit 72c2e0aa9b.

* Revert internals version bump

* Reapply to leak internals

This reverts commit 8f25a254e8.

* Add re-entrancy detection for internals creation

Prevent re-creation of internals after destruction during interpreter
shutdown. If pybind11 code runs after internals have been destroyed,
fail early with a clear error message instead of silently creating
new empty internals that would cause type lookup failures.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* Fix C++11/C++14 support

* Add lock under multiple interpreters

* Try fix tests

* Try fix tests

* Try fix tests

* Update comments and assertion messages

* Update comments and assertion messages

* Update comments

* Update lock scope

* Use original pointer type for Windows

* Change hard error to warning

* Update lock scope

* Update lock scope to resolve deadlock

* Remove scope release of GIL

* Update comments

* Lock pp on reset

* Mark content created after assignment

* Update comments

* Simplify implementation

* Update lock scope when delete unique_ptr

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

tests/test_multiple_interpreters.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import contextlib
 import os
 import pickle
-import subprocess
 import sys
 import textwrap
 
@@ -219,6 +218,7 @@ PREAMBLE_CODE = textwrap.dedent(
     def test():
         import sys
 
+        sys.path.insert(0, {os.path.dirname(env.__file__)!r})
         sys.path.insert(0, {os.path.dirname(pybind11_tests.__file__)!r})
 
         import collections
@@ -269,36 +269,13 @@ def test_import_module_with_singleton_per_interpreter():
         interp.exec(code)
 
 
-def check_script_success_in_subprocess(code: str, *, rerun: int = 8) -> None:
-    """Runs the given code in a subprocess."""
-    code = textwrap.dedent(code).strip()
-    try:
-        for _ in range(rerun):  # run flakily failing test multiple times
-            subprocess.check_output(
-                [sys.executable, "-c", code],
-                cwd=os.getcwd(),
-                stderr=subprocess.STDOUT,
-                text=True,
-            )
-    except subprocess.CalledProcessError as ex:
-        raise RuntimeError(
-            f"Subprocess failed with exit code {ex.returncode}.\n\n"
-            f"Code:\n"
-            f"```python\n"
-            f"{code}\n"
-            f"```\n\n"
-            f"Output:\n"
-            f"{ex.output}"
-        ) from None
-
-
 @pytest.mark.skipif(
     sys.platform.startswith("emscripten"), reason="Requires loadable modules"
 )
 @pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+")
 def test_import_in_subinterpreter_after_main():
     """Tests that importing a module in a subinterpreter after the main interpreter works correctly"""
-    check_script_success_in_subprocess(
+    env.check_script_success_in_subprocess(
         PREAMBLE_CODE
         + textwrap.dedent(
             """
@@ -319,7 +296,7 @@ def test_import_in_subinterpreter_after_main():
         )
     )
 
-    check_script_success_in_subprocess(
+    env.check_script_success_in_subprocess(
         PREAMBLE_CODE
         + textwrap.dedent(
             """
@@ -354,7 +331,7 @@ def test_import_in_subinterpreter_after_main():
 @pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+")
 def test_import_in_subinterpreter_before_main():
     """Tests that importing a module in a subinterpreter before the main interpreter works correctly"""
-    check_script_success_in_subprocess(
+    env.check_script_success_in_subprocess(
         PREAMBLE_CODE
         + textwrap.dedent(
             """
@@ -375,7 +352,7 @@ def test_import_in_subinterpreter_before_main():
         )
     )
 
-    check_script_success_in_subprocess(
+    env.check_script_success_in_subprocess(
         PREAMBLE_CODE
         + textwrap.dedent(
             """
@@ -401,7 +378,7 @@ def test_import_in_subinterpreter_before_main():
         )
     )
 
-    check_script_success_in_subprocess(
+    env.check_script_success_in_subprocess(
         PREAMBLE_CODE
         + textwrap.dedent(
             """
@@ -434,7 +411,7 @@ def test_import_in_subinterpreter_before_main():
 @pytest.mark.skipif(not CONCURRENT_INTERPRETERS_SUPPORT, reason="Requires 3.14.0b3+")
 def test_import_in_subinterpreter_concurrently():
     """Tests that importing a module in multiple subinterpreters concurrently works correctly"""
-    check_script_success_in_subprocess(
+    env.check_script_success_in_subprocess(
         PREAMBLE_CODE
         + textwrap.dedent(
             """