Support for sub-interpreters (#5564)

* Allow per-interpreter internals/local_internals

* Significant rewrite to avoid using thread_locals as much as possible.

Since we can avoid them by checking this atomic, the cmake config conditional shouldn't be necessary.

The slower path (with thread_locals and extra checks) only comes in when a second interpreter is actually instanciated.

* Add a test for per-interpreter GIL

Uses two extra threads to demonstrate that neither shares a GIL.

* Fix for nonconforming std::atomic constructors on some compilers

* style: pre-commit fixes

* Fix initializer to make MSVC happy.

* Switch to gil_scoped_acquire_simple, get rid of old copy of it from internals.h

* Use the PyThreadState's interp member rather than the thread state itself.

* Be more explicit about the type of the internalspp

* Suggested renamings and rewordings

* Rename find_internals_pp and change it to take in the state dict reference

* Use the old raise_from instead of pybind11_fail

* Move most of the internals initialization into its constructor.

* Move round_up_to_next_pow2 function upwards

* Remove redundant forward decl

* Add a python-driven subinterpreter test

* Disable the python subinterpreter test on emscripten

Can't load the native-built cpp modules.

* Switch the internals pointer pointer to a unique_ptr pointer

* Spelling

* Fix clang-tidy warning, compare pointer to nullptr

* Rename get_interpreter_counter to get_num_interpreters_seen

* Try simplifying the test's cmake set_target_properties

* Replace mod_* tags with a single tag w/enum

Update tests accordingly

* Add a test for shared-GIL (legacy) subinterpreters

* Update test to work around differences in the various versions of interpreters modules

* Fix unused parameter

* Rename tests and associated test modules.

* Switch get_internals_pp to a template function

* Rename curtstate to cur_tstate

* refactor: use simpler names

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* style: pre-commit fixes

* fix: return class, not enum

Co-authored-by: Ralf W. Grosse-Kunstleve <rwgkio@gmail.com>
Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* Have to join these threads to make sure they are totally done before the test returns.

* Wrap module_def initialization in a static so it only happens once.

If it happens concurrently in multiple threads, badness ensues....

* style: pre-commit fixes

---------

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Henry Schreiner <henryschreineriii@gmail.com>
Co-authored-by: Ralf W. Grosse-Kunstleve <rwgkio@gmail.com>

tests/test_multiple_interpreters.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import os
+import pickle
+import sys
+
+import pytest
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("emscripten"), reason="Requires loadable modules"
+)
+def test_independent_subinterpreters():
+    """Makes sure the internals object differs across independent subinterpreters"""
+
+    sys.path.append(".")
+
+    if sys.version_info >= (3, 14):
+        import interpreters
+    elif sys.version_info >= (3, 13):
+        import _interpreters as interpreters
+    elif sys.version_info >= (3, 12):
+        import _xxsubinterpreters as interpreters
+    else:
+        pytest.skip("Test requires a the interpreters stdlib module")
+
+    import mod_per_interpreter_gil as m
+
+    code = """
+import mod_per_interpreter_gil as m
+import pickle
+with open(pipeo, 'wb') as f:
+    pickle.dump(m.internals_at(), f)
+"""
+
+    interp1 = interpreters.create()
+    interp2 = interpreters.create()
+    try:
+        try:
+            res0 = interpreters.run_string(interp1, "import mod_shared_interpreter_gil")
+            if res0 is not None:
+                res0 = res0.msg
+        except Exception as e:
+            res0 = str(e)
+
+        pipei, pipeo = os.pipe()
+        interpreters.run_string(interp1, code, shared={"pipeo": pipeo})
+        with open(pipei, "rb") as f:
+            res1 = pickle.load(f)
+
+        pipei, pipeo = os.pipe()
+        interpreters.run_string(interp2, code, shared={"pipeo": pipeo})
+        with open(pipei, "rb") as f:
+            res2 = pickle.load(f)
+
+        # do this while the two interpreters are active
+        import mod_per_interpreter_gil as m2
+
+        assert m.internals_at() == m2.internals_at(), (
+            "internals should be the same within the main interpreter"
+        )
+    finally:
+        interpreters.destroy(interp1)
+        interpreters.destroy(interp2)
+
+    assert "does not support loading in subinterpreters" in res0, (
+        "cannot use shared_gil in a default subinterpreter"
+    )
+    assert res1 != m.internals_at(), "internals should differ from main interpreter"
+    assert res2 != m.internals_at(), "internals should differ from main interpreter"
+    assert res1 != res2, "internals should differ between interpreters"
+
+    # do this after the two interpreters are destroyed and only one remains
+    import mod_per_interpreter_gil as m3
+
+    assert m.internals_at() == m3.internals_at(), (
+        "internals should be the same within the main interpreter"
+    )
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("emscripten"), reason="Requires loadable modules"
+)
+def test_dependent_subinterpreters():
+    """Makes sure the internals object differs across subinterpreters"""
+
+    sys.path.append(".")
+
+    if sys.version_info >= (3, 14):
+        import interpreters
+    elif sys.version_info >= (3, 13):
+        import _interpreters as interpreters
+    elif sys.version_info >= (3, 12):
+        import _xxsubinterpreters as interpreters
+    else:
+        pytest.skip("Test requires a the interpreters stdlib module")
+
+    import mod_shared_interpreter_gil as m
+
+    code = """
+import mod_shared_interpreter_gil as m
+import pickle
+with open(pipeo, 'wb') as f:
+    pickle.dump(m.internals_at(), f)
+"""
+
+    try:
+        interp1 = interpreters.create("legacy")
+    except TypeError:
+        pytest.skip("interpreters module needs to support legacy config")
+
+    try:
+        pipei, pipeo = os.pipe()
+        interpreters.run_string(interp1, code, shared={"pipeo": pipeo})
+        with open(pipei, "rb") as f:
+            res1 = pickle.load(f)
+
+        # do this while the other interpreter is active
+        import mod_shared_interpreter_gil as m2
+
+        assert m.internals_at() == m2.internals_at(), (
+            "internals should be the same within the main interpreter"
+        )
+    finally:
+        interpreters.destroy(interp1)
+
+    assert res1 != m.internals_at(), "internals should differ from main interpreter"
+
+    # do this after the other interpreters are destroyed and only one remains
+    import mod_shared_interpreter_gil as m3
+
+    assert m.internals_at() == m3.internals_at(), (
+        "internals should be the same within the main interpreter"
+    )