Pin PyThreadStates so native extensions can cache them per thread (#137)

* Pin PyThreadStates so native extensions can cache them per thread

Py.GIL() acquires the GIL via PyGILState_Ensure/Release; the outermost
scope on a .NET thread creates a fresh PyThreadState and deletes it on
dispose. pybind11-based extensions (matplotlib >= 3.10 _path/ft2font,
scipy, PyTorch) cache the first PyThreadState* they see per OS thread
in pybind11 internals TLS and never invalidate it, so once a scope
deletes the thread state the next native GIL acquire on that thread
restores a dangling pointer: 0xC0000005 access violation or heap
corruption. Root cause of QuantConnect/Lean#9203 (ReportChartTests
crashing the test host on the first matplotlib render).

Pin each thread state with one extra never-released PyGILState_Ensure
per thread per runtime run, keeping the gilstate counter >= 1 so the
thread state lives until engine shutdown - the same lifetime CPython
gives thread states it binds itself. The extra Ensure runs with the
GIL already held (a bare counter increment): GIL acquire/release
timing is unchanged and empty-scope microbenchmarks are within noise
(0.212 -> 0.209 us/scope).

Regression test: threading.local values live in the thread state dict,
so a value stored in one Py.GIL scope survives a second scope on the
same thread only if the thread state was not deleted.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Update version to 2.0.62

Bump package <Version>, AssemblyVersion/AssemblyFileVersion and the
perf-test baseline reference to 2.0.62.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: Martin-Molinero

src/embed_tests/TestGILState.cs

@@ -1,5 +1,7 @@
 namespace Python.EmbeddingTest
 {
+    using System;
+    using System.Threading;
     using NUnit.Framework;
     using Python.Runtime;
 
@@ -18,6 +20,57 @@ public void CanDisposeMultipleTimes()
             }
         }
 
+        /// <summary>
+        /// The thread's PyThreadState must survive between GIL scopes. Native extensions
+        /// built with pybind11 (e.g. matplotlib >= 3.10 _path/ft2font) cache the pointer
+        /// per OS thread and crash with an access violation if a later scope runs after
+        /// the thread state was deleted. threading.local values live in the thread state
+        /// dictionary, so they survive a second scope only if the thread state did.
+        /// </summary>
+        [Test]
+        public void ThreadStateIsPreservedBetweenGILScopes()
+        {
+            var result = 0;
+            Exception error = null;
+            var thread = new Thread(() =>
+            {
+                try
+                {
+                    PyModule scope;
+                    using (Py.GIL())
+                    {
+                        scope = Py.CreateScope();
+                        scope.Exec("import threading\nlocal = threading.local()\nlocal.value = 42");
+                    }
+                    using (Py.GIL())
+                    using (scope)
+                    {
+                        using var value = scope.Eval("getattr(local, 'value', -1)");
+                        result = value.As<int>();
+                    }
+                }
+                catch (Exception e)
+                {
+                    error = e;
+                }
+            });
+            // the fixture initializes the engine with the GIL held on this thread:
+            // release it so the worker thread can acquire it
+            var ts = PythonEngine.BeginAllowThreads();
+            try
+            {
+                thread.Start();
+                thread.Join();
+            }
+            finally
+            {
+                PythonEngine.EndAllowThreads(ts);
+            }
+
+            Assert.IsNull(error);
+            Assert.AreEqual(42, result);
+        }
+
         [OneTimeSetUp]
         public void SetUp()
         {

src/perf_tests/Python.PerformanceTests.csproj

@@ -13,7 +13,7 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.*" />
-    <PackageReference Include="quantconnect.pythonnet" Version="2.0.61" GeneratePathProperty="true">
+    <PackageReference Include="quantconnect.pythonnet" Version="2.0.62" GeneratePathProperty="true">
       <IncludeAssets>compile</IncludeAssets>
     </PackageReference>
   </ItemGroup>
@@ -25,7 +25,7 @@
   </Target>
 
   <Target Name="CopyBaseline" AfterTargets="Build">
-    <Copy SourceFiles="$(NuGetPackageRoot)quantconnect.pythonnet\2.0.61\lib\net10.0\Python.Runtime.dll" DestinationFolder="$(OutDir)baseline" />
+    <Copy SourceFiles="$(NuGetPackageRoot)quantconnect.pythonnet\2.0.62\lib\net10.0\Python.Runtime.dll" DestinationFolder="$(OutDir)baseline" />
   </Target>
 
   <Target Name="CopyNewBuild" AfterTargets="Build">

src/runtime/Properties/AssemblyInfo.cs

@@ -4,5 +4,5 @@
 [assembly: InternalsVisibleTo("Python.EmbeddingTest, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")]
 [assembly: InternalsVisibleTo("Python.Test, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")]
 
-[assembly: AssemblyVersion("2.0.61")]
-[assembly: AssemblyFileVersion("2.0.61")]
+[assembly: AssemblyVersion("2.0.62")]
+[assembly: AssemblyFileVersion("2.0.62")]

src/runtime/Py.cs

@@ -28,12 +28,28 @@ public void Dispose()
 
     public class GILState : IDisposable
     {
+        // Tracks the runtime run for which this thread's PyThreadState has been pinned.
+        // Native extensions built with pybind11 (e.g. matplotlib >= 3.10 _path/ft2font)
+        // cache the PyThreadState pointer per OS thread and reuse it later; if the
+        // outermost PyGILState_Release deletes the thread state, that cached pointer
+        // dangles and the next native GIL acquire crashes with an access violation.
+        // Pinning: one extra, never-released PyGILState_Ensure per thread keeps the
+        // gilstate counter >= 1 so the thread state lives until engine shutdown.
+        [ThreadStatic] private static int _pinnedOnRun;
+
         private readonly PyGILState state;
         private bool isDisposed;
 
         internal GILState()
         {
             state = PythonEngine.AcquireLock();
+
+            var run = Runtime.GetRun();
+            if (_pinnedOnRun != run)
+            {
+                _pinnedOnRun = run;
+                Runtime.PyGILState_Ensure();
+            }
         }
 
         public virtual void Dispose()

src/runtime/Python.Runtime.csproj

@@ -5,7 +5,7 @@
     <RootNamespace>Python.Runtime</RootNamespace>
     <AssemblyName>Python.Runtime</AssemblyName>
     <PackageId>QuantConnect.pythonnet</PackageId>
-    <Version>2.0.61</Version>
+    <Version>2.0.62</Version>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
     <RepositoryUrl>https://github.com/pythonnet/pythonnet</RepositoryUrl>