From 3b2a9d61b087ae7682fe986f9beaea5ecbc96597 Mon Sep 17 00:00:00 2001 From: viogroza Date: Wed, 19 Aug 2026 16:45:54 +0300 Subject: [PATCH] Python: Fix venv isolation from user-site and base-install site-packages [STUD-81085] Python Scope's embedded interpreter never goes through CPython's own venv activation path (site.py's venv() function, which only triggers for a normally-launched /Scripts/python.exe), so pointing Path at a venv left both the PEP-370 user-site directory and the base install's own site-packages on sys.path unfiltered. A native package (e.g. pywin32) installed with a different build in either location could resolve its Python module from one and its native DLL dependency from the other, mismatched, copy, surfacing as "DLL load failed while importing win32api: The specified procedure could not be found." Fixes user-site and base-install leakage via PythonEngine.SetNoSiteFlag() (gated on whether the venv requests --system-site-packages), with Controller.ClearUserSiteEnvironmentOverride guarding against an ambient PYTHONNOUSERSITE leaking in from the host's own environment. Also fixes PythonHome being pointed at the venv root instead of its declared base install (breaks stdlib resolution entirely), tightens venv detection against false positives, and adds a version cross-check between a venv and the Python library actually being loaded. Co-Authored-By: Claude Sonnet 5 --- .../Client/Controller.cs | 15 + .../UiPath.Python.Tests/VenvDetectionTests.cs | 134 ++++++++ .../VenvUserSiteIsolationTests.cs | 313 ++++++++++++++++++ .../VenvVersionValidationTests.cs | 79 +++++ .../Python/UiPath.Python/EngineProvider.cs | 40 ++- .../Python/UiPath.Python/Impl/Engine.cs | 104 ++++-- .../UiPath.Python/Impl/OutOfProcessEngine.cs | 22 +- .../UiPath.Python/Impl/VenvDetection.cs | 84 +++++ .../UiPath.Python/Properties/AssemblyInfo.cs | 1 + .../Properties/UiPath.Python.Designer.cs | 9 + .../Properties/UiPath.Python.resx | 3 + 11 files changed, 781 insertions(+), 23 deletions(-) create mode 100644 Activities/Python/UiPath.Python.Tests/VenvDetectionTests.cs create mode 100644 Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs create mode 100644 Activities/Python/UiPath.Python.Tests/VenvVersionValidationTests.cs create mode 100644 Activities/Python/UiPath.Python/Impl/VenvDetection.cs diff --git a/Activities/Python/Shared/UiPath.Shared.Service/Client/Controller.cs b/Activities/Python/Shared/UiPath.Shared.Service/Client/Controller.cs index e7510269e..a1a7d1a91 100644 --- a/Activities/Python/Shared/UiPath.Shared.Service/Client/Controller.cs +++ b/Activities/Python/Shared/UiPath.Shared.Service/Client/Controller.cs @@ -28,6 +28,19 @@ internal class Controller internal TimeSpan StartTimeout { get; set; } = Config.DefaultServiceCreationTimeout; + // Must be set in this (parent) process, before the host is spawned: environment variables + // changed from managed code already running inside the host, right before it initializes + // Python, are not reliably observed by the native interpreter there. Only variables present + // in the process's environment block at OS process-creation time are. + // + // ProcessStartInfo.EnvironmentVariables starts as a copy of this (parent) process's own + // environment, so if PYTHONNOUSERSITE already happens to be set there for unrelated reasons + // (e.g. a customer's own leftover manual workaround), it would otherwise flow straight + // through to the host untouched. When true, explicitly clears it for the host's own + // environment instead, so a venv's declared --system-site-packages behavior is decided by + // that flag alone, not by whatever's ambient on the machine (STUD-81085 follow-up). + internal bool ClearUserSiteEnvironmentOverride { get; set; } + internal HostWrapper Create() { StartHostService(); @@ -97,6 +110,8 @@ private ProcessStartInfo CreateProcessStartInfo(string hostFullPath, string fold // never reach the host until the buffer fills or the interpreter exits — and on // forced shutdown (Process.Kill) any buffered output is lost. psi.EnvironmentVariables["PYTHONUNBUFFERED"] = "1"; + if (ClearUserSiteEnvironmentOverride) + psi.EnvironmentVariables.Remove("PYTHONNOUSERSITE"); if (!isExeMode) psi.ArgumentList.Add(hostFullPath); return psi; diff --git a/Activities/Python/UiPath.Python.Tests/VenvDetectionTests.cs b/Activities/Python/UiPath.Python.Tests/VenvDetectionTests.cs new file mode 100644 index 000000000..63f76f615 --- /dev/null +++ b/Activities/Python/UiPath.Python.Tests/VenvDetectionTests.cs @@ -0,0 +1,134 @@ +using System; +using System.IO; +using UiPath.Python.Impl; +using UiPath.TestUtils; +using Xunit; +using Assert = Xunit.Assert; + +namespace UiPath.Python.Tests +{ + // Direct, fast tests for VenvDetection.GetVenvInfo — no Python engine involved. Covers the + // detection shapes discussed during the STUD-81085 review: venv root, its Scripts/bin + // launcher folder, and the false-positive an unbounded ancestor walk used to allow (an + // unrelated, fully-standalone installation merely sitting near someone else's pyvenv.cfg). + public class VenvDetectionTests : IDisposable + { + private const string Category = "Python"; + + private readonly string _rootDir; + + public VenvDetectionTests() + { + _rootDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "venv-detection-tests", Guid.NewGuid().ToString("N"))).FullName; + } + + public void Dispose() + { + try { Directory.Delete(_rootDir, true); } catch { /* best effort cleanup */ } + } + + private string WriteVenvCfg(string venvDir, string home = @"C:\FakeBase", string extra = null) + { + Directory.CreateDirectory(venvDir); + var content = $"home = {home}{Environment.NewLine}version = 3.13.0{Environment.NewLine}{extra}"; + File.WriteAllText(Path.Combine(venvDir, "pyvenv.cfg"), content); + return venvDir; + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void VenvRoot_Is_Detected() + { + var venvDir = WriteVenvCfg(Path.Combine(_rootDir, "myvenv")); + + var venv = VenvDetection.GetVenvInfo(venvDir); + + Assert.NotNull(venv); + Assert.Equal(venvDir, venv.Root); + Assert.Equal(@"C:\FakeBase", venv.Home); + } + + [Theory] + [InlineData("Scripts")] + [InlineData("bin")] + [Trait(TestCategories.Category, Category)] + public void LauncherSubfolder_Is_Detected(string folderName) + { + var venvDir = WriteVenvCfg(Path.Combine(_rootDir, "myvenv")); + var launcherDir = Directory.CreateDirectory(Path.Combine(venvDir, folderName)).FullName; + + var venv = VenvDetection.GetVenvInfo(launcherDir); + + Assert.NotNull(venv); + Assert.Equal(venvDir, venv.Root); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void UnrelatedFolder_OneLevelBelow_WrongName_Is_Not_Detected() + { + // pyvenv.cfg one level up, but the intermediate folder isn't a real venv launcher + // name — a fully standalone install could legitimately live here and must not be + // mistaken for being inside someone else's venv. + var venvDir = WriteVenvCfg(Path.Combine(_rootDir, "someones_venv")); + var standaloneDir = Directory.CreateDirectory(Path.Combine(venvDir, "runtime")).FullName; + + var venv = VenvDetection.GetVenvInfo(standaloneDir); + + Assert.Null(venv); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void TwoLevelsUp_Is_Not_Detected() + { + // Even with the right launcher name one level further up, detection deliberately + // doesn't walk a second level — no real venv layout ever needs it. + var venvDir = WriteVenvCfg(Path.Combine(_rootDir, "myvenv")); + var scriptsDir = Directory.CreateDirectory(Path.Combine(venvDir, "Scripts")).FullName; + var nestedDir = Directory.CreateDirectory(Path.Combine(scriptsDir, "nested")).FullName; + + var venv = VenvDetection.GetVenvInfo(nestedDir); + + Assert.Null(venv); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void StrayFile_Without_HomeOrVersion_Is_Not_Treated_As_Venv() + { + var dir = Directory.CreateDirectory(Path.Combine(_rootDir, "notavenv")).FullName; + File.WriteAllText(Path.Combine(dir, "pyvenv.cfg"), "some-unrelated-key = value" + Environment.NewLine); + + var venv = VenvDetection.GetVenvInfo(dir); + + Assert.Null(venv); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void SystemSitePackages_Flag_Is_Captured() + { + var venvDir = WriteVenvCfg(Path.Combine(_rootDir, "myvenv"), extra: "include-system-site-packages = true" + Environment.NewLine); + + var venv = VenvDetection.GetVenvInfo(venvDir); + + Assert.NotNull(venv); + Assert.True(venv.IncludeSystemSitePackages); + Assert.False(venv.ShouldDisableUserSite); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void Default_DoesNotIncludeSystemSitePackages_ShouldDisableUserSite() + { + var venvDir = WriteVenvCfg(Path.Combine(_rootDir, "myvenv")); + + var venv = VenvDetection.GetVenvInfo(venvDir); + + Assert.NotNull(venv); + Assert.False(venv.IncludeSystemSitePackages); + Assert.True(venv.ShouldDisableUserSite); + } + } +} diff --git a/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs b/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs new file mode 100644 index 000000000..38f05bfa1 --- /dev/null +++ b/Activities/Python/UiPath.Python.Tests/VenvUserSiteIsolationTests.cs @@ -0,0 +1,313 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using UiPath.TestUtils; +using Xunit; +using Assert = Xunit.Assert; + +namespace UiPath.Python.Tests +{ + // Regression test for STUD-81085: pointing Python Scope's Path at a venv must stop the + // interpreter's PEP-370 user-site directory (%APPDATA%\Roaming\Python\PythonXY\site-packages) + // from being processed at all. pywin32 registers its native DLL search directory + // (os.add_dll_directory) from a .pth file in whichever site directory it's installed into; + // if the user-site copy still gets processed alongside the venv's own copy, a differently + // built pywin32 there collides with the venv's copy and surfaces as + // "DLL load failed while importing win32api: The specified procedure could not be found." + // + // This test doesn't need real pywin32 binaries: the bug is that the user-site .pth runs at + // all when a venv is configured, so a .pth marker is a faithful, hermetic reproduction of the + // exact mechanism pywin32 relies on. Runs out-of-process (the real production default) since + // the fix depends on an environment variable set in the parent process before the host spawns. + public class VenvUserSiteIsolationTests : IDisposable + { + private const string Category = "Python"; + + private static readonly string EmbeddedRuntimePath = EmbeddedPythonRuntimeBootstrap.EnsureRuntimePath(); + private static readonly string EmbeddedLibraryPath = EmbeddedPythonRuntimeBootstrap.GetPythonLibraryPath(EmbeddedRuntimePath); + + // Must match the running embeddable interpreter's sys.version_info (3.14.5) — Windows + // user-site resolves to \Python\site-packages. + private const string UserSiteVersionFolder = "Python314"; + + private readonly string _rootDir; + private readonly string _venvDir; + private readonly string _userBaseDir; + private readonly string _markerFile; + private readonly string _previousUserBase; + private readonly string _previousNoUserSite; + + public VenvUserSiteIsolationTests() + { + _rootDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "venv-usersite-tests", Guid.NewGuid().ToString("N"))).FullName; + _venvDir = Path.Combine(_rootDir, ".venv"); + _userBaseDir = Path.Combine(_rootDir, "userbase"); + _markerFile = Path.Combine(_rootDir, "marker.txt"); + + _previousUserBase = Environment.GetEnvironmentVariable("PYTHONUSERBASE"); + _previousNoUserSite = Environment.GetEnvironmentVariable("PYTHONNOUSERSITE"); + } + + public void Dispose() + { + Environment.SetEnvironmentVariable("PYTHONUSERBASE", _previousUserBase); + Environment.SetEnvironmentVariable("PYTHONNOUSERSITE", _previousNoUserSite); + try { Directory.Delete(_rootDir, true); } catch { /* best effort cleanup */ } + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_Does_Not_Process_UserSite_PthFiles() + { + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + var venvSitePackages = Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")).FullName; + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), $"home = {EmbeddedRuntimePath}{Environment.NewLine}"); + WriteMarkerPth(venvSitePackages, "venv"); + + var userSitePackages = Directory.CreateDirectory(Path.Combine(_userBaseDir, UserSiteVersionFolder, "site-packages")).FullName; + WriteMarkerPth(userSitePackages, "usersite"); + Environment.SetEnvironmentVariable("PYTHONUSERBASE", _userBaseDir); + + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + try + { + await engine.Initialize(null, CancellationToken.None, 60); + } + finally + { + await engine.Release(); + } + + var markerLines = File.Exists(_markerFile) + ? File.ReadAllLines(_markerFile) + : Array.Empty(); + + // The venv's own .pth must still run — this isn't about disabling site processing, + // only about excluding the unrelated per-user directory. + Assert.Contains("venv", markerLines); + Assert.DoesNotContain("usersite", markerLines); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_With_SystemSitePackages_Still_Processes_UserSite_PthFiles() + { + // Mirrors CPython's own site.py venv() function: it only forces + // ENABLE_USER_SITE = False when the venv was created *without* + // --system-site-packages. A real, natively-activated --system-site-packages venv + // leaves user-site enabled (the normal, non-venv computation applies instead) — so + // this fix must not suppress it there either, or it would diverge from native parity + // for a case that already accepts the same DLL-collision exposure as any other + // non-venv interpreter. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + var venvSitePackages = Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")).FullName; + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), + $"home = {EmbeddedRuntimePath}{Environment.NewLine}include-system-site-packages = true{Environment.NewLine}"); + WriteMarkerPth(venvSitePackages, "venv"); + + var userSitePackages = Directory.CreateDirectory(Path.Combine(_userBaseDir, UserSiteVersionFolder, "site-packages")).FullName; + WriteMarkerPth(userSitePackages, "usersite"); + Environment.SetEnvironmentVariable("PYTHONUSERBASE", _userBaseDir); + + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + try + { + await engine.Initialize(null, CancellationToken.None, 60); + } + finally + { + await engine.Release(); + } + + var markerLines = File.Exists(_markerFile) + ? File.ReadAllLines(_markerFile) + : Array.Empty(); + + Assert.Contains("venv", markerLines); + Assert.Contains("usersite", markerLines); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_PythonHome_Resolves_To_Declared_Base_Install() + { + // Regression test for a separate bug found while exercising this fix: pointing + // PythonHome directly at the venv root (rather than the base install its own + // pyvenv.cfg declares) makes native init fail outright once a real installer-based + // Python is used, since a venv has no standard library of its own. The embeddable test + // runtime's own ._pth-based bootstrap resolves its stdlib independently of PythonHome, + // so it can't catch that failure directly — but PythonHome still governs what the + // interpreter reports as sys.base_prefix regardless, which is what this asserts. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")); + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), $"home = {EmbeddedRuntimePath}{Environment.NewLine}"); + + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + string basePrefix; + try + { + await engine.Initialize(null, CancellationToken.None, 60); + + var script = await engine.LoadScript( + "import sys\ndef check():\n return sys.base_prefix\n", + CancellationToken.None); + var result = await engine.InvokeMethod(script, "check", null, CancellationToken.None); + basePrefix = (string)engine.Convert(result, typeof(string)); + } + finally + { + await engine.Release(); + } + + Assert.Equal( + Path.GetFullPath(EmbeddedRuntimePath).TrimEnd(Path.DirectorySeparatorChar), + Path.GetFullPath(basePrefix).TrimEnd(Path.DirectorySeparatorChar), + ignoreCase: true); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_Does_Not_Leak_BaseInstall_SitePackages() + { + // Regression test for a second, distinct leak path found by inspecting a real venv's + // actual sys.path: CPython's site.main() unconditionally adds the base install's own + // site-packages (and the bare base install prefix itself) unless something narrows + // PREFIXES first — which never happened for the embedded interpreter, since site.py's + // own venv() detection can't trigger for it. PYTHONNOUSERSITE never covered this; only + // SetNoSiteFlag (skipping site.main() entirely for this case) does. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")); + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), $"home = {EmbeddedRuntimePath}{Environment.NewLine}"); + + var sysPath = await GetSysPath(); + var normalizedBase = Path.GetFullPath(EmbeddedRuntimePath).TrimEnd(Path.DirectorySeparatorChar); + + Assert.DoesNotContain(sysPath, p => Path.GetFullPath(p).TrimEnd(Path.DirectorySeparatorChar) + .Equals(normalizedBase, StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_With_SystemSitePackages_Still_Includes_BaseInstall_SitePackages() + { + // The --system-site-packages case must keep behaving exactly as it did before this + // change: SetNoSiteFlag is only set for the default (ShouldDisableUserSite) case, so + // site.main() still runs normally here and still adds the base install unconditionally + // — which happens to already be correct for this specific flag. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")); + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), + $"home = {EmbeddedRuntimePath}{Environment.NewLine}include-system-site-packages = true{Environment.NewLine}"); + + var sysPath = await GetSysPath(); + var normalizedBase = Path.GetFullPath(EmbeddedRuntimePath).TrimEnd(Path.DirectorySeparatorChar); + + Assert.Contains(sysPath, p => Path.GetFullPath(p).TrimEnd(Path.DirectorySeparatorChar) + .Equals(normalizedBase, StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_SiteCustomize_Still_Runs() + { + // SetNoSiteFlag skips site.main() entirely for the default venv case, which would also + // silently skip sitecustomize.py auto-import (some environments rely on it for + // corporate setup) unless something restores it — Engine.PostInitializationVenvSetup + // explicitly calls site.execsitecustomize() for exactly this reason. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + var venvSitePackages = Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")).FullName; + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), $"home = {EmbeddedRuntimePath}{Environment.NewLine}"); + + var markerPath = _markerFile.Replace("\\", "/"); + File.WriteAllText(Path.Combine(venvSitePackages, "sitecustomize.py"), + $"import codecs{Environment.NewLine}codecs.open('{markerPath}', 'a', encoding='utf-8').write('sitecustomize\\n'){Environment.NewLine}"); + + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + try + { + await engine.Initialize(null, CancellationToken.None, 60); + } + finally + { + await engine.Release(); + } + + var markerLines = File.Exists(_markerFile) ? File.ReadAllLines(_markerFile) : Array.Empty(); + Assert.Contains("sitecustomize", markerLines); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public async Task Venv_With_SystemSitePackages_Ignores_Ambient_PYTHONNOUSERSITE() + { + // Regression test: a customer's own leftover PYTHONNOUSERSITE=1 (e.g. a manual + // workaround predating this fix, exactly like the one mentioned in the original + // ticket) sitting in the *ambient* environment used to leak straight into the spawned + // host via ProcessStartInfo.EnvironmentVariables (which starts as a copy of this + // process's own environment), silently defeating a --system-site-packages venv's + // intent to leave user-site enabled — regardless of what our own code did or didn't + // set. Controller.ClearUserSiteEnvironmentOverride exists specifically to guarantee + // this venv flag decides the outcome, not whatever's ambient on the machine. + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + Environment.SetEnvironmentVariable("PYTHONNOUSERSITE", "1"); + + var venvSitePackages = Directory.CreateDirectory(Path.Combine(_venvDir, "Lib", "site-packages")).FullName; + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), + $"home = {EmbeddedRuntimePath}{Environment.NewLine}include-system-site-packages = true{Environment.NewLine}"); + WriteMarkerPth(venvSitePackages, "venv"); + + var userSitePackages = Directory.CreateDirectory(Path.Combine(_userBaseDir, UserSiteVersionFolder, "site-packages")).FullName; + WriteMarkerPth(userSitePackages, "usersite"); + Environment.SetEnvironmentVariable("PYTHONUSERBASE", _userBaseDir); + + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + try + { + await engine.Initialize(null, CancellationToken.None, 60); + } + finally + { + await engine.Release(); + } + + var markerLines = File.Exists(_markerFile) ? File.ReadAllLines(_markerFile) : Array.Empty(); + + Assert.Contains("venv", markerLines); + Assert.Contains("usersite", markerLines); + } + + private async Task GetSysPath() + { + var engine = EngineProvider.Get(Version.Python_310, _venvDir, EmbeddedLibraryPath, inProcess: false); + try + { + await engine.Initialize(null, CancellationToken.None, 60); + + var script = await engine.LoadScript( + "import sys\ndef check():\n return list(sys.path)\n", + CancellationToken.None); + var result = await engine.InvokeMethod(script, "check", null, CancellationToken.None); + return (string[])engine.Convert(result, typeof(string[])); + } + finally + { + await engine.Release(); + } + } + + private void WriteMarkerPth(string siteDir, string tag) + { + var markerPath = _markerFile.Replace("\\", "/"); + var pthLine = $"import codecs; codecs.open('{markerPath}', 'a', encoding='utf-8').write('{tag}\\n')"; + File.WriteAllText(Path.Combine(siteDir, $"zzz_{tag}_marker.pth"), pthLine + Environment.NewLine); + } + } +} diff --git a/Activities/Python/UiPath.Python.Tests/VenvVersionValidationTests.cs b/Activities/Python/UiPath.Python.Tests/VenvVersionValidationTests.cs new file mode 100644 index 000000000..d0d9e89be --- /dev/null +++ b/Activities/Python/UiPath.Python.Tests/VenvVersionValidationTests.cs @@ -0,0 +1,79 @@ +using System; +using System.IO; +using UiPath.TestUtils; +using Xunit; +using Assert = Xunit.Assert; + +namespace UiPath.Python.Tests +{ + // Regression tests for EngineProvider.ValidateVenvVersion (called from the public + // ValidateInstallation): a venv's declared Python version (its own pyvenv.cfg) must match the + // version actually loaded from LibraryPath, or initialization would otherwise proceed with a + // mismatched interpreter/site-packages pairing and fail later with a confusing error instead + // of a clear one up front. + public class VenvVersionValidationTests : IDisposable + { + private const string Category = "Python"; + + private static readonly string EmbeddedRuntimePath = EmbeddedPythonRuntimeBootstrap.EnsureRuntimePath(); + private static readonly string EmbeddedLibraryPath = EmbeddedPythonRuntimeBootstrap.GetPythonLibraryPath(EmbeddedRuntimePath); + + private readonly string _rootDir; + private readonly string _venvDir; + + public VenvVersionValidationTests() + { + _rootDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "venv-version-tests", Guid.NewGuid().ToString("N"))).FullName; + _venvDir = Path.Combine(_rootDir, ".venv"); + } + + public void Dispose() + { + try { Directory.Delete(_rootDir, true); } catch { /* best effort cleanup */ } + } + + private void WriteVenvCfg(string version) + { + Directory.CreateDirectory(_venvDir); + File.WriteAllText(Path.Combine(_venvDir, "pyvenv.cfg"), + $"home = {EmbeddedRuntimePath}{Environment.NewLine}version = {version}{Environment.NewLine}"); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void MismatchedVenvVersion_Throws() + { + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + // The embeddable runtime bootstrap is 3.14.5 — declare something else entirely. + WriteVenvCfg("3.9.0"); + + Assert.Throws(() => EngineProvider.ValidateInstallation(_venvDir, EmbeddedLibraryPath)); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void MatchingVenvVersion_DoesNotThrow() + { + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + WriteVenvCfg("3.14.5"); + + // Must not throw. + EngineProvider.ValidateInstallation(_venvDir, EmbeddedLibraryPath); + } + + [Fact] + [Trait(TestCategories.Category, Category)] + public void NonVenvPath_DoesNotThrow() + { + Skip.IfNot(Directory.Exists(EmbeddedRuntimePath)); + + // _venvDir has no pyvenv.cfg at all here — not a venv, so the version cross-check + // must not even engage. + Directory.CreateDirectory(_venvDir); + + EngineProvider.ValidateInstallation(_venvDir, EmbeddedLibraryPath); + } + } +} diff --git a/Activities/Python/UiPath.Python/EngineProvider.cs b/Activities/Python/UiPath.Python/EngineProvider.cs index 7a0e5e58d..20d47fc49 100644 --- a/Activities/Python/UiPath.Python/EngineProvider.cs +++ b/Activities/Python/UiPath.Python/EngineProvider.cs @@ -142,13 +142,51 @@ private static bool Autodetect(string pythonFullPath, out Version version, out E /// executable) and its dynamic library (, via binary inspection). /// Either input may be absent or invalid independently; each check no-ops when its input is /// missing and throws / - /// when it finds a 32-bit or unsupported runtime. + /// when it finds a 32-bit or unsupported runtime. Also cross-checks a venv at + /// against the version actually loaded from (see + /// ). /// public static void ValidateInstallation(string path, string libraryPath) { // Library first: it is the exact artifact pythonnet loads and the check is cheap (no spawn). ValidatePythonLibrary(libraryPath); ValidatePythonExecutable(path); + ValidateVenvVersion(path, libraryPath); + } + + /// + /// If is a venv, cross-checks the Python version it was created with + /// (from its pyvenv.cfg) against the version actually loaded from . + /// A mismatch means the venv's site-packages — compiled for a different ABI — would end up on + /// sys.path for a differently-versioned interpreter. Surfaced here, before any native + /// initialization, rather than as a confusing failure deep inside the engine (e.g. a missing + /// _sysconfigdata module, or subtly wrong stdlib behavior). No-ops when either version can't be + /// determined — engine initialization will surface its own error in that case. + /// + private static void ValidateVenvVersion(string path, string libraryPath) + { + var venv = VenvDetection.GetVenvInfo(path); + if (venv?.Version == null || !TryParseVenvVersion(venv.Version, out int venvMajor, out int venvMinor)) + return; + + if (!TryGetLibraryVersion(libraryPath, out int libMajor, out int libMinor)) + return; + + if (venvMajor != libMajor || venvMinor != libMinor) + throw new NotSupportedException( + string.Format(Resources.PythonVenvVersionMismatchException, + venv.Root, $"{venvMajor}.{venvMinor}", $"{libMajor}.{libMinor}")); + } + + /// + /// Parses the "major.minor(.patch)" version string pyvenv.cfg's version key always carries. + /// + private static bool TryParseVenvVersion(string version, out int major, out int minor) + { + major = 0; + minor = 0; + var parts = version.Split('.'); + return parts.Length >= 2 && int.TryParse(parts[0], out major) && int.TryParse(parts[1], out minor); } /// diff --git a/Activities/Python/UiPath.Python/Impl/Engine.cs b/Activities/Python/UiPath.Python/Impl/Engine.cs index b87152941..9ab74d9c1 100644 --- a/Activities/Python/UiPath.Python/Impl/Engine.cs +++ b/Activities/Python/UiPath.Python/Impl/Engine.cs @@ -55,20 +55,71 @@ public async Task Initialize(string workingFolder, CancellationToken ct, double Trace.TraceInformation($"Initializing Python runtime using version {_version} and path {_path}"); Stopwatch sw = Stopwatch.StartNew(); + // Detected before Initialize(): a real venv (activated normally, e.g. + // \Scripts\python.exe) always disables the PEP-370 user-site + // directory on its own. Our embedded interpreter never goes through that + // activation path, so nothing does this for us — without it, a native + // package installed in both the venv and the user-site directory (e.g. + // pywin32) can resolve its Python module from one and its native DLL + // dependency from the other, mismatched, copy (STUD-81085). Suppression + // itself happens below, via PythonEngine.SetNoSiteFlag() — see that call + // for why (it also closes a second, related leak path, and is immune to + // whatever PYTHONNOUSERSITE happens to already be set in the ambient + // environment, which an env-var-based approach was not). + var venv = _version == Version.Python_310 ? VenvDetection.GetVenvInfo(_path) : null; + if (_isWindows && !_path.IsNullOrEmpty()) SetDllDirectory(Path.GetFullPath(_path)); if (!_libraryPath.IsNullOrEmpty()) Runtime.PythonDLL = _libraryPath; - if (!_path.IsNullOrEmpty()) - PythonEngine.PythonHome = _path; + // A venv's own folder is not a valid PythonHome: it only has + // Lib\site-packages, not the standard library (Lib\encodings etc.), so + // pointing the native interpreter at it fails at the very first import + // with "Fatal Python error: Failed to import encodings module". Use the + // base install recorded in the venv's own pyvenv.cfg instead — exactly + // what a normally-activated venv resolves to on its own. EngineProvider + // already validated that base install's version matches _libraryPath's. + var pythonHome = venv != null ? ResolvePythonHome(venv.Home) : _path; + if (!pythonHome.IsNullOrEmpty()) + PythonEngine.PythonHome = pythonHome; + + // For a default venv (no --system-site-packages), suppresses both the + // PEP-370 user-site leak (STUD-81085) and a second, distinct leak path + // found by inspecting a real venv's actual sys.path: CPython's own + // site.main() also unconditionally adds the *base install's* own + // site-packages (site.addsitepackages() against sys.prefix/exec_prefix, + // still pointing at the base install at this point). PYTHONNOUSERSITE + // would only ever have covered the first of these — SetNoSiteFlag + // (Py_NoSiteFlag) disables site.main()'s automatic run entirely, so + // *neither* ever gets added in the first place: "prevent, don't clean up + // after" — a cleanup-after-the-fact fix couldn't undo any .pth-triggered + // side effects, e.g. os.add_dll_directory calls, that already ran by the + // time managed code regains control. It's also an in-memory flag on this + // process's loaded Python DLL, never written to os.environ — unlike an + // env-var-based approach, it can't be defeated by (or leak into) whatever + // PYTHONNOUSERSITE the ambient environment happens to already carry, which + // is exactly the failure mode found in Controller.cs's + // ClearUserSiteEnvironmentOverride for the --system-site-packages case. + // `site` itself is still importable — + // this only skips its automatic invocation at startup — so the explicit + // site.addsitedir() call in PostInitializationVenvSetup for the venv's own + // site-packages, and the site.execsitecustomize() call there preserving + // sitecustomize.py support, both keep working. Must come after + // Runtime.PythonDLL/PythonHome are set, not before — calling it earlier + // left Runtime.PythonDLL null by the time the host tried to use it (and, + // per a known pythonnet issue, SetNoSiteFlag itself can be silently + // ignored on Windows unless another PythonEngine call already preceded + // it — PythonHome, set just above, already satisfies that). + if (venv != null && venv.ShouldDisableUserSite) + PythonEngine.SetNoSiteFlag(); PythonEngine.Initialize(); ct.ThrowIfCancellationRequested(); - PostInitializationVenvSetup(); + PostInitializationVenvSetup(venv); PythonEngine.BeginAllowThreads(); sw.Stop(); @@ -266,16 +317,23 @@ private string GetInitializationScript() return reader.ReadToEnd(); } - private static bool IsVenv(string path) => File.Exists(Path.Combine(path, "pyvenv.cfg")); - - private static string GetVenvPath(string venvPath, int maxLevels = 3) + /// + /// Normalizes a venv's pyvenv.cfg "home" value into a PythonHome-compatible prefix. On + /// Windows "home" already is the install root (no adjustment needed). On POSIX it records + /// the base install's bin folder (e.g. "/usr/bin"), one level below the prefix PythonHome + /// actually expects (e.g. "/usr") — strip it when present. + /// + private static string ResolvePythonHome(string venvHome) { - if (string.IsNullOrEmpty(venvPath) || maxLevels == 0) + if (venvHome.IsNullOrEmpty()) return null; - else if (IsVenv(venvPath)) - return venvPath; - else - return GetVenvPath(Path.GetDirectoryName(venvPath), maxLevels - 1); + + var trimmed = venvHome.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + var lastSegment = Path.GetFileName(trimmed); + if (string.Equals(lastSegment, "bin", StringComparison.Ordinal)) + return Path.GetDirectoryName(trimmed); + + return venvHome; } private static string GetEnvSitePackagesPath(string venvPath) @@ -296,29 +354,35 @@ private static string GetEnvSitePackagesPath(string venvPath) return sitePackages; } - private void PostInitializationVenvSetup() + private void PostInitializationVenvSetup(VenvDetection.VenvInfo venv) { - if (_version != Version.Python_310) - return; - - var venvPath = GetVenvPath(_path); - if (!string.IsNullOrWhiteSpace(venvPath)) + if (venv != null) { using (Py.GIL()) { dynamic sys = Py.Import("sys"); dynamic site = Py.Import("site"); - sys.prefix = venvPath; - sys.exec_prefix = venvPath; + sys.prefix = venv.Root; + sys.exec_prefix = venv.Root; - var sitePackagesPath = GetEnvSitePackagesPath(venvPath); + var sitePackagesPath = GetEnvSitePackagesPath(venv.Root); site.addsitedir(sitePackagesPath); if ((bool)sys.path.__contains__(sitePackagesPath)) sys.path.remove(sitePackagesPath); sys.path.insert(0, sitePackagesPath); + + // SetNoSiteFlag (see Initialize()) skips site.main() entirely for a default + // venv, which also skips its sitecustomize.py auto-import — some environments + // rely on that for corporate setup (proxies, logging, etc.), and it did run + // today before this change, so preserve it explicitly. Safe to call even when + // SetNoSiteFlag wasn't set (--system-site-packages venvs): site.main() already + // ran it there, and re-importing an already-imported module is a no-op. + // Deliberately not calling execusercustomize() — its user-site counterpart, + // consistent with suppressing user-site itself. + site.execsitecustomize(); } } } diff --git a/Activities/Python/UiPath.Python/Impl/OutOfProcessEngine.cs b/Activities/Python/UiPath.Python/Impl/OutOfProcessEngine.cs index 06bfe2612..987416847 100644 --- a/Activities/Python/UiPath.Python/Impl/OutOfProcessEngine.cs +++ b/Activities/Python/UiPath.Python/Impl/OutOfProcessEngine.cs @@ -50,11 +50,29 @@ public Task Initialize(string workingFolder, CancellationToken ct, double timeou Stopwatch sw = Stopwatch.StartNew(); - // TODO: expose visible as a property? + var venv = _version == Version.Python_310 ? VenvDetection.GetVenvInfo(_path) : null; + + // Actual user-site suppression for the default (ShouldDisableUserSite) case happens + // inside Engine.Initialize() itself, via PythonEngine.SetNoSiteFlag() — that runs in + // this same host process regardless, so no parent-side plumbing is needed for it + // (an env-var-based attempt used to live here; it turned out to be both unreliable + // when set this late from managed code in an already-spawned process, and defeatable + // by whatever PYTHONNOUSERSITE the ambient environment already carried). + // + // What *does* still need to happen here, in the parent, before the host spawns: for a + // --system-site-packages venv (ShouldDisableUserSite == false), the intent is to leave + // user-site exactly as a normal, non-embedded interpreter would — but ProcessStartInfo + // starts as a copy of this process's own environment, so if PYTHONNOUSERSITE already + // happens to be set there (e.g. a customer's own leftover workaround, unrelated to this + // fix), it would otherwise leak into the host and silently force user-site off anyway, + // regardless of what SetNoSiteFlag does or doesn't do for the other case. Clearing it + // explicitly for the child guarantees the venv's own IncludeSystemSitePackages flag is + // what decides this, not whatever's ambient on the machine. _provider = new Controller() { PythonHostLibFile = ServiceDll_x64, - Visible = _visible + Visible = _visible, + ClearUserSiteEnvironmentOverride = venv != null && venv.IncludeSystemSitePackages }; // Set LogTrace before Create() so the diagnostic file (if enabled) captures diff --git a/Activities/Python/UiPath.Python/Impl/VenvDetection.cs b/Activities/Python/UiPath.Python/Impl/VenvDetection.cs new file mode 100644 index 000000000..9431d0e67 --- /dev/null +++ b/Activities/Python/UiPath.Python/Impl/VenvDetection.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace UiPath.Python.Impl +{ + /// + /// Shared by (runs inside the process that calls Py_Initialize) and + /// (runs in the calling process, before the host is spawned). + /// + internal static class VenvDetection + { + internal sealed record VenvInfo(string Root, string Home, bool IncludeSystemSitePackages, string Version) + { + /// + /// Mirrors CPython's own site.py venv() function: it only forces + /// ENABLE_USER_SITE = False for a venv created without --system-site-packages. A + /// --system-site-packages venv leaves it to the normal (non-venv) computation, which is + /// True in the typical case — so a real, natively-activated venv like that does *not* + /// disable user-site. + /// + internal bool ShouldDisableUserSite => !IncludeSystemSitePackages; + } + + // Real venvs put their launcher/executable folder directly under the venv root, named + // exactly this — the same names EngineProvider looks for python.exe/python3 under. + private static readonly string[] VenvBinFolderNames = ["Scripts", "bin"]; + + /// + /// Parses pyvenv.cfg at , if present. Requiring at least one of the + /// keys a real venv config always has (home/version) avoids treating an unrelated file that + /// merely happens to be named pyvenv.cfg as a venv. + /// + private static VenvInfo TryReadVenvConfig(string path) + { + var cfgFile = Path.Combine(path, "pyvenv.cfg"); + if (!File.Exists(cfgFile)) + return null; + + var kv = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var line in File.ReadLines(cfgFile)) + { + var parts = line.Split('=', 2); + if (parts.Length == 2) + kv[parts[0].Trim()] = parts[1].Trim(); + } + + if (!kv.ContainsKey("home") && !kv.ContainsKey("version")) + return null; + + kv.TryGetValue("home", out var home); + kv.TryGetValue("version", out var version); + var includeSystemSitePackages = kv.TryGetValue("include-system-site-packages", out var include) + && string.Equals(include, "true", StringComparison.OrdinalIgnoreCase); + + return new VenvInfo(path, home, includeSystemSitePackages, version); + } + + /// + /// Detects whether is a venv root, or its immediate Scripts/bin + /// launcher folder — the only two layouts a real venv actually produces. Deliberately does + /// not walk further up than that, and only accepts the one-level-up case when the + /// intermediate folder is actually named Scripts/bin: a bare "is there a pyvenv.cfg within + /// N ancestor levels" search (the original implementation) can mistake an unrelated, + /// fully-standalone Python installation that merely happens to sit a level or two beneath + /// someone else's venv for being inside it. + /// + internal static VenvInfo GetVenvInfo(string path) + { + if (string.IsNullOrEmpty(path)) + return null; + + var direct = TryReadVenvConfig(path); + if (direct != null) + return direct; + + var folderName = Path.GetFileName(path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); + if (!Array.Exists(VenvBinFolderNames, name => string.Equals(name, folderName, StringComparison.OrdinalIgnoreCase))) + return null; + + return TryReadVenvConfig(Path.GetDirectoryName(path)); + } + } +} diff --git a/Activities/Python/UiPath.Python/Properties/AssemblyInfo.cs b/Activities/Python/UiPath.Python/Properties/AssemblyInfo.cs index 8798b2300..353866e2a 100644 --- a/Activities/Python/UiPath.Python/Properties/AssemblyInfo.cs +++ b/Activities/Python/UiPath.Python/Properties/AssemblyInfo.cs @@ -4,6 +4,7 @@ [assembly: XmlnsDefinition("http://schemas.uipath.com/workflow/activities/python", "UiPath.Python")] [assembly: InternalsVisibleTo("UiPath.Python.Activities.API.Tests")] +[assembly: InternalsVisibleTo("UiPath.Python.Tests")] // Setting ComVisible to false makes the types in this assembly not visible // to COM components. If you need to access a type in this assembly from diff --git a/Activities/Python/UiPath.Python/Properties/UiPath.Python.Designer.cs b/Activities/Python/UiPath.Python/Properties/UiPath.Python.Designer.cs index 93e778a99..1216a6083 100644 --- a/Activities/Python/UiPath.Python/Properties/UiPath.Python.Designer.cs +++ b/Activities/Python/UiPath.Python/Properties/UiPath.Python.Designer.cs @@ -212,5 +212,14 @@ public static string PythonVersionNotSupportedException { return ResourceManager.GetString("PythonVersionNotSupportedException", resourceCulture); } } + + /// + /// Looks up a localized string similar to The virtual environment at '{0}' was created with Python {1}, but LibraryPath points to Python {2}. Point LibraryPath to a Python {1} installation that matches the virtual environment. + /// + public static string PythonVenvVersionMismatchException { + get { + return ResourceManager.GetString("PythonVenvVersionMismatchException", resourceCulture); + } + } } } diff --git a/Activities/Python/UiPath.Python/Properties/UiPath.Python.resx b/Activities/Python/UiPath.Python/Properties/UiPath.Python.resx index d460bbd3a..2d2bced01 100644 --- a/Activities/Python/UiPath.Python/Properties/UiPath.Python.resx +++ b/Activities/Python/UiPath.Python/Properties/UiPath.Python.resx @@ -165,6 +165,9 @@ Python {0} is not supported. Supported versions are: {1}. + + The virtual environment at '{0}' was created with Python {1}, but LibraryPath points to Python {2}. Point LibraryPath to a Python {1} installation that matches the virtual environment. + The Python script data size ({0} MB) exceeds the configured limit ({1} MB). Pass large data via a file path instead of as a method argument, or increase the Script Data Size Limit property.