fix(toolchains): exclude pycache symlinks from runtime files

docs/environment-variables.md

@@ -98,7 +98,7 @@ be stored in.
 This directory may be reused between invocations, depending on the sandboxing
 configuration. Setting it to `/dev/null` will, in effect, disable runtime
 pyc caching. By setting e.g.
-`--sandbox_add_mount_pair=/tmp/rules_python_pycache`, it's possible for pyc
+`--sandbox_add_mount_pair=$HOME/.cache/rules_python_pycache`, it's possible for pyc
 caching to persist across invocations.
 
 **Behavior specific to downloaded runtimes:** 
@@ -109,13 +109,18 @@ Otherwise, the following environment variables are checked in the following
 order. Their values will have `rules_python_pycache` appended to them to form
 the root pycache directory:
 1. `XDG_CACHE_HOME`.
-2. `TMP` (non-Windows) or `TEMP` (Windows).
-3. The common platform-specific temporary directory (`/tmp` (non-Windows) or
-   `C:\Temp` (Windows)).
+2. `$HOME/.cache` (non-Windows), the XDG default when `XDG_CACHE_HOME` is
+   unset, or `LOCALAPPDATA` (Windows).
 
 If such a diretory cannot be found, or created, then `/dev/null` will be used,
 which will effectively disable pyc caching.
 
+:::{versionchanged} VERSION_NEXT_PATCH
+Temporary directories (`TMP`, `TEMP`, `/tmp` and `C:\Temp`) are no longer used.
+They get cleaned out while the fetched runtime persists, which left the
+`__pycache__` symlinks dangling.
+:::
+
 :::
 
 :::{envvar} RULES_PYTHON_PYPI_HUB_RESERVED

news/4198.changed.md

@@ -0,0 +1,3 @@
+(toolchains) Downloaded runtimes cache pyc files under `$HOME/.cache`
+(`LOCALAPPDATA` on Windows) instead of a temporary directory when
+`XDG_CACHE_HOME` is unset.

news/4198.fixed.md

@@ -0,0 +1,4 @@
+(toolchains) `__pycache__` symlinks in downloaded runtimes no longer end up in
+runfiles, and a runtime no longer loads pyc files compiled from another
+version's sources
+([#4198](https://github.com/bazel-contrib/rules_python/issues/4198)).

python/private/python_repository.bzl

@@ -63,38 +63,24 @@ def _get_pycache_root(rctx):
     os_name = repo_utils.get_platforms_os_name(rctx)
     is_windows = os_name == "windows"
 
-    # 1. RULES_PYTHON_PYCACHE_DIR
+    # Python checks a pyc file against its source's mtime and size, and the
+    # sources in a runtime's archive all share one mtime. Keying on the archive
+    # keeps a runtime from loading pyc files compiled from another's sources.
+    suffix = "{}/{}".format(rctx.name, rctx.attr.sha256[:16] or rctx.attr.python_version)
+
     res = rctx.getenv("RULES_PYTHON_PYCACHE_DIR")
     if res:
-        res = res + "/" + rctx.name
+        res = res + "/" + suffix
         return repo_utils.mkdir(rctx, res)
 
-    # Suffix for cases 2-4
-    # The first level directory is static and documented so that it is easy to
-    # use with e.g. --sandbox_add_mount_pair=/tmp/rules_python_pycache
-    suffix = "rules_python_pycache/{}/{}".format(hash(str(rctx.workspace_root)), rctx.name)
-
-    # 2. XDG_CACHE_HOME
     res = rctx.getenv("XDG_CACHE_HOME")
-    if res:
-        path = repo_utils.mkdir(rctx, rctx.path(res).get_child(suffix))
-        if path:
-            return path
-
-    # 3. TMP or TEMP
-    res = rctx.getenv("TMP") or rctx.getenv("TEMP")
-    if res:
-        path = repo_utils.mkdir(rctx, rctx.path(res).get_child(suffix))
-        if path:
-            return path
-
-    # 4. /tmp or Windows equivalent
-    if is_windows:
-        path = rctx.path("C:/Temp").get_child(suffix)
-    else:
-        path = rctx.path("/tmp").get_child(suffix)
-
-    return repo_utils.mkdir(rctx, path)
+    if not res:
+        res = rctx.getenv("LOCALAPPDATA" if is_windows else "HOME")
+        if res and not is_windows:
+            res += "/.cache"
+    if not res:
+        return None
+    return repo_utils.mkdir(rctx, rctx.path(res).get_child("rules_python_pycache").get_child(suffix))
 
 def _create_pycache_symlinks(rctx, logger):
     """Finds all directories with a .py file and creates __pycache__ symlinks.
@@ -252,9 +238,9 @@ def _python_repository_impl(rctx):
         # Exclude them from the glob because otherwise between the first time and second time a python toolchain is used,"
         # the definition of this filegroup will change, and depending rules will get invalidated."
         # See https://github.com/bazel-contrib/rules_python/issues/1008 for unconditionally adding these to toolchains so we can stop ignoring them."
-        # pyc* is ignored because pyc creation creates temporary .pyc.NNNN files
-        "**/__pycache__/*.pyc*",
-        "**/__pycache__/*.pyo*",
+        # The __pycache__ directories themselves are symlinks out of the repository.
+        "**/__pycache__",
+        "**/__pycache__/**",
     ]
 
     if "windows" in platform:

tests/integration/custom_commands_test.py

@@ -28,6 +28,17 @@ def test_run_build_python_zip_false(self):
         )
         self.assert_result_matches(result, "bazel-out")
 
+    # Regression test for https://github.com/bazel-contrib/rules_python/issues/4198
+    def test_runtime_files_exclude_pycache_symlinks(self):
+        result = self.run_bazel(
+            "cquery",
+            "--repo_env=RULES_PYTHON_PYCACHE_DIR=/dev/null",
+            "--output=files",
+            "deps(//:bin)",
+        )
+        pycache = [f for f in result.stdout.splitlines() if "__pycache__" in f]
+        self.assertEqual(pycache, [])
+
 
 if __name__ == "__main__":
     # Enabling this makes the runner log subprocesses as the test goes along.