From ef7a524ad76ba66c6abb4df9f837011b17f92553 Mon Sep 17 00:00:00 2001 From: Gleb Kolobkov Date: Fri, 9 Oct 2026 01:17:16 +0000 Subject: [PATCH 1/2] fix(bootstrap): restore imports for native Python targets Bazel 7 native Python rules leave newer launcher placeholders unexpanded. This crashes venv-symlink parsing in the Python bootstrap and passes a literal interpreter-argument token as a script filename in the shell bootstrap. The main-file fallback also omits imports. Handle the unexpanded placeholders and restore native import ordering in both bootstraps when falling back to the main file. Normalize external entry-point paths in the shell fallback and preserve normal stage-2 behavior, interpreter arguments, and inherited PYTHONPATH. Add regression tests for both launchers and a release-note entry. --- news/native-bootstrap-imports.fixed.md | 4 + python/private/python_bootstrap_template.txt | 33 ++- python/private/stage1_bootstrap_template.sh | 50 ++++ tests/bootstrap_impls/BUILD.bazel | 17 ++ .../bootstrap_impls/native_bootstrap_test.py | 242 ++++++++++++++++++ 5 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 news/native-bootstrap-imports.fixed.md create mode 100644 tests/bootstrap_impls/native_bootstrap_test.py diff --git a/news/native-bootstrap-imports.fixed.md b/news/native-bootstrap-imports.fixed.md new file mode 100644 index 0000000000..516aa18c05 --- /dev/null +++ b/news/native-bootstrap-imports.fixed.md @@ -0,0 +1,4 @@ +(bootstrap) Bazel 7 native Python targets can run with both the Python and +shell bootstraps without crashing on unexpanded launcher placeholders. Their +declared import roots and repository imports are available when the launcher +falls back to executing the main file directly. diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index 482918c038..29b1c2a392 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -39,6 +39,7 @@ import subprocess _STAGE2_BOOTSTRAP_SENTINEL = "%stage2" + "_bootstrap%" # runfiles-root-relative path STAGE2_BOOTSTRAP="%stage2_bootstrap%" +_USING_STAGE2_BOOTSTRAP_FALLBACK = False # NOTE: The fallback logic from stage2_bootstrap to main is only present # as a courtesy for an older, unsupported, configuration. It can be removed @@ -49,6 +50,7 @@ if STAGE2_BOOTSTRAP == _STAGE2_BOOTSTRAP_SENTINEL: _main = "%main%" if _main != _MAIN_SENTINEL and _main: STAGE2_BOOTSTRAP = _main + _USING_STAGE2_BOOTSTRAP_FALLBACK = True else: STAGE2_BOOTSTRAP = "" @@ -98,7 +100,13 @@ else: RUNTIME_VENV_SYMLINKS = """ %runtime_venv_symlinks% """.strip().split("\n") -RUNTIME_VENV_SYMLINKS = dict(line.split("|") for line in RUNTIME_VENV_SYMLINKS if line) +# Native rules leave this newer placeholder unchanged. They have no runtime +# venv symlinks; only parse entries when the placeholder was expanded. +_RUNTIME_VENV_SYMLINKS_SENTINEL = "%runtime_venv" + "_symlinks%" +if RUNTIME_VENV_SYMLINKS == [_RUNTIME_VENV_SYMLINKS_SENTINEL]: + RUNTIME_VENV_SYMLINKS = {} +else: + RUNTIME_VENV_SYMLINKS = dict(line.split("|") for line in RUNTIME_VENV_SYMLINKS if line) ADDITIONAL_INTERPRETER_ARGS = os.environ.get("RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS", "") EXTRACT_ROOT = os.environ.get("RULES_PYTHON_EXTRACT_ROOT") @@ -614,6 +622,29 @@ def main(): print_verbose("runfiles root:", runfiles_root) + if _USING_STAGE2_BOOTSTRAP_FALLBACK: + # Native rules expand %imports% and %import_all%, but do not generate + # stage 2 to set up sys.path. Preserve the native launcher's import order + # when executing the main file directly; leave regular stage-2 launches + # unchanged. These paths are runfiles-relative, not checkout-relative. + imports = "%imports%" + python_paths = [runfiles_root] + python_paths.extend(join(runfiles_root, path) for path in imports.split(":") if path) + if "%import_all%" == "True": + python_paths.extend( + join(runfiles_root, repo) for repo in sorted(os.listdir(runfiles_root)) + if os.path.isdir(join(runfiles_root, repo))) + else: + python_paths.append(join(runfiles_root, WORKSPACE_NAME)) + python_paths = [get_windows_path_with_unc_prefix(path) for path in python_paths] + existing_pythonpath = os.environ.get("PYTHONPATH") + if existing_pythonpath: + python_paths.extend(existing_pythonpath.split(os.pathsep)) + new_env["PYTHONPATH"] = os.pathsep.join(dict.fromkeys(python_paths)) + if IS_WINDOWS: + new_env["PYTHONPATH"] = new_env["PYTHONPATH"].replace("/", os.sep) + print_verbose("native fallback PYTHONPATH:", new_env["PYTHONPATH"]) + if os.environ.get("RULES_PYTHON_TESTING_TELL_RUNFILES_ROOT"): new_env["RULES_PYTHON_TESTING_RUNFILES_ROOT"] = runfiles_root diff --git a/python/private/stage1_bootstrap_template.sh b/python/private/stage1_bootstrap_template.sh index 4374ff95b0..4a26444c1b 100644 --- a/python/private/stage1_bootstrap_template.sh +++ b/python/private/stage1_bootstrap_template.sh @@ -33,6 +33,24 @@ function _symlink() { # runfiles-root-relative path STAGE2_BOOTSTRAP="%stage2_bootstrap%" +# Like the Python bootstrap, fall back to the main file for native rules, +# which expand %main% but leave newer stage-2 placeholders unchanged. +_STAGE2_BOOTSTRAP_SENTINEL="%stage2""_bootstrap%" +_INTERPRETER_ARGS_SENTINEL="%interpreter""_args%" +USING_STAGE2_BOOTSTRAP_FALLBACK=0 +if [[ "$STAGE2_BOOTSTRAP" == "$_STAGE2_BOOTSTRAP_SENTINEL" ]]; then + _MAIN_SENTINEL="%main""%" + main="%main%" + if [[ "$main" == "$_MAIN_SENTINEL" || -z "$main" ]]; then + echo >&2 "ERROR: %stage2_bootstrap% (or %main%) was not substituted." + exit 1 + fi + # Native external targets use workspace/../repo/main.py. The workspace + # directory need not exist, so remove that prefix before accessing the file. + STAGE2_BOOTSTRAP="${main#"%workspace_name%/../"}" + USING_STAGE2_BOOTSTRAP_FALLBACK=1 +fi + # runfiles-root-relative path to python interpreter to use. # This is the `bin/python3` path in the binary's venv. PYTHON_BINARY='%python_binary%' @@ -60,6 +78,11 @@ declare -a INTERPRETER_ARGS_FROM_TARGET=( %interpreter_args% ) +if [[ "$USING_STAGE2_BOOTSTRAP_FALLBACK" == "1" && + "${INTERPRETER_ARGS_FROM_TARGET[*]}" == "$_INTERPRETER_ARGS_SENTINEL" ]]; then + INTERPRETER_ARGS_FROM_TARGET=() +fi + if [[ "$IS_ZIPFILE" == "1" ]]; then # NOTE: Macs have an old version of mktemp, so we must use only the # minimal functionality of it. @@ -309,6 +332,33 @@ fi export RUNFILES_DIR +if [[ "$USING_STAGE2_BOOTSTRAP_FALLBACK" == "1" ]]; then + # Restore native import order when there is no stage 2 to set up sys.path. + # Declared imports are used as provided, without workspace-specific paths. + declare -a python_paths=("$RUNFILES_DIR") + imports="%imports%" + IFS=: read -r -a import_paths <<< "$imports" + for import_path in "${import_paths[@]}"; do + python_paths+=("$RUNFILES_DIR/$import_path") + done + + if [[ "%import_all%" == "True" ]]; then + for repo_root in "$RUNFILES_DIR"/*; do + if [[ -d "$repo_root" ]]; then + python_paths+=("$repo_root") + fi + done + else + python_paths+=("$RUNFILES_DIR/%workspace_name%") + fi + + python_path="$(IFS=:; echo "${python_paths[*]}")" + if [[ -n "${PYTHONPATH:-}" ]]; then + python_path+=":$PYTHONPATH" + fi + interpreter_env+=("PYTHONPATH=$python_path") +fi + if command -v env >/dev/null 2>&1; then ENV_CMD="env" else diff --git a/tests/bootstrap_impls/BUILD.bazel b/tests/bootstrap_impls/BUILD.bazel index 2c4eae8d21..1ef70702bf 100644 --- a/tests/bootstrap_impls/BUILD.bazel +++ b/tests/bootstrap_impls/BUILD.bazel @@ -20,6 +20,23 @@ load("//tests/support:support.bzl", "SUPPORTS_BOOTSTRAP_SCRIPT") load("//tests/support/pytest_test:pytest_test.bzl", "pytest_test") load(":venv_relative_path_tests.bzl", "relative_path_test_suite") +py_test( + name = "native_bootstrap_test", + srcs = ["native_bootstrap_test.py"], + data = ["//python/private:python_bootstrap_template.txt"], + deps = ["//python/runfiles"], +) + +py_test( + name = "native_bootstrap_script_test", + srcs = ["native_bootstrap_test.py"], + data = ["//python/private:stage1_bootstrap_template"], + env = {"BOOTSTRAP": "script"}, + main = "native_bootstrap_test.py", + target_compatible_with = SUPPORTS_BOOTSTRAP_SCRIPT, + deps = ["//python/runfiles"], +) + py_reconfig_binary( name = "bootstrap_script_zipapp_bin", srcs = ["bin.py"], diff --git a/tests/bootstrap_impls/native_bootstrap_test.py b/tests/bootstrap_impls/native_bootstrap_test.py new file mode 100644 index 0000000000..1a77d5f63f --- /dev/null +++ b/tests/bootstrap_impls/native_bootstrap_test.py @@ -0,0 +1,242 @@ +"""Tests both bootstraps with Bazel 7's native template substitutions. + +Render only the placeholders understood by native rules, leaving stage-2 and +venv placeholders untouched. This tests the fallback without requiring native +Python rules, which are unavailable in newer Bazel versions. +""" + +from __future__ import annotations + +import json +import os +import pathlib +import posixpath +import subprocess +import sys +import tempfile +import unittest + +from python.runfiles import runfiles + + +class NativeBootstrapTest(unittest.TestCase): + def setUp(self): + self.script_bootstrap = os.environ.get("BOOTSTRAP") == "script" + prefix = "native_bootstrap_" if self.script_bootstrap else "native bootstrap " + self.temp_dir = tempfile.TemporaryDirectory(prefix=prefix) + self.addCleanup(self.temp_dir.cleanup) + filename = "launcher.sh" if self.script_bootstrap else "launcher.py" + self.launcher = pathlib.Path(self.temp_dir.name) / filename + self.runfiles_root = pathlib.Path(str(self.launcher) + ".runfiles") + self.runfiles_root.mkdir() + template_name = ( + "stage1_bootstrap_template.sh" + if self.script_bootstrap + else "python_bootstrap_template.txt" + ) + self.template = ( + runfiles.CreateOrRaise().root() + / f"rules_python/python/private/{template_name}" + ).read_text(encoding="utf-8") + + def _write_runfile(self, path: str, contents: str) -> None: + file = self.runfiles_root / posixpath.normpath(path) + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(contents, encoding="utf-8") + + def _render(self, substitutions: dict[str, str] | None = None) -> None: + # This is the native Bazel 7 substitution contract, not a patched + # launcher. Unrecognized placeholders must remain in the template. + native_substitutions = { + "%shebang%": "#!/usr/bin/env python3", + "%main%": "_main/tool/main.py", + "%python_binary%": sys.executable.replace("\\", "/"), + "%imports%": "_main/lib", + "%import_all%": "False", + "%is_zipfile%": "False", + "%workspace_name%": "_main", + } + native_substitutions.update(substitutions or {}) + source = self.template + for placeholder, replacement in native_substitutions.items(): + source = source.replace(placeholder, replacement) + self.launcher.write_text(source, encoding="utf-8") + + def _run( + self, + substitutions: dict[str, str] | None = None, + extra_env: dict[str, str] | None = None, + *, + load_only: bool = False, + ) -> subprocess.CompletedProcess[str]: + self._render(substitutions) + env = dict(os.environ) + for key in ( + "RUNFILES_DIR", + "RUNFILES_MANIFEST_FILE", + "PYTHONPATH", + "PYTHONSAFEPATH", + ): + env.pop(key, None) + env.update(extra_env or {}) + if load_only: + command = [ + sys.executable, + "-c", + "import json, runpy, sys; " + "print(json.dumps(runpy.run_path(sys.argv[1])['RUNTIME_VENV_SYMLINKS']))", + str(self.launcher), + ] + elif self.script_bootstrap: + command = ["bash", str(self.launcher)] + else: + command = [sys.executable, str(self.launcher)] + return subprocess.run( + command, env=env, capture_output=True, text=True, timeout=30, check=False + ) + + def _assert_success( + self, result: subprocess.CompletedProcess[str], output: str + ) -> None: + self.assertEqual( + result.returncode, + 0, + "==================== STDOUT BEGIN ====================\n" + + result.stdout + + "==================== STDOUT END ====================\n" + + "==================== STDERR BEGIN ====================\n" + + result.stderr + + "==================== STDERR END ====================", + ) + self.assertEqual(result.stdout.strip(), output) + + def test_native_main_with_unexpanded_placeholders(self): + self._write_runfile("_main/tool/main.py", 'print("reached main")\n') + self._assert_success(self._run(), "reached main") + + def test_declared_import_roots_preserve_order(self): + self._write_runfile( + "_main/tool/main.py", + "from native_declared_fixture import MESSAGE\nprint(MESSAGE)\n", + ) + self._write_runfile( + "_main/lib/native_declared_fixture.py", 'MESSAGE = "first"\n' + ) + self._write_runfile( + "_main/second/native_declared_fixture.py", 'MESSAGE = "second"\n' + ) + result = self._run({"%imports%": "_main/lib:_main/second:_main/lib"}) + self._assert_success(result, "first") + + def test_workspace_root_import(self): + self._write_runfile( + "_main/tool/main.py", + "from native_workspace_fixture import MESSAGE\nprint(MESSAGE)\n", + ) + self._write_runfile( + "_main/native_workspace_fixture.py", 'MESSAGE = "workspace"\n' + ) + self._assert_success(self._run({"%imports%": ""}), "workspace") + + def test_external_main_with_import_all(self): + self._write_runfile( + "external_repo/tool/main.py", + "from native_external_fixture import MESSAGE\nprint(MESSAGE)\n", + ) + self._write_runfile( + "external_repo/native_external_fixture.py", 'MESSAGE = "external"\n' + ) + result = self._run( + { + "%main%": "_main/../external_repo/tool/main.py", + "%imports%": "", + "%import_all%": "True", + } + ) + self._assert_success(result, "external") + + def test_import_all_false_does_not_add_other_repositories(self): + self._write_runfile( + "_main/tool/main.py", + "import importlib.util\n" + "assert importlib.util.find_spec('native_other_fixture') is None\n" + "print('not imported')\n", + ) + self._write_runfile("other_repo/native_other_fixture.py", 'MESSAGE = "other"\n') + self._assert_success(self._run({"%imports%": ""}), "not imported") + + def test_existing_pythonpath_is_preserved_after_declared_imports(self): + self._write_runfile( + "_main/tool/main.py", + "from native_declared_fixture import MESSAGE\n" + "from native_environment_fixture import EXTRA\n" + "print(MESSAGE + EXTRA)\n", + ) + self._write_runfile( + "_main/lib/native_declared_fixture.py", 'MESSAGE = "declared"\n' + ) + inherited = pathlib.Path(self.temp_dir.name) / "inherited" + inherited.mkdir() + (inherited / "native_declared_fixture.py").write_text('MESSAGE = "inherited"\n') + (inherited / "native_environment_fixture.py").write_text( + 'EXTRA = "+environment"\n' + ) + result = self._run(extra_env={"PYTHONPATH": str(inherited)}) + self._assert_success(result, "declared+environment") + + def test_stage2_does_not_export_native_import_roots(self): + self._write_runfile( + "_main/stage2.py", "import os\nprint(os.environ.get('PYTHONPATH'))\n" + ) + result = self._run( + { + "%stage2_bootstrap%": "_main/stage2.py", + "%runtime_venv_symlinks%": "", + "%interpreter_args%": "", + }, + extra_env={"PYTHONPATH": "inherited-path"}, + ) + self._assert_success(result, "inherited-path") + + def test_stage2_preserves_substituted_interpreter_args(self): + self._write_runfile( + "_main/stage2.py", "import sys\nprint(sys._xoptions['BOOTSTRAP_TEST'])\n" + ) + result = self._run( + { + "%stage2_bootstrap%": "_main/stage2.py", + "%runtime_venv_symlinks%": "", + "%interpreter_args%": "-XBOOTSTRAP_TEST=preserved", + } + ) + self._assert_success(result, "preserved") + + def test_missing_stage2_and_main_report_error(self): + result = self._run({"%main%": ""}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("was not substituted", result.stderr) + + @unittest.skipIf(os.environ.get("BOOTSTRAP") == "script", "Python-only venv parser") + def test_substituted_venv_symlinks_are_preserved(self): + result = self._run( + { + "%stage2_bootstrap%": "_main/stage2.py", + "%runtime_venv_symlinks%": "lib/site-packages|_main/lib\nbin/helper|tools/helper", + }, + load_only=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + json.loads(result.stdout), + {"lib/site-packages": "_main/lib", "bin/helper": "tools/helper"}, + ) + + @unittest.skipIf(os.environ.get("BOOTSTRAP") == "script", "Python-only venv parser") + def test_malformed_substituted_symlinks_still_fail(self): + result = self._run({"%runtime_venv_symlinks%": "malformed"}, load_only=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("ValueError", result.stderr) + + +if __name__ == "__main__": + unittest.main() From 47aec0170f562e66d378e591ce6c87a26e5b2184 Mon Sep 17 00:00:00 2001 From: Gleb Kolobkov Date: Fri, 9 Oct 2026 04:44:56 +0000 Subject: [PATCH 2/2] test(bootstrap): make native bootstrap tests portable on Windows Use the Windows .exe.runfiles suffix when creating the launcher fixture. Build expected venv symlink paths with the platform path separator. This fixes the four blocking Windows jobs in PR #4235 without changing launcher behavior. --- tests/bootstrap_impls/native_bootstrap_test.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/bootstrap_impls/native_bootstrap_test.py b/tests/bootstrap_impls/native_bootstrap_test.py index 1a77d5f63f..c77939fac4 100644 --- a/tests/bootstrap_impls/native_bootstrap_test.py +++ b/tests/bootstrap_impls/native_bootstrap_test.py @@ -27,7 +27,9 @@ def setUp(self): self.addCleanup(self.temp_dir.cleanup) filename = "launcher.sh" if self.script_bootstrap else "launcher.py" self.launcher = pathlib.Path(self.temp_dir.name) / filename - self.runfiles_root = pathlib.Path(str(self.launcher) + ".runfiles") + # The Windows bootstrap looks beside the launcher for .exe.runfiles. + runfiles_suffix = ".exe.runfiles" if os.name == "nt" else ".runfiles" + self.runfiles_root = pathlib.Path(str(self.launcher) + runfiles_suffix) self.runfiles_root.mkdir() template_name = ( "stage1_bootstrap_template.sh" @@ -228,7 +230,10 @@ def test_substituted_venv_symlinks_are_preserved(self): self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual( json.loads(result.stdout), - {"lib/site-packages": "_main/lib", "bin/helper": "tools/helper"}, + { + os.path.join("lib", "site-packages"): os.path.join("_main", "lib"), + os.path.join("bin", "helper"): os.path.join("tools", "helper"), + }, ) @unittest.skipIf(os.environ.get("BOOTSTRAP") == "script", "Python-only venv parser")