diff --git a/news/native-bootstrap-imports.fixed.md b/news/native-bootstrap-imports.fixed.md new file mode 100644 index 0000000000..516aa18c05 --- /dev/null +++ b/news/native-bootstrap-imports.fixed.md @@ -0,0 +1,4 @@ +(bootstrap) Bazel 7 native Python targets can run with both the Python and +shell bootstraps without crashing on unexpanded launcher placeholders. Their +declared import roots and repository imports are available when the launcher +falls back to executing the main file directly. diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index 482918c038..29b1c2a392 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -39,6 +39,7 @@ import subprocess _STAGE2_BOOTSTRAP_SENTINEL = "%stage2" + "_bootstrap%" # runfiles-root-relative path STAGE2_BOOTSTRAP="%stage2_bootstrap%" +_USING_STAGE2_BOOTSTRAP_FALLBACK = False # NOTE: The fallback logic from stage2_bootstrap to main is only present # as a courtesy for an older, unsupported, configuration. It can be removed @@ -49,6 +50,7 @@ if STAGE2_BOOTSTRAP == _STAGE2_BOOTSTRAP_SENTINEL: _main = "%main%" if _main != _MAIN_SENTINEL and _main: STAGE2_BOOTSTRAP = _main + _USING_STAGE2_BOOTSTRAP_FALLBACK = True else: STAGE2_BOOTSTRAP = "" @@ -98,7 +100,13 @@ else: RUNTIME_VENV_SYMLINKS = """ %runtime_venv_symlinks% """.strip().split("\n") -RUNTIME_VENV_SYMLINKS = dict(line.split("|") for line in RUNTIME_VENV_SYMLINKS if line) +# Native rules leave this newer placeholder unchanged. They have no runtime +# venv symlinks; only parse entries when the placeholder was expanded. +_RUNTIME_VENV_SYMLINKS_SENTINEL = "%runtime_venv" + "_symlinks%" +if RUNTIME_VENV_SYMLINKS == [_RUNTIME_VENV_SYMLINKS_SENTINEL]: + RUNTIME_VENV_SYMLINKS = {} +else: + RUNTIME_VENV_SYMLINKS = dict(line.split("|") for line in RUNTIME_VENV_SYMLINKS if line) ADDITIONAL_INTERPRETER_ARGS = os.environ.get("RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS", "") EXTRACT_ROOT = os.environ.get("RULES_PYTHON_EXTRACT_ROOT") @@ -614,6 +622,29 @@ def main(): print_verbose("runfiles root:", runfiles_root) + if _USING_STAGE2_BOOTSTRAP_FALLBACK: + # Native rules expand %imports% and %import_all%, but do not generate + # stage 2 to set up sys.path. Preserve the native launcher's import order + # when executing the main file directly; leave regular stage-2 launches + # unchanged. These paths are runfiles-relative, not checkout-relative. + imports = "%imports%" + python_paths = [runfiles_root] + python_paths.extend(join(runfiles_root, path) for path in imports.split(":") if path) + if "%import_all%" == "True": + python_paths.extend( + join(runfiles_root, repo) for repo in sorted(os.listdir(runfiles_root)) + if os.path.isdir(join(runfiles_root, repo))) + else: + python_paths.append(join(runfiles_root, WORKSPACE_NAME)) + python_paths = [get_windows_path_with_unc_prefix(path) for path in python_paths] + existing_pythonpath = os.environ.get("PYTHONPATH") + if existing_pythonpath: + python_paths.extend(existing_pythonpath.split(os.pathsep)) + new_env["PYTHONPATH"] = os.pathsep.join(dict.fromkeys(python_paths)) + if IS_WINDOWS: + new_env["PYTHONPATH"] = new_env["PYTHONPATH"].replace("/", os.sep) + print_verbose("native fallback PYTHONPATH:", new_env["PYTHONPATH"]) + if os.environ.get("RULES_PYTHON_TESTING_TELL_RUNFILES_ROOT"): new_env["RULES_PYTHON_TESTING_RUNFILES_ROOT"] = runfiles_root diff --git a/python/private/stage1_bootstrap_template.sh b/python/private/stage1_bootstrap_template.sh index 4374ff95b0..4a26444c1b 100644 --- a/python/private/stage1_bootstrap_template.sh +++ b/python/private/stage1_bootstrap_template.sh @@ -33,6 +33,24 @@ function _symlink() { # runfiles-root-relative path STAGE2_BOOTSTRAP="%stage2_bootstrap%" +# Like the Python bootstrap, fall back to the main file for native rules, +# which expand %main% but leave newer stage-2 placeholders unchanged. +_STAGE2_BOOTSTRAP_SENTINEL="%stage2""_bootstrap%" +_INTERPRETER_ARGS_SENTINEL="%interpreter""_args%" +USING_STAGE2_BOOTSTRAP_FALLBACK=0 +if [[ "$STAGE2_BOOTSTRAP" == "$_STAGE2_BOOTSTRAP_SENTINEL" ]]; then + _MAIN_SENTINEL="%main""%" + main="%main%" + if [[ "$main" == "$_MAIN_SENTINEL" || -z "$main" ]]; then + echo >&2 "ERROR: %stage2_bootstrap% (or %main%) was not substituted." + exit 1 + fi + # Native external targets use workspace/../repo/main.py. The workspace + # directory need not exist, so remove that prefix before accessing the file. + STAGE2_BOOTSTRAP="${main#"%workspace_name%/../"}" + USING_STAGE2_BOOTSTRAP_FALLBACK=1 +fi + # runfiles-root-relative path to python interpreter to use. # This is the `bin/python3` path in the binary's venv. PYTHON_BINARY='%python_binary%' @@ -60,6 +78,11 @@ declare -a INTERPRETER_ARGS_FROM_TARGET=( %interpreter_args% ) +if [[ "$USING_STAGE2_BOOTSTRAP_FALLBACK" == "1" && + "${INTERPRETER_ARGS_FROM_TARGET[*]}" == "$_INTERPRETER_ARGS_SENTINEL" ]]; then + INTERPRETER_ARGS_FROM_TARGET=() +fi + if [[ "$IS_ZIPFILE" == "1" ]]; then # NOTE: Macs have an old version of mktemp, so we must use only the # minimal functionality of it. @@ -309,6 +332,33 @@ fi export RUNFILES_DIR +if [[ "$USING_STAGE2_BOOTSTRAP_FALLBACK" == "1" ]]; then + # Restore native import order when there is no stage 2 to set up sys.path. + # Declared imports are used as provided, without workspace-specific paths. + declare -a python_paths=("$RUNFILES_DIR") + imports="%imports%" + IFS=: read -r -a import_paths <<< "$imports" + for import_path in "${import_paths[@]}"; do + python_paths+=("$RUNFILES_DIR/$import_path") + done + + if [[ "%import_all%" == "True" ]]; then + for repo_root in "$RUNFILES_DIR"/*; do + if [[ -d "$repo_root" ]]; then + python_paths+=("$repo_root") + fi + done + else + python_paths+=("$RUNFILES_DIR/%workspace_name%") + fi + + python_path="$(IFS=:; echo "${python_paths[*]}")" + if [[ -n "${PYTHONPATH:-}" ]]; then + python_path+=":$PYTHONPATH" + fi + interpreter_env+=("PYTHONPATH=$python_path") +fi + if command -v env >/dev/null 2>&1; then ENV_CMD="env" else diff --git a/tests/bootstrap_impls/BUILD.bazel b/tests/bootstrap_impls/BUILD.bazel index 2c4eae8d21..1ef70702bf 100644 --- a/tests/bootstrap_impls/BUILD.bazel +++ b/tests/bootstrap_impls/BUILD.bazel @@ -20,6 +20,23 @@ load("//tests/support:support.bzl", "SUPPORTS_BOOTSTRAP_SCRIPT") load("//tests/support/pytest_test:pytest_test.bzl", "pytest_test") load(":venv_relative_path_tests.bzl", "relative_path_test_suite") +py_test( + name = "native_bootstrap_test", + srcs = ["native_bootstrap_test.py"], + data = ["//python/private:python_bootstrap_template.txt"], + deps = ["//python/runfiles"], +) + +py_test( + name = "native_bootstrap_script_test", + srcs = ["native_bootstrap_test.py"], + data = ["//python/private:stage1_bootstrap_template"], + env = {"BOOTSTRAP": "script"}, + main = "native_bootstrap_test.py", + target_compatible_with = SUPPORTS_BOOTSTRAP_SCRIPT, + deps = ["//python/runfiles"], +) + py_reconfig_binary( name = "bootstrap_script_zipapp_bin", srcs = ["bin.py"], diff --git a/tests/bootstrap_impls/native_bootstrap_test.py b/tests/bootstrap_impls/native_bootstrap_test.py new file mode 100644 index 0000000000..c77939fac4 --- /dev/null +++ b/tests/bootstrap_impls/native_bootstrap_test.py @@ -0,0 +1,247 @@ +"""Tests both bootstraps with Bazel 7's native template substitutions. + +Render only the placeholders understood by native rules, leaving stage-2 and +venv placeholders untouched. This tests the fallback without requiring native +Python rules, which are unavailable in newer Bazel versions. +""" + +from __future__ import annotations + +import json +import os +import pathlib +import posixpath +import subprocess +import sys +import tempfile +import unittest + +from python.runfiles import runfiles + + +class NativeBootstrapTest(unittest.TestCase): + def setUp(self): + self.script_bootstrap = os.environ.get("BOOTSTRAP") == "script" + prefix = "native_bootstrap_" if self.script_bootstrap else "native bootstrap " + self.temp_dir = tempfile.TemporaryDirectory(prefix=prefix) + self.addCleanup(self.temp_dir.cleanup) + filename = "launcher.sh" if self.script_bootstrap else "launcher.py" + self.launcher = pathlib.Path(self.temp_dir.name) / filename + # The Windows bootstrap looks beside the launcher for .exe.runfiles. + runfiles_suffix = ".exe.runfiles" if os.name == "nt" else ".runfiles" + self.runfiles_root = pathlib.Path(str(self.launcher) + runfiles_suffix) + self.runfiles_root.mkdir() + template_name = ( + "stage1_bootstrap_template.sh" + if self.script_bootstrap + else "python_bootstrap_template.txt" + ) + self.template = ( + runfiles.CreateOrRaise().root() + / f"rules_python/python/private/{template_name}" + ).read_text(encoding="utf-8") + + def _write_runfile(self, path: str, contents: str) -> None: + file = self.runfiles_root / posixpath.normpath(path) + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(contents, encoding="utf-8") + + def _render(self, substitutions: dict[str, str] | None = None) -> None: + # This is the native Bazel 7 substitution contract, not a patched + # launcher. Unrecognized placeholders must remain in the template. + native_substitutions = { + "%shebang%": "#!/usr/bin/env python3", + "%main%": "_main/tool/main.py", + "%python_binary%": sys.executable.replace("\\", "/"), + "%imports%": "_main/lib", + "%import_all%": "False", + "%is_zipfile%": "False", + "%workspace_name%": "_main", + } + native_substitutions.update(substitutions or {}) + source = self.template + for placeholder, replacement in native_substitutions.items(): + source = source.replace(placeholder, replacement) + self.launcher.write_text(source, encoding="utf-8") + + def _run( + self, + substitutions: dict[str, str] | None = None, + extra_env: dict[str, str] | None = None, + *, + load_only: bool = False, + ) -> subprocess.CompletedProcess[str]: + self._render(substitutions) + env = dict(os.environ) + for key in ( + "RUNFILES_DIR", + "RUNFILES_MANIFEST_FILE", + "PYTHONPATH", + "PYTHONSAFEPATH", + ): + env.pop(key, None) + env.update(extra_env or {}) + if load_only: + command = [ + sys.executable, + "-c", + "import json, runpy, sys; " + "print(json.dumps(runpy.run_path(sys.argv[1])['RUNTIME_VENV_SYMLINKS']))", + str(self.launcher), + ] + elif self.script_bootstrap: + command = ["bash", str(self.launcher)] + else: + command = [sys.executable, str(self.launcher)] + return subprocess.run( + command, env=env, capture_output=True, text=True, timeout=30, check=False + ) + + def _assert_success( + self, result: subprocess.CompletedProcess[str], output: str + ) -> None: + self.assertEqual( + result.returncode, + 0, + "==================== STDOUT BEGIN ====================\n" + + result.stdout + + "==================== STDOUT END ====================\n" + + "==================== STDERR BEGIN ====================\n" + + result.stderr + + "==================== STDERR END ====================", + ) + self.assertEqual(result.stdout.strip(), output) + + def test_native_main_with_unexpanded_placeholders(self): + self._write_runfile("_main/tool/main.py", 'print("reached main")\n') + self._assert_success(self._run(), "reached main") + + def test_declared_import_roots_preserve_order(self): + self._write_runfile( + "_main/tool/main.py", + "from native_declared_fixture import MESSAGE\nprint(MESSAGE)\n", + ) + self._write_runfile( + "_main/lib/native_declared_fixture.py", 'MESSAGE = "first"\n' + ) + self._write_runfile( + "_main/second/native_declared_fixture.py", 'MESSAGE = "second"\n' + ) + result = self._run({"%imports%": "_main/lib:_main/second:_main/lib"}) + self._assert_success(result, "first") + + def test_workspace_root_import(self): + self._write_runfile( + "_main/tool/main.py", + "from native_workspace_fixture import MESSAGE\nprint(MESSAGE)\n", + ) + self._write_runfile( + "_main/native_workspace_fixture.py", 'MESSAGE = "workspace"\n' + ) + self._assert_success(self._run({"%imports%": ""}), "workspace") + + def test_external_main_with_import_all(self): + self._write_runfile( + "external_repo/tool/main.py", + "from native_external_fixture import MESSAGE\nprint(MESSAGE)\n", + ) + self._write_runfile( + "external_repo/native_external_fixture.py", 'MESSAGE = "external"\n' + ) + result = self._run( + { + "%main%": "_main/../external_repo/tool/main.py", + "%imports%": "", + "%import_all%": "True", + } + ) + self._assert_success(result, "external") + + def test_import_all_false_does_not_add_other_repositories(self): + self._write_runfile( + "_main/tool/main.py", + "import importlib.util\n" + "assert importlib.util.find_spec('native_other_fixture') is None\n" + "print('not imported')\n", + ) + self._write_runfile("other_repo/native_other_fixture.py", 'MESSAGE = "other"\n') + self._assert_success(self._run({"%imports%": ""}), "not imported") + + def test_existing_pythonpath_is_preserved_after_declared_imports(self): + self._write_runfile( + "_main/tool/main.py", + "from native_declared_fixture import MESSAGE\n" + "from native_environment_fixture import EXTRA\n" + "print(MESSAGE + EXTRA)\n", + ) + self._write_runfile( + "_main/lib/native_declared_fixture.py", 'MESSAGE = "declared"\n' + ) + inherited = pathlib.Path(self.temp_dir.name) / "inherited" + inherited.mkdir() + (inherited / "native_declared_fixture.py").write_text('MESSAGE = "inherited"\n') + (inherited / "native_environment_fixture.py").write_text( + 'EXTRA = "+environment"\n' + ) + result = self._run(extra_env={"PYTHONPATH": str(inherited)}) + self._assert_success(result, "declared+environment") + + def test_stage2_does_not_export_native_import_roots(self): + self._write_runfile( + "_main/stage2.py", "import os\nprint(os.environ.get('PYTHONPATH'))\n" + ) + result = self._run( + { + "%stage2_bootstrap%": "_main/stage2.py", + "%runtime_venv_symlinks%": "", + "%interpreter_args%": "", + }, + extra_env={"PYTHONPATH": "inherited-path"}, + ) + self._assert_success(result, "inherited-path") + + def test_stage2_preserves_substituted_interpreter_args(self): + self._write_runfile( + "_main/stage2.py", "import sys\nprint(sys._xoptions['BOOTSTRAP_TEST'])\n" + ) + result = self._run( + { + "%stage2_bootstrap%": "_main/stage2.py", + "%runtime_venv_symlinks%": "", + "%interpreter_args%": "-XBOOTSTRAP_TEST=preserved", + } + ) + self._assert_success(result, "preserved") + + def test_missing_stage2_and_main_report_error(self): + result = self._run({"%main%": ""}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("was not substituted", result.stderr) + + @unittest.skipIf(os.environ.get("BOOTSTRAP") == "script", "Python-only venv parser") + def test_substituted_venv_symlinks_are_preserved(self): + result = self._run( + { + "%stage2_bootstrap%": "_main/stage2.py", + "%runtime_venv_symlinks%": "lib/site-packages|_main/lib\nbin/helper|tools/helper", + }, + load_only=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + json.loads(result.stdout), + { + os.path.join("lib", "site-packages"): os.path.join("_main", "lib"), + os.path.join("bin", "helper"): os.path.join("tools", "helper"), + }, + ) + + @unittest.skipIf(os.environ.get("BOOTSTRAP") == "script", "Python-only venv parser") + def test_malformed_substituted_symlinks_still_fail(self): + result = self._run({"%runtime_venv_symlinks%": "malformed"}, load_only=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("ValueError", result.stderr) + + +if __name__ == "__main__": + unittest.main()