diff --git a/.bazelrc.deleted_packages b/.bazelrc.deleted_packages index 61c79ae032..71a81de097 100644 --- a/.bazelrc.deleted_packages +++ b/.bazelrc.deleted_packages @@ -35,6 +35,7 @@ common --deleted_packages=tests/integration/custom_commands common --deleted_packages=tests/integration/local_toolchains common --deleted_packages=tests/integration/pip_parse common --deleted_packages=tests/integration/pip_parse/empty +common --deleted_packages=tests/integration/pip_parse_isolated common --deleted_packages=tests/integration/py_cc_toolchain_registered common --deleted_packages=tests/integration/toolchain_target_settings common --deleted_packages=tests/modules/another_module diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c19d1176e..2fbe0e8960 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,8 @@ END_UNRELEASED_TEMPLATE targets ([#3729](https://github.com/bazel-contrib/rules_python/issues/3729)). * (entry_point) From now on `mypy` type checking will be skipped on the generated files ([#3126](https://github.com/bazel-contrib/rules_python/issues/3126)). +* (pypi) Support `--experimental_isolated_extension_usages` + ([#3668](https://github.com/bazel-contrib/rules_python/issues/3668)). {#v0-0-0-added} ### Added diff --git a/MODULE.bazel b/MODULE.bazel index b5f67c204e..bb3a9dcab5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -64,141 +64,6 @@ register_toolchains("@pythons_hub//:all") # Install twine for our own runfiles wheel publishing and allow bzlmod users to use it. pip = use_extension("//python/extensions:pip.bzl", "pip") - -# NOTE @aignas 2025-07-06: we define these platforms to keep backwards compatibility. Whilst we -# stabilize the API this list may be updated with a mention in the CHANGELOG. -[ - pip.default( - arch_name = cpu, - config_settings = [ - "@platforms//cpu:{}".format(cpu), - "@platforms//os:linux", - "//python/config_settings:_is_py_freethreaded_{}".format( - "yes" if freethreaded else "no", - ), - ], - env = {"platform_version": "0"}, - marker = "python_version >= '3.13'" if freethreaded else "", - os_name = "linux", - platform = "linux_{}{}".format(cpu, freethreaded), - whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [ - "abi3", - "cp{major}{minor}", - ], - whl_platform_tags = [ - "linux_{}".format(cpu), - "manylinux_*_{}".format(cpu), - ], - ) - for cpu in [ - "x86_64", - "aarch64", - ] - for freethreaded in [ - "", - "_freethreaded", - ] -] - -[ - pip.default( - arch_name = cpu, - config_settings = [ - "@platforms//cpu:{}".format(cpu), - "@platforms//os:osx", - "//python/config_settings:_is_py_freethreaded_{}".format( - "yes" if freethreaded else "no", - ), - ], - # We choose the oldest non-EOL version at the time when we release `rules_python`. - # See https://endoflife.date/macos - env = {"platform_version": "14.0"}, - marker = "python_version >= '3.13'" if freethreaded else "", - os_name = "osx", - platform = "osx_{}{}".format(cpu, freethreaded), - whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [ - "abi3", - "cp{major}{minor}", - ], - whl_platform_tags = [ - "macosx_*_{}".format(suffix) - for suffix in platform_tag_cpus - ], - ) - for cpu, platform_tag_cpus in { - "aarch64": [ - "universal2", - "arm64", - ], - "x86_64": [ - "universal2", - "x86_64", - ], - }.items() - for freethreaded in [ - "", - "_freethreaded", - ] -] - -[ - pip.default( - arch_name = cpu, - config_settings = [ - "@platforms//cpu:{}".format(cpu), - "@platforms//os:windows", - "//python/config_settings:_is_py_freethreaded_{}".format( - "yes" if freethreaded else "no", - ), - ], - env = {"platform_version": "0"}, - marker = "python_version >= '3.13'" if freethreaded else "", - os_name = "windows", - platform = "windows_{}{}".format(cpu, freethreaded), - whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [ - "abi3", - "cp{major}{minor}", - ], - whl_platform_tags = whl_platform_tags, - ) - for cpu, whl_platform_tags in { - "x86_64": ["win_amd64"], - }.items() - for freethreaded in [ - "", - "_freethreaded", - ] -] - -[ - pip.default( - arch_name = cpu, - config_settings = [ - "@platforms//cpu:{}".format(cpu), - "@platforms//os:windows", - "//python/config_settings:_is_py_freethreaded_{}".format( - "yes" if freethreaded else "no", - ), - ], - env = {"platform_version": "0"}, - marker = "python_version >= '3.13'" if freethreaded else "python_version >= '3.11'", - os_name = "windows", - platform = "windows_{}{}".format(cpu, freethreaded), - whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [ - "abi3", - "cp{major}{minor}", - ], - whl_platform_tags = whl_platform_tags, - ) - for cpu, whl_platform_tags in { - "aarch64": ["win_arm64"], - }.items() - for freethreaded in [ - "", - "_freethreaded", - ] -] - pip.parse( hub_name = "rules_python_publish_deps", python_version = "3.11", diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 78e93b7edd..e6052782fa 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -55,6 +55,128 @@ def _whl_mods_impl(whl_mods_dict): whl_mods = whl_mods, ) +def default_platforms(): + """Return the built-in default platform definitions. + + These provide the platform metadata needed for pip wheel resolution + (whl_abi_tags, whl_platform_tags, config_settings, etc.) across all + common OS/arch combinations. They are always used as the starting point + for build_config; root modules can override individual platforms via + pip.default tags. + + Returns: + A dict of platform name to platform config dicts. + """ + # NOTE @aignas 2025-07-06: we define these platforms to keep backwards compatibility. Whilst we + # stabilize the API this list may be updated with a mention in the CHANGELOG. + + platforms = {} + + # Linux platforms + for cpu in ["x86_64", "aarch64"]: + for freethreaded in ["", "_freethreaded"]: + platform_name = "linux_{}{}".format(cpu, freethreaded) + platforms[platform_name] = { + "arch_name": cpu, + "config_settings": [ + "@platforms//cpu:{}".format(cpu), + "@platforms//os:linux", + "//python/config_settings:_is_py_freethreaded_{}".format( + "yes" if freethreaded else "no", + ), + ], + "env": {"platform_version": "0"}, + "marker": "python_version >= '3.13'" if freethreaded else "", + "name": platform_name, + "os_name": "linux", + "whl_abi_tags": ["cp{major}{minor}t"] if freethreaded else [ + "abi3", + "cp{major}{minor}", + ], + "whl_platform_tags": [ + "linux_{}".format(cpu), + "manylinux_*_{}".format(cpu), + ], + } + + # macOS platforms + for cpu, platform_tag_cpus in { + "aarch64": ["universal2", "arm64"], + "x86_64": ["universal2", "x86_64"], + }.items(): + for freethreaded in ["", "_freethreaded"]: + platform_name = "osx_{}{}".format(cpu, freethreaded) + platforms[platform_name] = { + "arch_name": cpu, + "config_settings": [ + "@platforms//cpu:{}".format(cpu), + "@platforms//os:osx", + "//python/config_settings:_is_py_freethreaded_{}".format( + "yes" if freethreaded else "no", + ), + ], + "env": {"platform_version": "14.0"}, + "marker": "python_version >= '3.13'" if freethreaded else "", + "name": platform_name, + "os_name": "osx", + "whl_abi_tags": ["cp{major}{minor}t"] if freethreaded else [ + "abi3", + "cp{major}{minor}", + ], + "whl_platform_tags": [ + "macosx_*_{}".format(suffix) + for suffix in platform_tag_cpus + ], + } + + # Windows x86_64 platforms + for freethreaded in ["", "_freethreaded"]: + platform_name = "windows_x86_64{}".format(freethreaded) + platforms[platform_name] = { + "arch_name": "x86_64", + "config_settings": [ + "@platforms//cpu:x86_64", + "@platforms//os:windows", + "//python/config_settings:_is_py_freethreaded_{}".format( + "yes" if freethreaded else "no", + ), + ], + "env": {"platform_version": "0"}, + "marker": "python_version >= '3.13'" if freethreaded else "", + "name": platform_name, + "os_name": "windows", + "whl_abi_tags": ["cp{major}{minor}t"] if freethreaded else [ + "abi3", + "cp{major}{minor}", + ], + "whl_platform_tags": ["win_amd64"], + } + + # Windows aarch64 platforms + for freethreaded in ["", "_freethreaded"]: + platform_name = "windows_aarch64{}".format(freethreaded) + platforms[platform_name] = { + "arch_name": "aarch64", + "config_settings": [ + "@platforms//cpu:aarch64", + "@platforms//os:windows", + "//python/config_settings:_is_py_freethreaded_{}".format( + "yes" if freethreaded else "no", + ), + ], + "env": {"platform_version": "0"}, + "marker": "python_version >= '3.13'" if freethreaded else "python_version >= '3.11'", + "name": platform_name, + "os_name": "windows", + "whl_abi_tags": ["cp{major}{minor}t"] if freethreaded else [ + "abi3", + "cp{major}{minor}", + ], + "whl_platform_tags": ["win_arm64"], + } + + return platforms + def _configure(config, *, override = False, **kwargs): """Set the value in the config if the value is provided""" env = kwargs.get("env") @@ -82,7 +204,7 @@ def build_config( A struct with the configuration. """ defaults = { - "platforms": {}, + "platforms": default_platforms(), } for mod in module_ctx.modules: if not (mod.is_root or mod.name == "rules_python"): diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index 33ef907af8..5f2d20c103 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -67,6 +67,10 @@ rules_python_integration_test( name = "pip_parse_test", ) +rules_python_integration_test( + name = "pip_parse_isolated_test", +) + rules_python_integration_test( name = "pip_parse_workspace_test", bzlmod = False, diff --git a/tests/integration/pip_parse_isolated/.bazelrc b/tests/integration/pip_parse_isolated/.bazelrc new file mode 100644 index 0000000000..227ce5a4cd --- /dev/null +++ b/tests/integration/pip_parse_isolated/.bazelrc @@ -0,0 +1,8 @@ +# Bazel configuration flags + +build --enable_runfiles + +common --experimental_isolated_extension_usages + +# https://docs.bazel.build/versions/main/best-practices.html#using-the-bazelrc-file +try-import %workspace%/user.bazelrc diff --git a/tests/integration/pip_parse_isolated/BUILD.bazel b/tests/integration/pip_parse_isolated/BUILD.bazel new file mode 100644 index 0000000000..2f825107f1 --- /dev/null +++ b/tests/integration/pip_parse_isolated/BUILD.bazel @@ -0,0 +1,7 @@ +load("@rules_python//python:py_test.bzl", "py_test") + +py_test( + name = "test_isolated", + srcs = ["test_isolated.py"], + deps = ["@pypi//six"], +) diff --git a/tests/integration/pip_parse_isolated/MODULE.bazel b/tests/integration/pip_parse_isolated/MODULE.bazel new file mode 100644 index 0000000000..6c44257acb --- /dev/null +++ b/tests/integration/pip_parse_isolated/MODULE.bazel @@ -0,0 +1,19 @@ +module(name = "pip_parse_isolated") + +bazel_dep(name = "rules_python") +local_path_override( + module_name = "rules_python", + path = "../../..", +) + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain(python_version = "3.13") + +# This test module verifies that dependencies can be used with `isolate = True`. +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip", isolate = True) +pip.parse( + hub_name = "pypi", + python_version = "3.13", + requirements_lock = "//:requirements_lock.txt", +) +use_repo(pip, "pypi") diff --git a/tests/integration/pip_parse_isolated/WORKSPACE b/tests/integration/pip_parse_isolated/WORKSPACE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/pip_parse_isolated/requirements_lock.txt b/tests/integration/pip_parse_isolated/requirements_lock.txt new file mode 100644 index 0000000000..b1445a37ae --- /dev/null +++ b/tests/integration/pip_parse_isolated/requirements_lock.txt @@ -0,0 +1,2 @@ +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 diff --git a/tests/integration/pip_parse_isolated/test_isolated.py b/tests/integration/pip_parse_isolated/test_isolated.py new file mode 100644 index 0000000000..f889f071fb --- /dev/null +++ b/tests/integration/pip_parse_isolated/test_isolated.py @@ -0,0 +1,16 @@ +""" +Verify that a dependency added using an isolated extension can be imported. +See MODULE.bazel. +""" + +import six +import unittest + + +class TestIsolated(unittest.TestCase): + def test_import(self): + self.assertTrue(hasattr(six, "PY3")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index ca6da9ac1b..5a40714b64 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -16,7 +16,8 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:truth.bzl", "subjects") -load("//python/private/pypi:extension.bzl", "build_config", "parse_modules") # buildifier: disable=bzl-visibility +load("//python/private/pypi:extension.bzl", "build_config", "default_platforms", "parse_modules") # buildifier: disable=bzl-visibility +load("//python/private/pypi:platform.bzl", _plat = "platform") # buildifier: disable=bzl-visibility load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") load(":pip_parse.bzl", _parse = "pip_parse") @@ -38,59 +39,6 @@ simple==0.0.1 \ }, ) -def _mod(*, name, default = [], parse = [], override = [], whl_mods = [], is_root = True): - return mocks.module( - name, - is_root = is_root, - parse = parse, - override = override, - whl_mods = whl_mods, - default = default or [ - _default( - platform = "{}_{}{}".format(os, cpu, freethreaded), - os_name = os, - arch_name = cpu, - config_settings = [ - "@platforms//os:{}".format(os), - "@platforms//cpu:{}".format(cpu), - ], - whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else ["abi3", "cp{major}{minor}"], - whl_platform_tags = whl_platform_tags, - ) - for (os, cpu, freethreaded), whl_platform_tags in { - ("linux", "x86_64", ""): ["linux_x86_64", "manylinux_*_x86_64"], - ("linux", "x86_64", "_freethreaded"): ["linux_x86_64", "manylinux_*_x86_64"], - ("linux", "aarch64", ""): ["linux_aarch64", "manylinux_*_aarch64"], - ("osx", "aarch64", ""): ["macosx_*_arm64"], - ("windows", "aarch64", ""): ["win_arm64"], - }.items() - ], - ) - -def _parse_modules(env, **kwargs): - return env.expect.that_struct( - parse_modules( - **kwargs - ), - attrs = dict( - exposed_packages = subjects.dict, - hub_group_map = subjects.dict, - hub_whl_map = subjects.dict, - whl_libraries = subjects.dict, - whl_mods = subjects.dict, - ), - ) - -def _build_config(env, enable_pipstar_extract = True, **kwargs): - return env.expect.that_struct( - build_config(enable_pipstar_extract = enable_pipstar_extract, **kwargs), - attrs = dict( - auth_patterns = subjects.dict, - netrc = subjects.str, - platforms = subjects.dict, - ), - ) - def _default( *, arch_name = None, @@ -118,6 +66,65 @@ def _default( whl_platform_tags = whl_platform_tags or [], ) +# The default value for the default platforms tags use in `_mod`. +_default_tags_default = [ + _default( + platform = "{}_{}{}".format(os, cpu, freethreaded), + os_name = os, + arch_name = cpu, + config_settings = [ + "@platforms//os:{}".format(os), + "@platforms//cpu:{}".format(cpu), + ], + whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else ["abi3", "cp{major}{minor}"], + whl_platform_tags = whl_platform_tags, + ) + for (os, cpu, freethreaded), whl_platform_tags in { + ("linux", "x86_64", ""): ["linux_x86_64", "manylinux_*_x86_64"], + ("linux", "x86_64", "_freethreaded"): ["linux_x86_64", "manylinux_*_x86_64"], + ("linux", "aarch64", ""): ["linux_aarch64", "manylinux_*_aarch64"], + ("osx", "aarch64", ""): ["macosx_*_arm64"], + ("windows", "aarch64", ""): ["win_arm64"], + }.items() +] + +def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], is_root = True): + return struct( + name = name, + tags = struct( + parse = parse, + override = override, + whl_mods = whl_mods, + default = default, + ), + is_root = is_root, + ) + +def _parse_modules(env, **kwargs): + return env.expect.that_struct( + parse_modules(**kwargs), + attrs = dict( + exposed_packages = subjects.dict, + hub_group_map = subjects.dict, + hub_whl_map = subjects.dict, + whl_libraries = subjects.dict, + whl_mods = subjects.dict, + ), + ) + +def _build_config(env, **kwargs): + return env.expect.that_struct( + build_config( + enable_pipstar_extract = True, + **kwargs + ), + attrs = dict( + auth_patterns = subjects.dict, + netrc = subjects.str, + platforms = subjects.dict, + ), + ) + def _test_simple(env): pypi = _parse_modules( env, @@ -165,6 +172,59 @@ def _test_simple(env): _tests.append(_test_simple) +def _test_simple_isolated(env): + """Simulate `isolate = True` with parse_modules. + + No pip.default tags, but requirements parsing still produces the expected + hub output. + """ + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "my_module", + default = [], # no platform tags + parse = [ + _parse( + hub_name = "pypi", + python_version = "3.15", + simpleapi_skip = ["simple"], + requirements_lock = "requirements.txt", + ), + ], + ), + os_name = "linux", + arch_name = "x86_64", + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + + pypi.exposed_packages().contains_exactly({"pypi": ["simple"]}) + pypi.hub_group_map().contains_exactly({"pypi": {}}) + pypi.hub_whl_map().contains_exactly({"pypi": { + "simple": { + "pypi_315_simple": [ + whl_config_setting( + version = "3.15", + ), + ], + }, + }}) + pypi.whl_libraries().contains_exactly({ + "pypi_315_simple": { + "config_load": "@pypi//:config.bzl", + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf", + }, + }) + pypi.whl_mods().contains_exactly({}) + +_tests.append(_test_simple_isolated) + def _test_build_pipstar_platform(env): config = _build_config( env, @@ -216,6 +276,9 @@ def _test_build_pipstar_platform(env): whl_abi_tags = ["none", "abi3", "cp{major}{minor}"], whl_platform_tags = ["any"], ), + } | { + name: _plat(**values) + for name, values in default_platforms().items() }) _tests.append(_test_build_pipstar_platform)