From fcef7d99117748fd45ae44de66059f75fe5da27d Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Fri, 19 Jun 2026 10:07:14 -0700 Subject: [PATCH 1/2] Consolidate crate_universe generated `defs.bzl` into `crates.bzl` --- crate_universe/3rdparty/BUILD.bazel | 1 - crate_universe/defs.bzl | 11 + crate_universe/extensions.bzl | 32 +- crate_universe/private/crates_vendor.bzl | 99 ++++- crate_universe/private/srcs.bzl | 2 +- crate_universe/src/rendering.rs | 353 ++++++++++-------- .../src/rendering/template_engine.rs | 20 +- .../src/rendering/templates/defs_bzl_shim.j2 | 17 + .../src/rendering/templates/module_bzl.j2 | 10 +- .../src/rendering/templates/vendor_module.j2 | 32 -- crate_universe/src/utils/starlark.rs | 2 +- 11 files changed, 330 insertions(+), 249 deletions(-) create mode 100644 crate_universe/src/rendering/templates/defs_bzl_shim.j2 delete mode 100644 crate_universe/src/rendering/templates/vendor_module.j2 diff --git a/crate_universe/3rdparty/BUILD.bazel b/crate_universe/3rdparty/BUILD.bazel index 9da855da06..a8350de187 100644 --- a/crate_universe/3rdparty/BUILD.bazel +++ b/crate_universe/3rdparty/BUILD.bazel @@ -28,6 +28,5 @@ filegroup( name = "bzl_srcs", srcs = glob(["*.bzl"]) + [ "//crate_universe/3rdparty/crates:crates.bzl", - "//crate_universe/3rdparty/crates:defs.bzl", ], ) diff --git a/crate_universe/defs.bzl b/crate_universe/defs.bzl index 3e3e85ed20..6bbb281fee 100644 --- a/crate_universe/defs.bzl +++ b/crate_universe/defs.bzl @@ -11,11 +11,16 @@ load( load( "//crate_universe/private:crates_vendor.bzl", _crates_vendor = "crates_vendor", + _crates_vendor_remote_repository = "crates_vendor_remote_repository", ) load( "//crate_universe/private:generate_utils.bzl", _render_config = "render_config", ) +load( + "//crate_universe/private:local_crate_mirror.bzl", + _local_crate_mirror = "local_crate_mirror", +) load( "//crate_universe/private:splicing_utils.bzl", _splicing_config = "splicing_config", @@ -25,6 +30,12 @@ load( crates_repository = _crates_repository crates_vendor = _crates_vendor +# Repository rules consumed by generated `crates.bzl` files. Exposed here +# so the generated files only `load` from this public surface, insulating +# committed vendor outputs from `//crate_universe/private:...` churn. +crates_vendor_remote_repository = _crates_vendor_remote_repository +local_crate_mirror = _local_crate_mirror + # Utility Macros crate = _crate render_config = _render_config diff --git a/crate_universe/extensions.bzl b/crate_universe/extensions.bzl index bc84f51f56..919fc6dab2 100644 --- a/crate_universe/extensions.bzl +++ b/crate_universe/extensions.bzl @@ -175,9 +175,9 @@ rust_binary( ]), deps = [ # External crates - "@crates//:serde", - "@crates//:serde_json", - "@crates//:tokio", + "@crates//serde", + "@crates//serde_json", + "@crates//tokio", ], visibility = ["//visibility:public"], ) @@ -388,6 +388,7 @@ load("//crate_universe/private:crates_repository.bzl", "SUPPORTED_PLATFORM_TRIPL load( "//crate_universe/private:crates_vendor.bzl", "CRATES_VENDOR_ATTRS", + "crates_vendor_remote_repository", "generate_config_file", "generate_splicing_manifest", ) @@ -422,24 +423,6 @@ def _get_or_insert(d, key, value): d[key] = value return d[key] -def _generate_repo_impl(repository_ctx): - for path, contents in repository_ctx.attr.contents.items(): - repository_ctx.file(path, contents) - repository_ctx.file("WORKSPACE.bazel", """workspace(name = "{}")""".format( - repository_ctx.name, - )) - -_generate_repo = repository_rule( - doc = "A utility for generating a hub repo.", - implementation = _generate_repo_impl, - attrs = { - "contents": attr.string_dict( - doc = "A mapping of file names to text they should contain.", - mandatory = True, - ), - }, -) - def _annotations_for_repo(module_annotations, repo_specific_annotations): """Merges the set of global annotations with the repo-specific ones @@ -704,11 +687,12 @@ def _generate_hub_and_spokes( print("WARN: {}".format(warning)) crates_dir = tag_path.get_child(cfg.name) - _generate_repo( + crates_vendor_remote_repository( name = cfg.name, contents = { "BUILD.bazel": module_ctx.read(crates_dir.get_child("BUILD.bazel")), "alias_rules.bzl": module_ctx.read(crates_dir.get_child("alias_rules.bzl")), + "crates.bzl": module_ctx.read(crates_dir.get_child("crates.bzl")), "defs.bzl": module_ctx.read(crates_dir.get_child("defs.bzl")), }, ) @@ -1520,8 +1504,8 @@ can be found below where the supported keys for each template can be found in th default = "//:BUILD.{name}-{version}.bazel", ), "crate_alias_template": attr.string( - doc = "The base template to use for crate labels. The available format keys are [`{repository}`, `{name}`, `{version}`, `{target}`].", - default = "@{repository}//:{name}-{version}-{target}", + doc = "The base template to use for crate aliases. The available format keys are [`{repository}`, `{name}`, `{version}`, `{target}`].", + default = "//:{name}-{version}", ), "crate_label_template": attr.string( doc = "The base template to use for crate labels. The available format keys are [`{repository}`, `{name}`, `{version}`, `{target}`].", diff --git a/crate_universe/private/crates_vendor.bzl b/crate_universe/private/crates_vendor.bzl index 09b8bec83d..2e3d3f4f80 100644 --- a/crate_universe/private/crates_vendor.bzl +++ b/crate_universe/private/crates_vendor.bzl @@ -324,9 +324,7 @@ def generate_config_file( if workspace_name != "": build_file_base_template = "@{}//{}:BUILD.{{name}}-{{version}}.bazel".format(workspace_name, output_pkg) crate_label_template = render_config["crate_label_template"] - crate_alias_template = "@{{repository}}//:{{name}}-{{version}}".format( - output_pkg, - ) + crate_alias_template = render_config["crate_alias_template"] # If `workspace_name` is blank (such as when using modules), the `@{}//{}:{{file}}` template would generate # a reference like `Label(@//)`. This causes issues if the module doing the `crates_vendor`ing is not the root module. @@ -346,11 +344,15 @@ def generate_config_file( "vendor_mode": mode, } - # "crate_label_template" is explicitly supported above in non-local modes - excluded_from_key_check = ["crate_label_template", "crate_alias_template"] + excluded_from_key_check = [ + "crate_label_template", + "crate_alias_template", + ] for key in updates: - if (render_config[key] != default_render_config[key]) and key not in excluded_from_key_check: + if key in excluded_from_key_check: + continue + if render_config[key] != default_render_config[key]: if hasattr(ctx, "label"): label = ctx.label else: @@ -668,28 +670,93 @@ call against the generated workspace. The following table describes how to contr toolchains = ["@rules_rust//rust:toolchain_type"], ) +def _resolve_vendor_files(repository_ctx): + """Resolve the (BUILD.bazel, crates.bzl, defs.bzl) paths the hub mirrors. + + Supports a lean form (just `crates_module`) and a legacy form + (`build_file` + `defs_module`, optionally with `crates_module`). + Mixing forms is rejected. + + Args: + repository_ctx: The repository rule's context. + + Returns: + struct: A `struct(build_file, crates_module, defs_module)` of paths. + """ + attr = repository_ctx.attr + lean = bool(attr.crates_module) and not (attr.build_file or attr.defs_module) + legacy = bool(attr.build_file) or bool(attr.defs_module) + + if lean: + crates_module = repository_ctx.path(attr.crates_module) + vendor_dir = crates_module.dirname + return struct( + build_file = vendor_dir.get_child("BUILD.bazel"), + crates_module = crates_module, + defs_module = vendor_dir.get_child("defs.bzl"), + ) + + if legacy: + if not (attr.build_file and attr.defs_module): + fail( + "crates_vendor_remote_repository: legacy interface requires both " + + "`build_file` and `defs_module`. Prefer the lean form: pass only " + + "`crates_module = Label(\":crates.bzl\")` instead.", + ) + build_file = repository_ctx.path(attr.build_file) + if attr.crates_module: + crates_module = repository_ctx.path(attr.crates_module) + else: + crates_module = build_file.dirname.get_child("crates.bzl") + return struct( + build_file = build_file, + crates_module = crates_module, + defs_module = repository_ctx.path(attr.defs_module), + ) + + fail( + "crates_vendor_remote_repository: must set either `crates_module` (preferred) " + + "or both `build_file` and `defs_module` (legacy).", + ) + def _crates_vendor_remote_repository_impl(repository_ctx): - build_file = repository_ctx.path(repository_ctx.attr.build_file) - defs_module = repository_ctx.path(repository_ctx.attr.defs_module) + if repository_ctx.attr.contents: + contents = repository_ctx.attr.contents + else: + srcs = _resolve_vendor_files(repository_ctx) + contents = { + "BUILD.bazel": repository_ctx.read(srcs.build_file), + "crates.bzl": repository_ctx.read(srcs.crates_module), + "defs.bzl": repository_ctx.read(srcs.defs_module), + } + + for path, text in contents.items(): + repository_ctx.file(path, text) - repository_ctx.file("BUILD.bazel", repository_ctx.read(build_file)) - repository_ctx.file("defs.bzl", repository_ctx.read(defs_module)) - repository_ctx.file("crates.bzl", "") repository_ctx.file("WORKSPACE.bazel", """workspace(name = "{}")""".format( repository_ctx.name, )) crates_vendor_remote_repository = repository_rule( - doc = "Creates a repository paired with `crates_vendor` targets using the `remote` vendor mode.", + doc = ( + "Materializes the crate_universe hub repo. Accepts either a `contents` " + + "dict of `{filename: text}` (used by the bzlmod extension, whose " + + "scratch outputs aren't addressable as labels) or the vendor-side " + + "label attrs (`crates_module`, or legacy `build_file` + `defs_module`)." + ), implementation = _crates_vendor_remote_repository_impl, attrs = { "build_file": attr.label( - doc = "The BUILD file to use for the root package", - mandatory = True, + doc = "Legacy: the vendored root `BUILD.bazel`. Pair with `defs_module`.", + ), + "contents": attr.string_dict( + doc = "Hub file contents keyed by relative path. Mutually exclusive with the label attrs.", + ), + "crates_module": attr.label( + doc = "Preferred: a label to the vendored `crates.bzl`; siblings derive from its directory.", ), "defs_module": attr.label( - doc = "The `defs.bzl` file to use in the repository", - mandatory = True, + doc = "Legacy: the vendored `defs.bzl` re-export shim. Pair with `build_file`.", ), }, ) diff --git a/crate_universe/private/srcs.bzl b/crate_universe/private/srcs.bzl index a6fe888bd8..95a994e716 100644 --- a/crate_universe/private/srcs.bzl +++ b/crate_universe/private/srcs.bzl @@ -31,13 +31,13 @@ CARGO_BAZEL_SRCS = [ Label("//crate_universe:src/metadata/metadata_annotation.rs"), Label("//crate_universe:src/rendering.rs"), Label("//crate_universe:src/rendering/template_engine.rs"), + Label("//crate_universe:src/rendering/templates/defs_bzl_shim.j2"), Label("//crate_universe:src/rendering/templates/module_bzl.j2"), Label("//crate_universe:src/rendering/templates/partials/header.j2"), Label("//crate_universe:src/rendering/templates/partials/module/aliases_map.j2"), Label("//crate_universe:src/rendering/templates/partials/module/deps_map.j2"), Label("//crate_universe:src/rendering/templates/partials/module/repo_git.j2"), Label("//crate_universe:src/rendering/templates/partials/module/repo_http.j2"), - Label("//crate_universe:src/rendering/templates/vendor_module.j2"), Label("//crate_universe:src/rendering/verbatim/alias_rules.bzl"), Label("//crate_universe:src/select.rs"), Label("//crate_universe:src/splicing.rs"), diff --git a/crate_universe/src/rendering.rs b/crate_universe/src/rendering.rs index 4470e6ecc3..c3e21129ca 100644 --- a/crate_universe/src/rendering.rs +++ b/crate_universe/src/rendering.rs @@ -30,6 +30,20 @@ use crate::utils::{self, sanitize_repository_name}; // to platform labels like "@rules_rust//rust/platform:x86_64-unknown-linux-gnu". pub(crate) type Platforms = BTreeMap>; +/// A single hub alias — the `alias()` target and the `alias_rule` it uses. +struct HubAlias { + alias_rule: AliasRule, + alias: Alias, +} + +/// Every top-level alias the hub repository would emit, split by source so +/// the renderer can preserve the existing "# Workspace Member Dependencies" +/// / "# Binaries" comment sections when rendering into the flat root BUILD. +struct HubAliases { + workspace_member: Vec, + binaries: Vec, +} + pub(crate) struct Renderer { config: Arc, supported_platform_triples: Arc>, @@ -54,24 +68,14 @@ impl Renderer { let conditions = Arc::new(context.conditions.clone()); let engine = self.create_engine(Arc::clone(&conditions)); - let mut output = BTreeMap::new(); + let aliases = self.collect_hub_aliases(context)?; + let mut files = BTreeMap::new(); let platforms = self.render_platform_labels(conditions); - output.extend(self.render_build_files(&engine, context, &platforms)?); - output.extend(self.render_crates_module(&engine, context, &platforms, generator)?); - - if let Some(vendor_mode) = &self.config.vendor_mode { - match vendor_mode { - crate::config::VendorMode::Local => { - // Nothing to do for local vendor crate - } - crate::config::VendorMode::Remote => { - output.extend(self.render_vendor_support_files(&engine, context)?); - } - } - } + files.extend(self.render_build_files(&engine, context, &platforms)?); + files.extend(self.render_crates_module(&engine, context, &platforms, generator, &aliases)?); - Ok(output) + Ok(files) } pub(crate) fn create_engine( @@ -114,41 +118,39 @@ impl Renderer { context: &Context, platforms: &Platforms, generator: Option