Rust: Update rust-analyzer update instructions and update script

Author: paldepind

MODULE.bazel

@@ -166,13 +166,15 @@ use_repo(
 
 http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-# rust-analyzer sources needed by the rust ast-generator (see `rust/ast-generator/README.md`)
+# rust-analyzer sources needed by the rust ast-generator (see `rust/ast-generator/README.md`).
+# These should be updated through the `rust/scripts/update_rust_analyzer.py` script.
 RUST_ANALYZER_SRC_TAG = "2026-04-13"
+RUST_ANALYZER_SRC_INTEGRITY = "sha256-UB/+EVx/6j4VGvnb7jfRqPaoc7Uwci3rEt6il+2J1Ds="
 
 http_archive(
     name = "rust-analyzer-src",
     build_file = "//rust/ast-generator:BUILD.rust-analyzer-src.bazel",
-    integrity = "sha256-UB/+EVx/6j4VGvnb7jfRqPaoc7Uwci3rEt6il+2J1Ds=",
+    integrity = RUST_ANALYZER_SRC_INTEGRITY,
     patch_args = ["-p1"],
     patches = [
         "//rust/ast-generator:patches/rust-analyzer.patch",

rust/scripts/update_rust_analyzer.py

@@ -1,6 +1,9 @@
 #!/usr/bin/env python3
 
+import base64
+import hashlib
 import json
+import re
 import shlex
 import subprocess
 import sys
@@ -10,6 +13,7 @@
 
 
 REPO_ROOT = Path(__file__).resolve().parents[2]
+MODULE_BAZEL = REPO_ROOT / "MODULE.bazel"
 RUST_EXTRACTOR_MANIFEST = REPO_ROOT / "rust/extractor/Cargo.toml"
 
 # The files that mention the fixed Rust toolchain version
@@ -32,7 +36,7 @@ def run_codegen() -> None:
         run("bazel", "run", "//rust/codegen")
     except subprocess.CalledProcessError as error:
         print(
-            "\nCodegen failed. Carry out the instructions from step 4 in rust/updating-rust-analyzer.md manually.",
+            "\nCodegen failed. Carry out the instructions from step 5 in rust/updating-rust-analyzer.md manually.",
             file=sys.stderr,
             flush=True,
         )
@@ -61,12 +65,48 @@ def fetch(url: str) -> bytes:
         return response.read()
 
 
-def get_compatible_rust_toolchain(rust_analyzer_version: str) -> str:
-    """Get the latest Rust toolchain released no later than rust-analyzer."""
-    # Get the release date of the rust-analyzer version
+def get_rust_analyzer_release_date(rust_analyzer_version: str) -> str:
+    """Get the release date of a rust-analyzer crate version."""
     crate_url = f"https://crates.io/api/v1/crates/ra_ap_syntax/{rust_analyzer_version}"
     crate = json.loads(fetch(crate_url))
-    rust_analyzer_release = crate["version"]["created_at"].split("T")[0]
+    return crate["version"]["created_at"].split("T")[0]
+
+
+def update_rust_analyzer_sources(rust_analyzer_version: str) -> None:
+    """
+    Update the rust-analyzer source archive used by the AST generator.
+
+    Concretly, this updates `RUST_ANALYZER_SRC_TAG` and
+    `RUST_ANALYZER_SRC_INTEGRITY` in the MODULE.bazel file.
+    """
+
+    release_date = get_rust_analyzer_release_date(rust_analyzer_version)
+    archive_url = (
+        "https://github.com/rust-lang/rust-analyzer/archive/refs/tags/"
+        f"{release_date}.tar.gz"
+    )
+    archive = fetch(archive_url)
+    integrity = "sha256-" + base64.b64encode(hashlib.sha256(archive).digest()).decode()
+
+    module = MODULE_BAZEL.read_text()
+    module = re.sub(
+        r'RUST_ANALYZER_SRC_TAG = "[^"]+"',
+        f'RUST_ANALYZER_SRC_TAG = "{release_date}"',
+        module,
+        count=1,
+    )
+    module = re.sub(
+        r'RUST_ANALYZER_SRC_INTEGRITY = "[^"]+"',
+        f'RUST_ANALYZER_SRC_INTEGRITY = "{integrity}"',
+        module,
+        count=1,
+    )
+    MODULE_BAZEL.write_text(module)
+
+
+def get_compatible_rust_toolchain(rust_analyzer_version: str) -> str:
+    """Get the latest Rust toolchain released no later than rust-analyzer."""
+    rust_analyzer_release = get_rust_analyzer_release_date(rust_analyzer_version)
 
     # `manifests.txt` is a list of all toolchains. The one we're interested in looks like
     # ```
@@ -168,20 +208,24 @@ def main() -> None:
         run("cargo", "update")
     commit_all("Cargo: Upgrade dependencies")
 
-    print_step(2, "Update the fixed Rust toolchain used by the extractor")
+    print_step(2, "Update the rust-analyzer sources used by the AST generator")
+    update_rust_analyzer_sources(new_rust_analyzer_version)
+    commit_all("Rust: Update rust-analyzer sources")
+
+    print_step(3, "Update the fixed Rust toolchain used by the extractor")
     rust_toolchain = get_compatible_rust_toolchain(new_rust_analyzer_version)
     update_fixed_rust_toolchain_versions(rust_toolchain)
     commit_all("Rust: Update fixed toolchain")
 
-    print_step(3, "Regenerate vendored bazel files")
+    print_step(4, "Regenerate vendored bazel files")
     run("misc/bazel/3rdparty/update_tree_sitter_extractors_deps.sh")
     commit_all("Bazel: Regenerate vendored cargo dependencies")
 
-    print_step(4, "Run codegen")
+    print_step(5, "Run codegen")
     run_codegen()
     commit_all("Rust: Run codegen")
 
-    print_step(5, "Try compiling")
+    print_step(6, "Try compiling")
     run("bazel", "run", "//rust:install")
 
 

rust/updating-rust-analyzer.md

@@ -28,7 +28,16 @@ pkg-config`.
       git commit -am 'Cargo: upgrade dependencies' --no-verify
       ```
 
-2. Update the fixed Rust toolchain used by the extractor.
+2. Update the rust-analyzer sources used by the AST generator.
+
+   In `MODULE.bazel`, update `RUST_ANALYZER_SRC_TAG` to the release date of the
+   new `ra_ap_` crates and update `RUST_ANALYZER_SRC_INTEGRITY` with the
+   integrity of the corresponding source archive.
+
+   Commit the changes:
+   `git commit -am 'Rust: Update rust-analyzer sources'`
+
+3. Update the fixed Rust toolchain used by the extractor.
 
    The version should be updated to the latest release that precedes the new
    rust-analyzer version.
@@ -38,23 +47,23 @@ pkg-config`.
 
    Commit the changes: `git commit -am 'Rust: Update fixed toolchain'`
 
-3. Regenerate vendored bazel files (these allow faster builds, particularly on
+4. Regenerate vendored bazel files (these allow faster builds, particularly on
    CI where it has to start from scratch each time), commit the changes:
    ```
    misc/bazel/3rdparty/update_tree_sitter_extractors_deps.sh
    git add .
    git commit -am 'Bazel: regenerate vendored cargo dependencies' --no-verify
    ```
    > [!NOTE]
-   > If in step 5 you also bump `rules_rust` or the Rust toolchain used for
+   > If in step 6 you also bump `rules_rust` or the Rust toolchain used for
    > building the extractor, those changes invalidate _all_ vendored files
    > (including the Python ones under `misc/bazel/3rdparty/py_deps`), not just
    > the tree-sitter ones. In that case run the umbrella script
    > `misc/bazel/3rdparty/update_cargo_deps.sh` instead (it regenerates both
    > `py_deps` and `tree_sitter_extractors_deps`, and runs `bazel mod tidy`),
    > then commit all the regenerated files.
 
-4. Run codegen
+5. Run codegen
    ```
    bazel run //rust/codegen
    ```
@@ -70,7 +79,7 @@ pkg-config`.
    new tests and/or downgrade/upgrade scripts down the line.
 
 
-5. Try compiling
+6. Try compiling
    ```
    bazel run //rust:install
    ```
@@ -88,7 +97,7 @@ pkg-config`.
         to a more recent date while you're at it.
       * a toolchain and/or `rules_rust` bump invalidates the vendored files, so
        re-run `misc/bazel/3rdparty/update_cargo_deps.sh` (see the note in step
-       3) and commit the regenerated files.
+       4) and commit the regenerated files.
    * if it fails while compiling rust extractor code, you will need to adapt it
      to the new library version.
       * for example updating annotations in `annotations.py`, adding / removing
@@ -98,11 +107,11 @@ pkg-config`.
    toolchain, running `rust/lint.py` might reformat or apply new lints to the
    code.
 
-6. Check with CI if everything is in order.
+7. Check with CI if everything is in order.
 
-7. Run DCA with database caching disabled. Iterate on the code if needed.
+8. Run DCA with database caching disabled. Iterate on the code if needed.
 
-8. If in step 4 the schema was updated, add upgrade/downgrade scripts and a
+9. If in step 5 the schema was updated, add upgrade/downgrade scripts and a
    change note. This is best done last to reduce the chance of merge conflicts
    (none of the other testing depends on having upgrade and downgrade scripts
    in place). See [Upgrading a language database