diff --git a/.github/actions/build-native-ci/action.yaml b/.github/actions/build-native-ci/action.yaml new file mode 100644 index 0000000000..9b7ece3807 --- /dev/null +++ b/.github/actions/build-native-ci/action.yaml @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build or restore the Linux CI native library +description: 'Reuse an exact-input native library, otherwise build it with the CI profile' +runs: + using: composite + steps: + - name: Pin native build flags + shell: bash + run: echo 'RUSTFLAGS=-Ctarget-cpu=x86-64-v3 -Clink-arg=-fuse-ld=bfd' >> "$GITHUB_ENV" + + # Call after checkout and setup-builder. Compute once, before Cargo writes + # generated Rust files, and use the same keys for both restore and save. + - name: Fingerprint native build inputs + id: key + shell: bash + run: python3 dev/ci/native-cache-key.py --profile ci --github-output "$GITHUB_OUTPUT" + + - name: Restore native library cache + id: binary-cache + uses: actions/cache/restore@v6 + with: + path: native/target/ci/libcomet.so + key: ${{ steps.key.outputs.binary-key }} + # Main restores the incremental cache below, including libcomet.so. + # A lookup avoids downloading the same library separately. + lookup-only: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Restore incremental Cargo cache + id: cargo-cache + if: steps.binary-cache.outputs.cache-hit != 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main') + uses: actions/cache/restore@v6 + with: + path: native/target + key: ${{ steps.key.outputs.source-key }} + restore-keys: ${{ steps.key.outputs.restore-prefix }} + + - name: Build native library (CI profile) + if: steps.binary-cache.outputs.cache-hit != 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.cargo-cache.outputs.cache-hit != 'true') + shell: bash + run: | + cd native + # A library miss always builds, even on an exact incremental hit. + # Main also builds to populate a missing incremental entry; two exact + # hits already supply the library and leave neither cache to publish. + cargo build --locked --profile ci + + - name: Save native library cache + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.binary-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: native/target/ci/libcomet.so + key: ${{ steps.key.outputs.binary-key }} + + - name: Save incremental Cargo cache + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.cargo-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: native/target + key: ${{ steps.key.outputs.source-key }} diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 3b8f9c56a6..c7e3fb4f57 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -404,6 +404,42 @@ entry through `restore-keys` and downloads whatever else it needs, which is what a cold pull request already did. See the push-tier discussion above for which jobs do run on main and therefore do write. +## Reusing Linux native builds + +The Linux, Spark SQL, Iceberg and manual writer workflows call +`.github/actions/build-native-ci` after checkout and `setup-builder`. PR, queue, +scheduled and manual runs restore `native/target/ci/libcomet.so` and skip Cargo +on an exact library match. Only pushes to `main` save caches. Main skips Cargo +when both the library and incremental cache match exactly, and builds when either +lacks an exact match to replenish it. +An incremental cache hit alone never replaces compilation. Builds use +`cargo build --locked --profile ci`; manifest changes requiring a lockfile update +must include that update to `native/Cargo.lock`. Artifact paths remain unchanged. + +`dev/ci/native-cache-key.py` snapshots native sources, protobufs, dependencies, +Cargo configuration and shared build/setup actions before source generation. +It includes Rust versions, installed package versions, architecture, JDK +release/path, and Cargo/Rust, C/C++ compiler/flag and HDFS environment overrides. +Caller workflows are excluded because their selected tools and environment are +observed directly. Spark edits, documentation, generated files and disabled +contrib sources preserve the key; contrib manifests remain inputs for `--locked`. +Benchmarks enter only the debug key. The input lists and glob matcher are shared +with main's routing in `compute-changes.py`; code generation uses `x86-64-v3`. + +The helper supports the official Rust container and `setup-builder`. Introducing +external tools or files requires updating that contract: an override's path does +not identify arbitrary contents stored there. Both binary and incremental keys +retain package and JDK identity because native dependencies compile against JNI +headers and link `libjvm`, and Cargo does not fully track external tool/header +changes. Unrelated package updates can therefore cause conservative misses. + +The CI and debug incremental caches hold only `native/target`, including compiled +dependencies. Cargo fetches registry and Git dependency sources as needed; those +downloads are not duplicated in the repository's limited cache storage. Fallback +restores permit source changes within the same dependency/build environment. +Rust checks and tests always run with their separate debug cache. Preflight checks +fingerprint invalidation, main's routing, and the action's cache-hit/miss behavior. + ## Retrying flaky network operations **Maven.** `.mvn/maven.config` tunes the Maven Resolver HTTP transport: six diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5bdde5aa6..7e8267bcb5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,6 +134,11 @@ jobs: - name: Check Iceberg shard inventory validation run: python3 dev/ci/test-iceberg-shards.py + - name: Check native caching + run: | + python3 dev/ci/test-native-cache-key.py + python3 dev/ci/test-native-cache-workflow.py + - name: Check CI config invariants run: python3 dev/ci/check-ci-config.py diff --git a/.github/workflows/iceberg_spark_test_reusable.yml b/.github/workflows/iceberg_spark_test_reusable.yml index ae20b6d768..829202b2b4 100644 --- a/.github/workflows/iceberg_spark_test_reusable.yml +++ b/.github/workflows/iceberg_spark_test_reusable.yml @@ -81,33 +81,8 @@ jobs: id: shards run: python3 dev/ci/check-iceberg-shards.py --github-output "$GITHUB_OUTPUT" - - name: Restore Cargo cache - uses: actions/cache/restore@v6 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - native/target - key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} - restore-keys: | - ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}- - - - name: Build native library - # Use CI profile for faster builds (no LTO) and to share cache with pr_build_linux.yml. - run: | - cd native && cargo build --profile ci - env: - RUSTFLAGS: "-Ctarget-cpu=x86-64-v3 -Clink-arg=-fuse-ld=bfd" - - - name: Save Cargo cache - uses: actions/cache/save@v6 - if: github.ref == 'refs/heads/main' - with: - path: | - ~/.cargo/registry - ~/.cargo/git - native/target - key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} + - name: Build or restore native library + uses: ./.github/actions/build-native-ci - name: Upload native library uses: ./.github/actions/upload-artifact-retry diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index fd1b0a6c0a..3fb9ba3249 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -369,27 +369,10 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: ${{ env.RUST_VERSION }} - jdk-version: 17 # JDK only needed for JVM module proto generation + jdk-version: 17 # JNI headers and libjvm are native build inputs. - - name: Restore Cargo cache - uses: actions/cache/restore@v6 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - native/target - key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} - restore-keys: | - ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}- - - - name: Build native library (CI profile) - run: | - cd native - # CI profile: same overflow behavior as release, but faster compilation - # (no LTO, parallel codegen) - cargo build --profile ci - env: - RUSTFLAGS: "-Ctarget-cpu=x86-64-v3 -Clink-arg=-fuse-ld=bfd" + - name: Build or restore native library + uses: ./.github/actions/build-native-ci - name: Upload native library uses: ./.github/actions/upload-artifact-retry @@ -398,18 +381,6 @@ jobs: path: native/target/ci/libcomet.so retention-days: 1 - - name: Save Cargo cache - uses: actions/cache/save@v6 - # The push run is the cache warmer (see the header); a scheduled run - # at the same sha would only re-archive an entry that already exists. - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - with: - path: | - ~/.cargo/registry - ~/.cargo/git - native/target - key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} - # Run Rust tests (runs in parallel with build-native, uses debug builds). # Owns main's `cargo-debug` cache entry, so it runs in cache-refresh-only mode # as well. The tests themselves are a small slice of its runtime -- almost all @@ -432,17 +403,19 @@ jobs: rust-version: ${{ env.RUST_VERSION }} jdk-version: 17 + - name: Fingerprint Rust test build inputs + # Default HDFS support compiles against JNI headers and links libjvm. + # Cached build-script outputs also contain JDK paths (core/build.rs). + id: cargo-key + run: python3 dev/ci/native-cache-key.py --profile debug --github-output "$GITHUB_OUTPUT" + - name: Restore Cargo cache + id: cargo-cache uses: actions/cache/restore@v6 with: - path: | - ~/.cargo/registry - ~/.cargo/git - native/target - # Note: Java version intentionally excluded - Rust target is JDK-independent - key: ${{ runner.os }}-cargo-debug-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} - restore-keys: | - ${{ runner.os }}-cargo-debug-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}- + path: native/target + key: ${{ steps.cargo-key.outputs.source-key }} + restore-keys: ${{ steps.cargo-key.outputs.restore-prefix }} - name: Rust test steps uses: ./.github/actions/rust-test @@ -451,13 +424,10 @@ jobs: uses: actions/cache/save@v6 # The push run is the cache warmer (see the header); a scheduled run # at the same sha would only re-archive an entry that already exists. - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.cargo-cache.outputs.cache-hit != 'true' with: - path: | - ~/.cargo/registry - ~/.cargo/git - native/target - key: ${{ runner.os }}-cargo-debug-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} + path: native/target + key: ${{ steps.cargo-key.outputs.source-key }} linux-test: # `lint` is already upstream via build-native; it is listed here so this @@ -643,17 +613,6 @@ jobs: # Download to release/ since Maven's -Prelease expects libcomet.so there path: native/target/release/ - # Restore cargo registry cache (for any cargo commands that might run) - - name: Restore Cargo registry - uses: actions/cache/restore@v6 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('native/**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-registry- - - name: Java test steps uses: ./.github/actions/java-test with: diff --git a/.github/workflows/spark_sql_test_reusable.yml b/.github/workflows/spark_sql_test_reusable.yml index 9fa254f9b6..c5805a1197 100644 --- a/.github/workflows/spark_sql_test_reusable.yml +++ b/.github/workflows/spark_sql_test_reusable.yml @@ -97,33 +97,8 @@ jobs: rust-version: ${{ env.RUST_VERSION }} jdk-version: ${{ inputs.java }} - - name: Restore Cargo cache - uses: actions/cache/restore@v6 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - native/target - key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} - restore-keys: | - ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}- - - - name: Build native library (CI profile) - run: | - cd native - cargo build --profile ci - env: - RUSTFLAGS: "-Ctarget-cpu=x86-64-v3 -Clink-arg=-fuse-ld=bfd" - - - name: Save Cargo cache - uses: actions/cache/save@v6 - if: github.ref == 'refs/heads/main' - with: - path: | - ~/.cargo/registry - ~/.cargo/git - native/target - key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} + - name: Build or restore native library + uses: ./.github/actions/build-native-ci - name: Upload native library uses: ./.github/actions/upload-artifact-retry diff --git a/.github/workflows/spark_sql_writer_tests.yml b/.github/workflows/spark_sql_writer_tests.yml index c813a8b39e..1ceb73b16d 100644 --- a/.github/workflows/spark_sql_writer_tests.yml +++ b/.github/workflows/spark_sql_writer_tests.yml @@ -83,23 +83,8 @@ jobs: rust-version: ${{ env.RUST_VERSION }} jdk-version: ${{ steps.resolve.outputs.java }} - - name: Restore Cargo cache - uses: actions/cache/restore@v6 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - native/target - key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} - restore-keys: | - ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}- - - - name: Build native library (CI profile) - run: | - cd native - cargo build --profile ci - env: - RUSTFLAGS: "-Ctarget-cpu=x86-64-v3 -Clink-arg=-fuse-ld=bfd" + - name: Build or restore native library + uses: ./.github/actions/build-native-ci - name: Stage native library at release path run: | diff --git a/dev/ci/check-ci-config.py b/dev/ci/check-ci-config.py index abb6b1f2d4..0e0f9e5a1e 100644 --- a/dev/ci/check-ci-config.py +++ b/dev/ci/check-ci-config.py @@ -129,6 +129,10 @@ [".github/actions/maven-bootstrap/action.yaml"], {"build_linux", "build_linux_full", "build_linux_all_profiles"}, ), + # Linux's compact native cache is shared by every Spark/Iceberg producer. + # Helper edits also match macOS's broad dev/ci filter, as before. + ([".github/actions/build-native-ci/action.yaml"], BUILD_JOBS - {"build_macos"}), + (["dev/ci/native-cache-key.py"], BUILD_JOBS), # Spot checks that the additions above did not widen unrelated routes. (["docs/source/user-guide/overview.md"], {"docs"}), (["native/core/benches/parquet_read.rs"], {"benchmark"}), @@ -399,7 +403,7 @@ CACHE_REFRESH_WORKFLOW = WORKFLOWS / "pr_build_linux.yml" CACHE_REFRESH_JOBS = { "lint": "gates build-native and linux-test-rust, and costs 40 seconds", - "build-native": "writes the cargo-ci cache (native/target, CI profile)", + "build-native": "writes the compact native library and cargo-ci caches", "linux-test-rust": "writes the cargo-debug cache (native/target, debug)", "verify-benchmark-results-tpch": "writes the TPC-H SF=1 dataset and java-maven caches", "verify-benchmark-results-tpcds": "writes the TPC-DS SF=1 dataset and java-maven caches", diff --git a/dev/ci/compute-changes.py b/dev/ci/compute-changes.py index 799b791624..57eb13ff7c 100644 --- a/dev/ci/compute-changes.py +++ b/dev/ci/compute-changes.py @@ -38,6 +38,23 @@ import sys from pathlib import Path +# Shared cache recipes affect every native producer. Their tests run in +# Preflight and retain the existing dev/ci/** routes, without adding consumers. +NATIVE_CACHE_RECIPES = ( + ".github/actions/build-native-ci/**", + "dev/ci/native-cache-key.py", "dev/ci/compute-changes.py", +) + +# Cargo validates optional contrib manifests against native/Cargo.lock even +# with their features disabled. Their Rust sources and standalone lockfiles +# do not enter the default CI/debug builds. +NATIVE_BUILD_INPUTS = ( + "native/**", "contrib/*/native/Cargo.toml", ".cargo/**", + ".github/actions/setup-builder/**", *NATIVE_CACHE_RECIPES, + "rust-toolchain", "rust-toolchain.toml", "!**.md", +) +NATIVE_LIBRARY_INPUTS = (*NATIVE_BUILD_INPUTS, "!**/benches/**") + FILTERS = { "build_linux": [ "native/**", @@ -54,6 +71,7 @@ ".github/workflows/ci.yml", ".github/workflows/pr_build_linux.yml", ".github/actions/setup-builder/**", + ".github/actions/build-native-ci/**", ".github/actions/java-test/**", ".github/actions/maven-bootstrap/**", ".github/actions/rust-test/**", @@ -388,6 +406,14 @@ "mvnw", ], } +# Spark and Iceberg producers share these recipes. Linux routes the action +# above and already covers the Python helpers through dev/ci/**. +for _native_consumer in ( + "spark_3_4", "spark_3_5", "spark_4_0", "spark_4_1", + "iceberg_1_8", "iceberg_1_9", "iceberg_1_10", "iceberg_1_11", +): + FILTERS[_native_consumer].extend(NATIVE_CACHE_RECIPES) + FILTERS["spark_4_1_hive"] = FILTERS["spark_4_1"] FILTERS["build_linux_full"] = FILTERS["build_linux"] FILTERS["build_linux_all_profiles"] = FILTERS["build_linux"] @@ -546,11 +572,17 @@ def event_allows(job, event): def compute(files, event): - """Return {job: bool}, folding the path filter and the event policy.""" - return { + """Return job flags, including main's warmer for shared native cache inputs.""" + selected = { name: event_allows(name, event) and matches(patterns, files) for name, patterns in FILTERS.items() } + # Use the fingerprint's exact patterns and matcher for main's producer, + # including inputs owned by other workflows, without broadening PR jobs. + if (event.get("name") == "push" and event_allows("build_linux", event) + and matches(NATIVE_LIBRARY_INPUTS, files)): + selected["build_linux"] = True + return selected def event_from_env(): diff --git a/dev/ci/native-cache-key.py b/dev/ci/native-cache-key.py new file mode 100644 index 0000000000..f07cad84a3 --- /dev/null +++ b/dev/ci/native-cache-key.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Fingerprint the clean Linux checkout and toolchain used by Comet CI. + +Run after setup-builder and before Cargo generates source files. This helper +supports the official Rust container, setup-builder's JDK/packages, and the +build commands in our workflows; it is not a general local-build cache. +""" + +import argparse +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import subprocess + + +# Share both the input patterns and their glob semantics with main's warmer. +SPEC = importlib.util.spec_from_file_location("compute_changes", Path(__file__).with_name("compute-changes.py")) +CHANGES = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CHANGES) + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + + +def command(args, cwd): + return subprocess.check_output(args, cwd=cwd, text=True).strip() + + +def source_inputs(root, profile="ci"): + """Return dependency and source maps for the selected native build profile. + + Each map contains relative names, Git modes and content digests. Untracked + generated Rust, target files and documentation are excluded. CI library + builds omit benchmarks; debug checks compile them. Trust only this checkout + for the Git read: container steps can run as a different owner than checkout. + """ + patterns = CHANGES.NATIVE_LIBRARY_INPUTS if profile == "ci" else CHANGES.NATIVE_BUILD_INPUTS + inventory = command(["git", "-c", f"safe.directory={root}", + "ls-files", "--stage", "-z"], root) + sources = {} + for record in inventory.split("\0"): + if not record: + continue + metadata, name = record.split("\t", 1) + if CHANGES.matches(patterns, [name]): + sources[name] = [metadata.split()[0], + hashlib.sha256((root / name).read_bytes()).hexdigest()] + dependencies = {name: value for name, value in sources.items() + if Path(name).name in {"Cargo.toml", "Cargo.lock"}} + return dependencies, sources + + +def environment_inputs(root, env): + """Identify the official tools installed by setup-builder without modifying them. + + Rust's versions include the compiler commit; dpkg identifies the installed + C/C++/protobuf tools and system libraries. The JDK release file identifies + the vendor/build supplying JNI headers and libjvm. Record build overrides, + including target-qualified cc variables and HDFS linking options, without + including unrelated per-run GitHub variables. The shared setup/build actions + are hashed separately; caller test configuration does not affect the library. + """ + java_home = Path(env["JAVA_HOME"]) + return { + "workspace": str(root), + "architecture": command(["uname", "-m"], root), + "rust": {tool: command([tool, flag], root / "native") + for tool, flag in (("rustc", "-vV"), ("cargo", "--version"), + ("rustfmt", "--version"))}, + "packages": sorted(command(["dpkg-query", "-W", + "-f=${binary:Package}\t${Version}\t${Architecture}\n"], root).splitlines()), + "java_home": str(java_home), + "java_release": (java_home / "release").read_text(), + "cargo_home": env.get("CARGO_HOME", str(Path.home() / ".cargo")), + "env": {name: value for name, value in env.items() + if name.startswith(("CARGO_", "RUST", "HOST_", "TARGET_", "HDFS_")) + or name.split("_", 1)[0] in {"CC", "CXX", "CFLAGS", "CXXFLAGS", "CXXSTDLIB", + "LDFLAGS", "AR", "ARFLAGS", "RANLIB", "RANLIBFLAGS", "PROTOC"} + or name in {"JAVA_HOME", "PATH", "HADOOP_HOME", "DOCS_RS", + "CRATE_CC_NO_DEFAULTS", "CROSS_COMPILE"}}, + } + + +def cache_keys(profile, dependencies, sources, environment): + """Return output keys for one pre-build snapshot. + + Only the incremental Cargo cache has a source-independent restore prefix. + The library key includes all tracked build inputs and never uses fallback. + Both retain the environment: native build scripts can reuse C objects + without detecting changes to external compiler binaries or JNI headers. + """ + prefix = f"Linux-cargo-{profile}-v3-{digest([environment, dependencies])}-" + return { + "source-key": prefix + digest(sources), + "restore-prefix": prefix, + "binary-key": f"Linux-native-ci-v2-{digest([environment, sources])}" if profile == "ci" else "", + } + + +def main(): + """Snapshot the checkout root and publish keys after all reads succeed. + + Run from the repository root. --github-output appends the same key=value + records printed to stdout. Git/tool/file failures propagate and fail CI. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", required=True, choices=("ci", "debug")) + parser.add_argument("--github-output", type=Path) + args = parser.parse_args() + cwd = Path.cwd().resolve() + root = Path(command(["git", "-c", f"safe.directory={cwd}", + "rev-parse", "--show-toplevel"], cwd)) + dependencies, sources = source_inputs(root, args.profile) + environment = environment_inputs(root, os.environ) + keys = cache_keys(args.profile, dependencies, sources, environment) + output = "".join(f"{key}={value}\n" for key, value in keys.items()) + if args.github_output: + with args.github_output.open("a") as stream: + stream.write(output) + print(output, end="") + + +if __name__ == "__main__": + main() diff --git a/dev/ci/test-native-cache-key.py b/dev/ci/test-native-cache-key.py new file mode 100644 index 0000000000..a5f7a004b7 --- /dev/null +++ b/dev/ci/test-native-cache-key.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Check native cache boundaries and container checkout ownership with real Git.""" + +import importlib.util +import io +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + + +SPEC = importlib.util.spec_from_file_location("native_cache_key", Path(__file__).with_name("native-cache-key.py")) +CACHE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CACHE) + + +class NativeCacheKeyTests(unittest.TestCase): + """Use disposable Git repositories and mock only installed tool versions.""" + + def setUp(self): + """Create tracked native/JVM fixtures and JDK metadata; clean up after each test.""" + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + subprocess.run(["git", "init", "--quiet", str(self.root)], check=True) + self.inputs = {"native/Cargo.toml": "[workspace]\n", "native/Cargo.lock": "version = 4\n", + "native/lib.rs": "fn native() {}\n", "native/proto/expr.proto": "message Expr {}\n", + "spark/Plan.scala": "object Plan {}\n", "README.md": "Comet\n", + ".github/workflows/README.md": "CI documentation\n", + ".github/workflows/pr_build_linux.yml": "jobs: {}\n", + ".github/workflows/spark_sql_test_reusable.yml": "jobs: {}\n", + ".github/workflows/iceberg_spark_test_reusable.yml": "jobs: {}\n", + ".github/workflows/spark_sql_writer_tests.yml": "jobs: {}\n", + ".github/workflows/check_pr_title.yml": "jobs: {}\n", + ".github/actions/build-native-ci/action.yaml": "runs: {}\n", + ".github/actions/setup-builder/action.yaml": "runs: {}\n", + "dev/ci/compute-changes.py": "# shared native input rules\n", + "contrib/delta/native/Cargo.toml": '[package]\nname = "delta"\n', + "contrib/delta/native/src/lib.rs": "fn delta() {}\n", + "contrib/delta/native/Cargo.lock": "version = 4\n", + "contrib/a/b/native/Cargo.toml": '[package]\nname = "nested"\n', + "native/core/benches/perf.rs": "fn benchmark() {}\n"} + for name, content in self.inputs.items(): + self.write(name, content) + subprocess.run(["git", "add", "."], cwd=self.root, check=True) + self.write("jdk/release", 'JAVA_VERSION="17.0.1"\n') + self.env = {"JAVA_HOME": str(self.root / "jdk"), "CARGO_HOME": str(self.root / "cargo"), + "RUSTFLAGS": "-Ctarget-cpu=x86-64-v3 -Clink-arg=-fuse-ld=bfd"} + self.versions = {"rustc": "rustc 1.90\nhost: x86_64-unknown-linux-gnu\n", + "cargo": "cargo 1.90\n", "rustfmt": "rustfmt 1.8\n", + "dpkg-query": "libc6\t2.40\tamd64\n", "uname": "x86_64\n"} + + def write(self, name, content): + """Write fixture text under the temporary repository, creating its parents.""" + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + def keys(self, profile="ci"): + """Return keys from real tracked files and deterministic tool version responses.""" + dependencies, sources = CACHE.source_inputs(self.root, profile) + with patch.object(CACHE, "command", side_effect=lambda args, cwd: self.versions[args[0]]): + environment = CACHE.environment_inputs(self.root, self.env) + return CACHE.cache_keys(profile, dependencies, sources, environment) + + def test_source_and_dependency_changes_invalidate_the_right_keys(self): + """Native/protobuf edits retain the dependency prefix; dependency edits replace it.""" + before = self.keys() + for name in ("native/lib.rs", "native/proto/expr.proto", "native/Cargo.toml", "native/Cargo.lock", + "contrib/delta/native/Cargo.toml", "dev/ci/compute-changes.py", + ".github/actions/build-native-ci/action.yaml", ".github/actions/setup-builder/action.yaml"): + with self.subTest(name=name): + self.write(name, self.inputs[name] + "changed\n") + after = self.keys() + self.assertNotEqual(before["source-key"], after["source-key"]) + self.assertNotEqual(before["binary-key"], after["binary-key"]) + if name.endswith(("Cargo.toml", "Cargo.lock")): + self.assertNotEqual(before["restore-prefix"], after["restore-prefix"]) + else: + self.assertEqual(before["restore-prefix"], after["restore-prefix"]) + self.write(name, self.inputs[name]) + + def test_generated_files_and_unrelated_jvm_edits_preserve_keys(self): + """Generated files and non-build edits preserve reuse; debug still tracks benchmarks.""" + before = self.keys() + debug = self.keys("debug") + for name in self.inputs: + if name.startswith(".github/workflows/"): + self.write(name, "unrelated test configuration\n") + self.env["GITHUB_RUN_ID"] = "12345" + self.assertEqual(before, self.keys()) + self.assertEqual(debug, self.keys("debug")) + self.write("native/proto/src/generated/expr.rs", "generated Rust") + self.write("native/target/ci/libcomet.so", "compiled library") + self.write("spark/Plan.scala", "object NewPlan {}") + self.write("README.md", "updated docs") + self.write("contrib/delta/native/src/lib.rs", "fn changed_delta() {}") + self.write("contrib/delta/native/Cargo.lock", "version = 3\n") + self.write("contrib/a/b/native/Cargo.toml", '[package]\nname = "changed_nested"\n') + self.write("native/core/benches/perf.rs", "fn changed_benchmark() {}") + self.assertEqual(before, self.keys()) + self.assertNotEqual(debug["source-key"], self.keys("debug")["source-key"]) + + def test_native_input_routing(self): + """Library inputs warm main; helper tests retain Linux coverage without extra consumers.""" + route = CACHE.CHANGES.compute + inputs = ("native/core/src/lib.rs", "native/proto/expr.proto", + "contrib/new/native/Cargo.toml", ".cargo/config.toml", + ".github/actions/setup-builder/action.yaml", + ".github/actions/build-native-ci/action.yaml", + "dev/ci/native-cache-key.py", "dev/ci/compute-changes.py", + "rust-toolchain", "rust-toolchain.toml") + for name in inputs: + self.assertTrue(CACHE.CHANGES.matches(CACHE.CHANGES.NATIVE_LIBRARY_INPUTS, [name]), name) + self.assertTrue(route([name], {"name": "push"})["build_linux"], name) + for name in ("native/core/README.md", "native/core/benches/perf.rs", + "contrib/delta/native/src/lib.rs", "contrib/delta/native/Cargo.lock", + "contrib/a/b/native/Cargo.toml", "contrib/a/b/native/x.rs"): + self.assertFalse(CACHE.CHANGES.matches(CACHE.CHANGES.NATIVE_LIBRARY_INPUTS, [name]), name) + self.assertFalse(route([name], {"name": "push"})["build_linux"], name) + with patch.dict(CACHE.CHANGES.POLICY, {"build_linux": ["pr", "queue"]}): + self.assertFalse(route([".cargo/config.toml"], {"name": "push"})["build_linux"]) + self.assertFalse(route(["contrib/new/native/Cargo.toml"], {"name": "pull_request"})["build_linux"]) + for event in ("merge_group", "schedule"): + routed = route(["dev/ci/test-native-cache-key.py"], {"name": event}) + self.assertTrue(routed["build_linux" if event == "merge_group" else "build_linux_all_profiles"]) + self.assertFalse(any(selected for name, selected in routed.items() + if name.startswith(("spark_", "iceberg_")))) + + def test_tools_jdk_flags_and_tracked_build_configuration_invalidate(self): + """Capture build overrides only; tools, Java metadata and tracked configs invalidate keys.""" + before = self.keys() + for tool in self.versions: + with self.subTest(tool=tool): + old = self.versions[tool] + self.versions[tool] += "changed\n" + self.assertNotEqual(before["binary-key"], self.keys()["binary-key"]) + self.versions[tool] = old + self.write("jdk/release", 'JAVA_VERSION="17.0.2"\n') + self.assertNotEqual(before["binary-key"], self.keys()["binary-key"]) + self.write("jdk/release", 'JAVA_VERSION="17.0.1"\n') + self.env["RUSTFLAGS"] += " -Copt-level=1" + self.assertNotEqual(before["binary-key"], self.keys()["binary-key"]) + self.env["RUSTFLAGS"] = "-Ctarget-cpu=x86-64-v3 -Clink-arg=-fuse-ld=bfd" + overrides = ("CC", "CXX", "CFLAGS", "LDFLAGS", "AR", "PROTOC", "PROTOC_INCLUDE", + "RUSTC_WRAPPER", "CARGO_BUILD_TARGET", "CARGO_PROFILE_CI_OPT_LEVEL", + "CC_x86_64_unknown_linux_gnu", "HOST_CC", "TARGET_CFLAGS", + "HDFS_LIB_DIR", "HADOOP_HOME", "HDFS_STATIC", "DOCS_RS", "PATH") + build_env = {**self.env, **dict.fromkeys(overrides, "build override")} + with patch.object(CACHE, "command", side_effect=lambda args, cwd: self.versions[args[0]]): + environment = CACHE.environment_inputs( + self.root, {**build_env, "GITHUB_RUN_ID": "12345", "UNRELATED": "ignored"}) + self.assertEqual(environment["env"], build_env) + self.env["TARGET_CFLAGS"] = "build override" + after = self.keys() + for key in ("binary-key", "source-key", "restore-prefix"): + self.assertNotEqual(before[key], after[key]) + del self.env["TARGET_CFLAGS"] + self.write(".cargo/config.toml", "[build]\nincremental = false\n") + subprocess.run(["git", "add", ".cargo/config.toml"], cwd=self.root, check=True) + self.assertNotEqual(before["binary-key"], self.keys()["binary-key"]) + + def test_profiles_have_separate_cargo_caches(self): + """CI/debug keys stay separate and only CI produces a reusable library key.""" + ci, debug = self.keys("ci"), self.keys("debug") + self.assertNotEqual(ci["source-key"], debug["source-key"]) + self.assertNotEqual(ci["restore-prefix"], debug["restore-prefix"]) + self.assertEqual(debug["binary-key"], "") + self.assertTrue(ci["binary-key"].startswith("Linux-native-ci-")) + self.assertTrue(ci["source-key"].startswith(ci["restore-prefix"])) + + def test_container_ownership_works_without_global_git_config_changes(self): + """A differently owned checkout permits helper root/inventory reads without global trust.""" + self.write("global.gitconfig", "[user]\n\tname = Cache Test\n") + config = self.root / "global.gitconfig" + original = config.read_bytes() + output = self.root / "github-output" + environment = {**self.env, "GIT_TEST_ASSUME_DIFFERENT_OWNER": "1", + "GIT_CONFIG_GLOBAL": str(config), "GIT_CONFIG_NOSYSTEM": "1"} + arguments = ["native-cache-key.py", "--profile", "ci", "--github-output", str(output)] + with patch.dict(os.environ, environment): + ordinary = subprocess.run(["git", "rev-parse", "--show-toplevel"], cwd=self.root, + text=True, capture_output=True) + self.assertNotEqual(ordinary.returncode, 0) + self.assertIn("dubious ownership", ordinary.stderr) + with patch.object(CACHE.Path, "cwd", return_value=self.root), \ + patch.object(sys, "argv", arguments), \ + patch.object(CACHE, "environment_inputs", return_value={"cargo_home": self.env["CARGO_HOME"]}), \ + patch.object(sys, "stdout", new_callable=io.StringIO): + CACHE.main() + self.assertIn("source-key=Linux-cargo-ci-", output.read_text()) + self.assertEqual(config.read_bytes(), original) + + +if __name__ == "__main__": + unittest.main() diff --git a/dev/ci/test-native-cache-workflow.py b/dev/ci/test-native-cache-workflow.py new file mode 100644 index 0000000000..9655cc9f02 --- /dev/null +++ b/dev/ci/test-native-cache-workflow.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Check the native action's actual conditions for cache hits and misses.""" + +from pathlib import Path +import re +import shlex +import subprocess +import unittest + + +def condition_matches(expression, context): + """Evaluate the action's string comparisons and boolean operators with Bash. + + Substitute fixed test context values into the expression read from YAML. + Bash [[ ]] supports the comparisons, parentheses, && and || used here; + unsupported syntax fails the test instead of emulating the Actions runner. + """ + expression = expression.removeprefix("${{ ").removesuffix(" }}") + for name, value in context.items(): + expression = expression.replace(name, shlex.quote(value)) + result = subprocess.run(["bash", "-c", f"[[ {expression} ]]"], capture_output=True, text=True) + if result.returncode not in (0, 1): + raise AssertionError(result.stderr) + return result.returncode == 0 + + +class NativeCacheWorkflowTests(unittest.TestCase): + def test_cache_hit_decisions(self): + """Protect compilation, lookup-only restores, and main-only publication.""" + project = Path(__file__).resolve().parents[2] + action = (project / ".github/actions/build-native-ci/action.yaml").read_text() + conditions = {} + for block in re.split(r"^ - name: ", action, flags=re.MULTILINE)[1:]: + name, _, body = block.partition("\n") + for line in body.splitlines(): + if line.startswith(" if: "): + conditions[name] = line.removeprefix(" if: ") + elif line.startswith(" lookup-only: "): + conditions["lookup-only"] = line.removeprefix(" lookup-only: ") + names = ("lookup-only", "Restore incremental Cargo cache", "Build native library (CI profile)", + "Save native library cache", "Save incremental Cargo cache") + self.assertEqual(set(conditions), set(names)) + # Expected: lookup only, restore Cargo, build, save library, save Cargo. + cases = [ + ("pull_request", "refs/pull/1/merge", "true", "", (False, False, False, False, False)), + ("pull_request", "refs/pull/1/merge", "", "true", (False, True, True, False, False)), + ("merge_group", "refs/heads/gh-readonly-queue/main/test", "true", "", (False, False, False, False, False)), + ("merge_group", "refs/heads/gh-readonly-queue/main/test", "", "false", (False, True, True, False, False)), + ("push", "refs/heads/main", "true", "true", (True, True, False, False, False)), + ("push", "refs/heads/main", "true", "false", (True, True, True, False, True)), + ("push", "refs/heads/main", "true", "", (True, True, True, False, True)), + ("push", "refs/heads/main", "", "true", (True, True, True, True, False)), + ("push", "refs/heads/main", "", "", (True, True, True, True, True)), + ("push", "refs/heads/branch", "", "", (False, True, True, False, False)), + ("schedule", "refs/heads/main", "", "", (False, True, True, False, False)), + ("workflow_dispatch", "refs/heads/main", "", "", (False, True, True, False, False)), + ] + for event, ref, library_hit, cargo_hit, expected in cases: + with self.subTest(event=event, ref=ref, library_hit=library_hit, cargo_hit=cargo_hit): + context = {"github.event_name": event, "github.ref": ref, + "steps.binary-cache.outputs.cache-hit": library_hit, + "steps.cargo-cache.outputs.cache-hit": cargo_hit} + self.assertEqual(tuple(condition_matches(conditions[name], context) for name in names), expected) + + +if __name__ == "__main__": + unittest.main()