ci: share one Linux native build across CI workflows

.github/workflows/build_linux_native.yml

@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Single producer for the default Linux CI native library. The umbrella runs
+# this whenever any Linux, Spark SQL, or Iceberg test consumer is selected.
+name: Build Linux Native Library
+
+on:
+  workflow_call:
+
+permissions:
+  contents: read
+
+env:
+  RUST_VERSION: stable
+  RUST_BACKTRACE: 1
+
+jobs:
+  # Build native library once and share with all test jobs
+  build-native:
+    name: Build Native Library
+    runs-on: ubuntu-24.04
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@v7
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: ${{ env.RUST_VERSION }}
+          jdk-version: 17  # Matches the existing Linux producer, including libjvm linkage.
+
+      - name: Check Rust formatting
+        run: cd native && cargo fmt --all -- --check
+
+      - name: Restore Cargo cache
+        uses: actions/cache/restore@v6
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            native/target
+          key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-
+
+      - name: Build native library (CI profile)
+        run: |
+          cd native
+          # CI profile: same overflow behavior as release, but faster compilation
+          # (no LTO, parallel codegen)
+          cargo build --profile ci
+        env:
+          RUSTFLAGS: "-Ctarget-cpu=x86-64-v3 -Clink-arg=-fuse-ld=bfd"
+
+      - name: Upload native library
+        uses: ./.github/actions/upload-artifact-retry
+        with:
+          name: native-lib-linux
+          path: native/target/ci/libcomet.so
+          retention-days: 1
+
+      - name: Save Cargo cache
+        uses: actions/cache/save@v6
+        # Nightly runs reuse the cache already refreshed by the push run.
+        if: github.event_name == 'push'
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            native/target
+          key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}

.github/workflows/ci.yml

@@ -135,7 +135,10 @@ jobs:
         run: python3 dev/ci/test-iceberg-shards.py
 
       - name: Check CI config invariants
-        run: python3 dev/ci/check-ci-config.py
+        run: |
+          python3 dev/ci/check-ci-config.py
+          python3 dev/ci/test-ci-config.py
+          python3 dev/ci/test-native-build-selection.py
 
       - name: Install actionlint
         # Pure network, and preflight gates every other job, so a single reset
@@ -178,6 +181,7 @@ jobs:
       actions: read
       contents: read
     outputs:
+      build_linux_native: ${{ steps.compute.outputs.build_linux_native }}
       build_linux:  ${{ steps.compute.outputs.build_linux }}
       build_linux_full: ${{ steps.compute.outputs.build_linux_full }}
       build_linux_all_profiles: ${{ steps.compute.outputs.build_linux_all_profiles }}
@@ -278,15 +282,37 @@ jobs:
   # can actually test it.
   # ---------------------------------------------------------------------------
 
+  # Independent checks start immediately after change selection.
+  pr_build_linux_checks:
+    name: PR Checks (Linux)
+    needs: changes
+    permissions:
+      contents: read
+    if: needs.changes.outputs.build_linux == 'true' || needs.changes.outputs.build_linux_all_profiles == 'true'
+    uses: ./.github/workflows/pr_build_linux_checks.yml
+    with:
+      cache-refresh-only: ${{ needs.changes.outputs.build_linux_full != 'true' && needs.changes.outputs.build_linux_all_profiles != 'true' }}
+      profiles: >-
+        ${{ needs.changes.outputs.build_linux_all_profiles != 'true' && 'pr'
+            || needs.changes.outputs.build_linux_full != 'true' && 'nightly'
+            || 'all' }}
+
+  # compute-changes.py derives this output from the selected native consumers.
+  build_linux_native:
+    name: Shared Linux Native Library
+    needs: changes
+    permissions:
+      contents: read
+    if: needs.changes.outputs.build_linux_native == 'true'
+    uses: ./.github/workflows/build_linux_native.yml
+
   pr_build_linux:
     name: PR Build (Linux)
-    needs: changes
-    # Three POLICY outputs feed one call, the same shape as spark_4_1 below.
-    # `build_linux` decides whether the workflow runs at all; `build_linux_full`
-    # whether it runs the lints and the test matrix as well as the jobs that
-    # populate main's actions/cache entries; `build_linux_all_profiles` whether
-    # the test matrix covers every Spark profile or only the default one. The
-    # combinations that occur:
+    needs: [changes, build_linux_native]
+    # Three POLICY outputs configure both Linux callers above and here.
+    # `build_linux` or `build_linux_all_profiles` selects them; `build_linux_full`
+    # enables the independent checks and tests beyond main's cache writers.
+    # Together with `build_linux_all_profiles`, it selects the test profiles:
     #
     #   pull request          linux, full            -> profiles: pr
     #   ... with the label    linux, full, all       -> profiles: all
@@ -302,6 +328,7 @@ jobs:
     if: needs.changes.outputs.build_linux == 'true' || needs.changes.outputs.build_linux_all_profiles == 'true'
     uses: ./.github/workflows/pr_build_linux.yml
     with:
+      native-library-artifact: native-lib-linux
       cache-refresh-only: ${{ needs.changes.outputs.build_linux_full != 'true' && needs.changes.outputs.build_linux_all_profiles != 'true' }}
       profiles: >-
         ${{ needs.changes.outputs.build_linux_all_profiles != 'true' && 'pr'
@@ -351,7 +378,7 @@ jobs:
 
   spark_3_4:
     name: Spark SQL Tests (Spark 3.4)
-    needs: changes
+    needs: [changes, build_linux_native]
     # Spark 3.4 is deprecated, so this is the one test job the merge queue does
     # not run: it needs the `run-spark-3.4-tests` label on a pull request, or a
     # manual workflow_dispatch.
@@ -368,36 +395,39 @@ jobs:
     if: needs.changes.outputs.spark_3_4 == 'true'
     uses: ./.github/workflows/spark_sql_test_reusable.yml
     with:
+      native-library-artifact: native-lib-linux
       spark-short: '3.4'
       spark-full: '3.4.3'
       java: 17
 
   spark_3_5:
     name: Spark SQL Tests (Spark 3.5)
-    needs: changes
+    needs: [changes, build_linux_native]
     # Nightly by default; PRs need the `run-spark-3.5-tests` label.
     if: needs.changes.outputs.spark_3_5 == 'true'
     uses: ./.github/workflows/spark_sql_test_reusable.yml
     with:
+      native-library-artifact: native-lib-linux
       spark-short: '3.5'
       spark-full: '3.5.9'
       java: 17
 
   spark_4_0:
     name: Spark SQL Tests (Spark 4.0)
-    needs: changes
+    needs: [changes, build_linux_native]
     # Nightly by default; PRs need the `run-spark-4.0-tests` label. Spark 4.1
     # is the one Spark SQL suite the queue runs, being the default profile.
     if: needs.changes.outputs.spark_4_0 == 'true'
     uses: ./.github/workflows/spark_sql_test_reusable.yml
     with:
+      native-library-artifact: native-lib-linux
       spark-short: '4.0'
       spark-full: '4.0.4'
       java: 17
 
   spark_4_1:
     name: Spark SQL Tests (Spark 4.1)
-    needs: changes
+    needs: [changes, build_linux_native]
     # Queue-only by default, like every other Spark SQL suite. Two POLICY
     # outputs feed one call, so the queue gets every module from a single
     # 40-minute build instead of two. `spark_4_1` covers catalyst and the
@@ -407,6 +437,7 @@ jobs:
     if: needs.changes.outputs.spark_4_1 == 'true' || needs.changes.outputs.spark_4_1_hive == 'true'
     uses: ./.github/workflows/spark_sql_test_reusable.yml
     with:
+      native-library-artifact: native-lib-linux
       spark-short: '4.1'
       spark-full: '4.1.3'
       java: 17
@@ -417,11 +448,12 @@ jobs:
 
   iceberg_1_8:
     name: Iceberg Spark SQL Tests (Iceberg 1.8)
-    needs: changes
+    needs: [changes, build_linux_native]
     # Nightly by default; PRs need the `run-iceberg-tests` label.
     if: needs.changes.outputs.iceberg_1_8 == 'true'
     uses: ./.github/workflows/iceberg_spark_test_reusable.yml
     with:
+      native-library-artifact: native-lib-linux
       iceberg-short: '1.8'
       iceberg-full: '1.8.1'
       spark-short: '3.4'
@@ -430,11 +462,12 @@ jobs:
 
   iceberg_1_9:
     name: Iceberg Spark SQL Tests (Iceberg 1.9)
-    needs: changes
+    needs: [changes, build_linux_native]
     # Nightly by default; PRs need the `run-iceberg-tests` label.
     if: needs.changes.outputs.iceberg_1_9 == 'true'
     uses: ./.github/workflows/iceberg_spark_test_reusable.yml
     with:
+      native-library-artifact: native-lib-linux
       iceberg-short: '1.9'
       iceberg-full: '1.9.1'
       spark-short: '3.5'
@@ -443,11 +476,12 @@ jobs:
 
   iceberg_1_10:
     name: Iceberg Spark SQL Tests (Iceberg 1.10)
-    needs: changes
+    needs: [changes, build_linux_native]
     # Nightly by default; PRs need the `run-iceberg-tests` label.
     if: needs.changes.outputs.iceberg_1_10 == 'true'
     uses: ./.github/workflows/iceberg_spark_test_reusable.yml
     with:
+      native-library-artifact: native-lib-linux
       iceberg-short: '1.10'
       iceberg-full: '1.10.0'
       spark-short: '3.5'
@@ -456,13 +490,14 @@ jobs:
 
   iceberg_1_11:
     name: Iceberg Spark SQL Tests (Iceberg 1.11)
-    needs: changes
+    needs: [changes, build_linux_native]
     # Queue-only by default; PRs need the `run-iceberg-tests` label. Iceberg
     # 1.11 is our only Spark 4.1 Iceberg coverage, which is why it is the one
     # Iceberg version the queue runs while 1.8/1.9/1.10 run nightly.
     if: needs.changes.outputs.iceberg_1_11 == 'true'
     uses: ./.github/workflows/iceberg_spark_test_reusable.yml
     with:
+      native-library-artifact: native-lib-linux
       iceberg-short: '1.11'
       iceberg-full: '1.11.0'
       spark-short: '4.1'
@@ -508,6 +543,8 @@ jobs:
     needs:
       - preflight
       - changes
+      - pr_build_linux_checks
+      - build_linux_native
       - pr_build_linux
       - pr_build_macos
       - pr_benchmark_check