ci: run the non-default Spark and Iceberg suites nightly instead of in the merge queue

.github/workflows/ci.yml

@@ -20,19 +20,25 @@
 # change touched files relevant to that workflow.
 #
 # Merging goes through GitHub's merge queue (see `rulesets` in `.asf.yaml`), so
-# there are two tiers:
+# there are three tiers:
 #
 #   pull_request  fast feedback. The Linux build only, with its test matrix
 #                 run against the default Spark profile (4.1) alone.
-#   merge_group   the authoritative gate. The Linux build with the test
-#                 matrix against every Spark profile, plus the macOS build,
+#   merge_group   the authoritative gate. The PR tier plus the macOS build,
 #                 the benchmark compile check, the Delta contrib build gate,
-#                 the PyArrow UDF suite, Spark 3.5/4.0/4.1 and Iceberg
-#                 1.8/1.9/1.10/1.11, evaluated against the merge result
-#                 rather than the PR head.
+#                 the PyArrow UDF suite, Spark SQL on Spark 4.1 and Iceberg
+#                 1.11, evaluated against the merge result rather than the
+#                 PR head. One Spark version and one Iceberg version, both
+#                 the default profile's.
+#   schedule      the nightly regression sweep of everything else: the Linux
+#                 test matrix against the other Spark profiles, Spark SQL on
+#                 Spark 3.5 and 4.0, and Iceberg 1.8/1.9/1.10, routed by the
+#                 same path filters over what landed on main since the last
+#                 green nightly. A failure opens (or comments on) an issue
+#                 labelled `ci-nightly-failure`.
 #
-# Spark 3.4 is deprecated and sits outside both tiers: it runs only when a
-# pull request carries `run-spark-3.4-tests`, or from a manual dispatch.
+# Spark 3.4 is deprecated and sits outside all three tiers: it runs only when
+# a pull request carries `run-spark-3.4-tests`, or from a manual dispatch.
 #
 # Which tier a job sits in is POLICY in dev/ci/compute-changes.py, not an
 # expression here. Heavy jobs deliberately have no `push` tier: the queue
@@ -49,10 +55,12 @@ name: CI
 # A `labeled` event (e.g. the run-spark-*-tests gates, or dependabot's automatic
 # `dependencies` label added ~1s after open) fires at the same commit as the
 # opened/synchronize run. Keying the group on the label name keeps labeled runs
-# in their own subgroup so they never cancel the real commit run; opened and
-# synchronize both map to `commit` so a new push still supersedes its predecessor.
+# in their own subgroup so they never cancel the real commit run. Every other
+# run is keyed on its event name: opened, synchronize and reopened all map to
+# `pull_request`, so a new push still supersedes its predecessor, while a
+# scheduled run at the tip of main never cancels that commit's push run.
 concurrency:
-  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-${{ github.event.action == 'labeled' && github.event.label.name || 'commit' }}
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-${{ github.event.action == 'labeled' && github.event.label.name || github.event_name }}
   cancel-in-progress: true
 
 on:
@@ -62,6 +70,11 @@ on:
   push:
     branches:
       - main
+  schedule:
+    # 06:00 UTC daily: after the evening's merges in the Americas and before
+    # the working day starts in Europe, so a red nightly is waiting when the
+    # first people look. Miri runs at 04:00 and the snapshot publish at 03:00.
+    - cron: '0 6 * * *'
   workflow_dispatch:
 
 jobs:
@@ -159,6 +172,11 @@ jobs:
     name: Detect changes
     needs: preflight
     runs-on: ubuntu-slim
+    # `actions: read` is for dev/ci/nightly-base.py, which lists previous
+    # scheduled runs to pick the nightly's diff base.
+    permissions:
+      actions: read
+      contents: read
     outputs:
       build_linux:  ${{ steps.compute.outputs.build_linux }}
       build_linux_full: ${{ steps.compute.outputs.build_linux_full }}
@@ -198,6 +216,7 @@ jobs:
           MQ_HEAD_SHA:  ${{ github.event.merge_group.head_sha }}
           PUSH_BEFORE:  ${{ github.event.before }}
           PUSH_AFTER:   ${{ github.sha }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           set -euo pipefail
           : > changed_files.txt
@@ -206,6 +225,29 @@ jobs:
             # every output true for this event so a manual run can exercise
             # any gated job.
             :
+          elif [[ "$EVENT_NAME" == "schedule" ]]; then
+            # The nightly's base is the commit the last successful scheduled
+            # run tested, so everything that landed since is covered exactly
+            # once and a red nightly keeps its commits in scope until a green
+            # one supersedes it. A quiet day diffs to nothing and runs
+            # nothing; so does a docs-only day.
+            #
+            # Without a base -- the first nightly, an API error, or a base no
+            # longer on main -- run the whole tier by treating every tracked
+            # file as changed, the same way the first push to a branch does
+            # below. Guessing a narrower base would be worse than running
+            # nothing: a window that starts after a commit no nightly has
+            # covered yet skips the suites that commit needs, goes green
+            # anyway, and then becomes the base for tomorrow, dropping that
+            # coverage for good.
+            prev=$(python3 dev/ci/nightly-base.py)
+            if [[ -n "$prev" ]] && git merge-base --is-ancestor "$prev" HEAD 2>/dev/null; then
+              echo "Nightly base: $prev"
+              git diff --name-only "$prev"..HEAD > changed_files.txt
+            else
+              echo "Nightly base unknown; running the whole nightly tier"
+              git ls-tree -r --name-only HEAD > changed_files.txt
+            fi
           elif [[ "$EVENT_NAME" == "pull_request" ]]; then
             git diff --name-only "$PR_BASE_SHA"..."$PR_HEAD_SHA" > changed_files.txt
           elif [[ "$EVENT_NAME" == "merge_group" ]]; then
@@ -248,21 +290,22 @@ jobs:
     #
     #   pull request          linux, full            -> profiles: pr
     #   ... with the label    linux, full, all       -> profiles: all
-    #   `labeled` run         all                    -> profiles: queue-only
-    #   merge queue           linux, full, all       -> profiles: all
+    #   `labeled` run         all                    -> profiles: nightly
+    #   merge queue           linux, full            -> profiles: pr
+    #   nightly               all                    -> profiles: nightly
     #   push to main          linux                  -> cache-refresh-only
     #
     # Only push to main sets `build_linux` without `build_linux_full`, which is
     # the whole point: the queue has already tested that tree, so the push run
-    # is there for the caches alone. A `labeled` run sets only the third, and
-    # then runs just the profiles the PR tier skipped.
+    # is there for the caches alone. A `labeled` run and the nightly set only
+    # the third, and then run just the profiles the PR and queue tiers skip.
     if: needs.changes.outputs.build_linux == 'true' || needs.changes.outputs.build_linux_all_profiles == 'true'
     uses: ./.github/workflows/pr_build_linux.yml
     with:
       cache-refresh-only: ${{ needs.changes.outputs.build_linux_full != 'true' && needs.changes.outputs.build_linux_all_profiles != 'true' }}
       profiles: >-
         ${{ needs.changes.outputs.build_linux_all_profiles != 'true' && 'pr'
-            || needs.changes.outputs.build_linux_full != 'true' && 'queue-only'
+            || needs.changes.outputs.build_linux_full != 'true' && 'nightly'
             || 'all' }}
 
   pr_build_macos:
@@ -332,7 +375,7 @@ jobs:
   spark_3_5:
     name: Spark SQL Tests (Spark 3.5)
     needs: changes
-    # Queue-only by default; PRs need the `run-spark-3.5-tests` label.
+    # Nightly by default; PRs need the `run-spark-3.5-tests` label.
     if: needs.changes.outputs.spark_3_5 == 'true'
     uses: ./.github/workflows/spark_sql_test_reusable.yml
     with:
@@ -343,9 +386,8 @@ jobs:
   spark_4_0:
     name: Spark SQL Tests (Spark 4.0)
     needs: changes
-    # Queue-only by default; PRs need the `run-spark-4.0-tests` label. Swapped
-    # with spark_4_1 on the `oom` branch to validate the memory caps against
-    # Spark 4.1 by default.
+    # Nightly by default; PRs need the `run-spark-4.0-tests` label. Spark 4.1
+    # is the one Spark SQL suite the queue runs, being the default profile.
     if: needs.changes.outputs.spark_4_0 == 'true'
     uses: ./.github/workflows/spark_sql_test_reusable.yml
     with:
@@ -376,7 +418,7 @@ jobs:
   iceberg_1_8:
     name: Iceberg Spark SQL Tests (Iceberg 1.8)
     needs: changes
-    # Queue-only by default; PRs need the `run-iceberg-tests` label.
+    # Nightly by default; PRs need the `run-iceberg-tests` label.
     if: needs.changes.outputs.iceberg_1_8 == 'true'
     uses: ./.github/workflows/iceberg_spark_test_reusable.yml
     with:
@@ -389,7 +431,7 @@ jobs:
   iceberg_1_9:
     name: Iceberg Spark SQL Tests (Iceberg 1.9)
     needs: changes
-    # Queue-only by default; PRs need the `run-iceberg-tests` label.
+    # Nightly by default; PRs need the `run-iceberg-tests` label.
     if: needs.changes.outputs.iceberg_1_9 == 'true'
     uses: ./.github/workflows/iceberg_spark_test_reusable.yml
     with:
@@ -402,7 +444,7 @@ jobs:
   iceberg_1_10:
     name: Iceberg Spark SQL Tests (Iceberg 1.10)
     needs: changes
-    # Queue-only by default; PRs need the `run-iceberg-tests` label.
+    # Nightly by default; PRs need the `run-iceberg-tests` label.
     if: needs.changes.outputs.iceberg_1_10 == 'true'
     uses: ./.github/workflows/iceberg_spark_test_reusable.yml
     with:
@@ -416,8 +458,8 @@ jobs:
     name: Iceberg Spark SQL Tests (Iceberg 1.11)
     needs: changes
     # Queue-only by default; PRs need the `run-iceberg-tests` label. Iceberg
-    # 1.11 is our only Spark 4.1 Iceberg coverage, and it was the PR-tier
-    # Iceberg job for that reason until every suite moved behind the queue.
+    # 1.11 is our only Spark 4.1 Iceberg coverage, which is why it is the one
+    # Iceberg version the queue runs while 1.8/1.9/1.10 run nightly.
     if: needs.changes.outputs.iceberg_1_11 == 'true'
     uses: ./.github/workflows/iceberg_spark_test_reusable.yml
     with:
@@ -493,3 +535,67 @@ jobs:
         run: |
           echo "::error::One or more upstream jobs did not succeed. See the results above."
           exit 1
+
+  # ---------------------------------------------------------------------------
+  # nightly_report: a red nightly has no pull request to show up on, and a
+  # scheduled run only emails whoever last touched the workflow file, so
+  # without this a regression in the nightly tier sits unnoticed on the
+  # Actions tab. On the scheduled event, and only when `required_checks` is
+  # not green, it opens an issue labelled `ci-nightly-failure` listing the
+  # failed jobs, or comments on the one that is already open so consecutive
+  # red nights accumulate in one place instead of one issue a day. Closing
+  # the issue is how the failure is acknowledged; the next red night opens a
+  # new one.
+  #
+  # It sits downstream of the aggregator rather than of the nightly jobs
+  # themselves so the job list here does not have to be kept in step with
+  # POLICY.
+  # ---------------------------------------------------------------------------
+  nightly_report:
+    name: Nightly failure report
+    needs: required_checks
+    if: always() && github.event_name == 'schedule' && needs.required_checks.result != 'success'
+    permissions:
+      actions: read
+      issues: write
+    runs-on: ubuntu-slim
+    steps:
+      - uses: actions/github-script@v9
+        with:
+          script: |
+            const label = 'ci-nightly-failure';
+            const { owner, repo } = context.repo;
+            const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`;
+            const shortSha = context.sha.slice(0, 10);
+            const day = new Date().toISOString().slice(0, 10);
+
+            const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
+              owner, repo, run_id: context.runId, filter: 'latest', per_page: 100,
+            });
+            const failed = jobs
+              .filter((job) => job.conclusion === 'failure' || job.conclusion === 'cancelled')
+              .map((job) => `- [${job.name}](${job.html_url}) (${job.conclusion})`);
+
+            const body = [
+              `The nightly CI run against \`main\` at ${shortSha} did not pass: ${runUrl}`,
+              '',
+              failed.length ? 'Jobs that did not succeed:' : 'No individual job reported a failure; see the run for details.',
+              ...failed,
+              '',
+              'The nightly tier runs the Spark and Iceberg versions the merge queue does not',
+              '(see `POLICY` in `dev/ci/compute-changes.py`). Close this issue once the failure',
+              'is understood; the next red nightly opens a new one.',
+            ].join('\n');
+
+            const { data: open } = await github.rest.issues.listForRepo({
+              owner, repo, labels: label, state: 'open', per_page: 1,
+            });
+            if (open.length > 0) {
+              await github.rest.issues.createComment({ owner, repo, issue_number: open[0].number, body });
+              core.info(`Commented on #${open[0].number}`);
+            } else {
+              const { data: issue } = await github.rest.issues.create({
+                owner, repo, title: `Nightly CI failed on ${day}`, body, labels: [label],
+              });
+              core.info(`Opened #${issue.number}`);
+            }