From e42d2f89ab23af3f468e167eb908f2163046b32f Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 5 Sep 2026 14:12:55 -0400 Subject: [PATCH 1/4] ci: cap NVHPC lane concurrency with max-parallel The 30 NVHPC lanes dispatched all at once and occupied 30 standard-runner slots for roughly two hours, leaving every other workflow queued behind them. Minutes are free on this repo; concurrency is the scarce resource. `max-parallel` is the direct control, but it applies to an entire job matrix, and the NVHPC entries shared the `github` job with the ubuntu and macOS lanes -- throttling in place would have throttled those too. So the NVHPC entries move to their own `nvhpc` job that carries the cap, and the 30 hand-written include: entries collapse to a two-axis product. A matrix entry queued behind max-parallel holds no runner, so this trades wall-clock for slots without dropping any compiler coverage: all 15 releases from 23.11 through 26.3 still build, cpu still tests, gpu still builds acc then omp plus the case-optimized guard. It also staggers the nvcr.io pulls that the retry loop in the pull step exists to survive. Job names are unchanged ("NVHPC 25.5 (cpu)"), so no check-name churn. No job depends on `github`, so the split adds no ordering constraints. Claude-Session: https://claude.ai/code/session_01XPqfEaUBG7ZaZVzeMWnKHd --- .github/workflows/test.yml | 218 ++++++++++++++++++------------------- 1 file changed, 105 insertions(+), 113 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8c57bb559..3ab0a5b89 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,7 +66,7 @@ jobs: list-files: shell github: - name: ${{ matrix.nvhpc && format('NVHPC {0} ({1})', matrix.nvhpc, matrix.target) || format('Github ({0}, {1}, {2}, {3})', matrix.os, matrix.mpi, matrix.debug, matrix.intel && 'intel' || 'GNU') }} + name: ${{ format('Github ({0}, {1}, {2}, {3})', matrix.os, matrix.mpi, matrix.debug, matrix.intel && 'intel' || 'GNU') }} needs: [lint-gate, file-changes] if: >- !cancelled() && @@ -80,8 +80,6 @@ jobs: precision: [''] debug: ['reldebug', 'no-debug'] intel: [true, false] - nvhpc: [''] - target: [''] exclude: - os: macos intel: true @@ -93,50 +91,113 @@ jobs: debug: no-debug intel: false - # NVHPC compiler matrix: cpu (build+test), gpu (build-only, acc then omp) - # Every release from 23.11 through 26.3 (current) - - { nvhpc: '23.11', target: cpu } - - { nvhpc: '23.11', target: gpu } - - { nvhpc: '24.1', target: cpu } - - { nvhpc: '24.1', target: gpu } - - { nvhpc: '24.3', target: cpu } - - { nvhpc: '24.3', target: gpu } - - { nvhpc: '24.5', target: cpu } - - { nvhpc: '24.5', target: gpu } - - { nvhpc: '24.7', target: cpu } - - { nvhpc: '24.7', target: gpu } - - { nvhpc: '24.9', target: cpu } - - { nvhpc: '24.9', target: gpu } - - { nvhpc: '24.11', target: cpu } - - { nvhpc: '24.11', target: gpu } - - { nvhpc: '25.1', target: cpu } - - { nvhpc: '25.1', target: gpu } - - { nvhpc: '25.3', target: cpu } - - { nvhpc: '25.3', target: gpu } - - { nvhpc: '25.5', target: cpu } - - { nvhpc: '25.5', target: gpu } - - { nvhpc: '25.7', target: cpu } - - { nvhpc: '25.7', target: gpu } - - { nvhpc: '25.9', target: cpu } - - { nvhpc: '25.9', target: gpu } - - { nvhpc: '25.11', target: cpu } - - { nvhpc: '25.11', target: gpu } - - { nvhpc: '26.1', target: cpu } - - { nvhpc: '26.1', target: gpu } - - { nvhpc: '26.3', target: cpu } - - { nvhpc: '26.3', target: gpu } + fail-fast: false + continue-on-error: true + runs-on: ${{ format('{0}-latest', matrix.os) }} + + steps: + - name: Clone + uses: actions/checkout@v5 + + # ── Standard (non-NVHPC) setup ───────────────────────────────────── + - name: Setup MacOS + if: matrix.os == 'macos' + run: | + brew update + brew upgrade || true + brew install coreutils python fftw hdf5 gcc@15 boost open-mpi lapack + echo "FC=gfortran-15" >> $GITHUB_ENV + echo "BOOST_INCLUDE=/opt/homebrew/include/" >> $GITHUB_ENV + - name: Setup Ubuntu + if: matrix.os == 'ubuntu' && matrix.intel == false + run: | + sudo apt update -y + sudo apt install -y cmake gcc g++ python3 python3-dev hdf5-tools \ + libfftw3-dev libhdf5-dev openmpi-bin libopenmpi-dev \ + libblas-dev liblapack-dev + + - name: Setup Ubuntu (Intel) + if: matrix.os == 'ubuntu' && matrix.intel == true + run: | + wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" + sudo apt-get update + sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel + # Export only new/changed env vars from setvars.sh. + # `printenv >> $GITHUB_ENV` dumps all vars including shell internals + # with special characters that corrupt GITHUB_ENV parsing. + printenv | sort > /tmp/env_before + source /opt/intel/oneapi/setvars.sh + printenv | sort > /tmp/env_after + diff /tmp/env_before /tmp/env_after | grep '^>' | sed 's/^> //' >> $GITHUB_ENV + echo "FC=ifx" >> $GITHUB_ENV + echo "CC=icx" >> $GITHUB_ENV + echo "CXX=icpx" >> $GITHUB_ENV + echo "MPIFC=mpiifx" >> $GITHUB_ENV + echo "MPICC=mpiicx" >> $GITHUB_ENV + echo "MPICXX=mpiicpx" >> $GITHUB_ENV + + # ── Standard build + test ─────────────────────────────────────────── + - name: Build + run: | + /bin/bash mfc.sh test -v --dry-run -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} $PRECISION $TEST_ALL + env: + TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }} + PRECISION: ${{ matrix.precision != '' && format('--{0}', matrix.precision) || '' }} + + - name: Test + run: | + # Coverage-based test selection ENFORCED on PRs: runs only the tests whose + # recorded coverage overlaps the PR's changed files (conservative ladder in + # coverage.py — non-.fpp changes and .fpp files no test covers fall back to + # run-all). Pushes to master run the full suite (SELECT empty) as a backstop, + # and the nvhpc job still runs --test-all as a pre-merge full check. + SELECT=() + [ "${{ github.event_name }}" = "pull_request" ] && SELECT=(--select-enforce --changed-files "$CHANGED_FILES") + /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) "${SELECT[@]}" $TEST_ALL $TEST_PCT $PRECISION + env: + TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }} + TEST_PCT: ${{ matrix.debug == 'reldebug' && '-% 20' || '' }} + PRECISION: ${{ matrix.precision != '' && format('--{0}', matrix.precision) || '' }} + CHANGED_FILES: ${{ needs.file-changes.outputs.changed_files }} + + # Split out of the `github` job so max-parallel can throttle these lanes + # without also throttling the fast ubuntu/macos ones: max-parallel applies to + # a whole job matrix, and the two sets used to share one. + nvhpc: + name: NVHPC ${{ matrix.nvhpc }} (${{ matrix.target }}) + needs: [lint-gate, file-changes] + if: >- + !cancelled() && + needs.lint-gate.result == 'success' && + needs.file-changes.result == 'success' && + needs.file-changes.outputs.checkall == 'true' + strategy: fail-fast: false + # Cap concurrent runners. These 30 lanes are advisory compile coverage + # (continue-on-error), but dispatching them all at once consumed the + # account's standard-runner concurrency and left other workflows queued + # behind them. A queued matrix entry holds no runner, so this buys slots + # back at the cost of wall-clock. It also staggers the nvcr.io pulls that + # the retry loop below exists to survive. + max-parallel: 6 + matrix: + # NVHPC compiler matrix: cpu (build+test), gpu (build-only, acc then omp) + # Every release from 23.11 through 26.3 (current) + nvhpc: ['23.11', '24.1', '24.3', '24.5', '24.7', '24.9', '24.11', + '25.1', '25.3', '25.5', '25.7', '25.9', '25.11', + '26.1', '26.3'] + target: [cpu, gpu] continue-on-error: true - runs-on: ${{ matrix.nvhpc && 'ubuntu-22.04' || format('{0}-latest', matrix.os) }} + runs-on: ubuntu-22.04 env: - # Image tag for NVHPC jobs; empty for non-NVHPC jobs. - NVHPC_IMAGE: ${{ matrix.nvhpc && format('nvcr.io/nvidia/nvhpc:{0}-devel-cuda_multi-ubuntu22.04', matrix.nvhpc) || '' }} + NVHPC_IMAGE: nvcr.io/nvidia/nvhpc:${{ matrix.nvhpc }}-devel-cuda_multi-ubuntu22.04 steps: # ── NVHPC: free disk before pulling the ~25-30 GB cuda_multi image ── - name: Free disk space - if: matrix.nvhpc run: | echo "=== Disk before cleanup ===" df -h / @@ -160,7 +221,6 @@ jobs: # exceeded") under load, and ~30 matrix jobs hit it at once. Pulls # resume completed layers, so retries are cheap. - name: Pull NVHPC container - if: matrix.nvhpc run: | for attempt in 1 2 3 4 5; do docker pull "$NVHPC_IMAGE" && exit 0 @@ -171,7 +231,6 @@ jobs: exit 1 - name: Start NVHPC container - if: matrix.nvhpc run: | docker run -d --name nvhpc \ --security-opt seccomp=unconfined \ @@ -191,7 +250,6 @@ jobs: "$NVHPC_IMAGE" sleep infinity - name: Setup NVHPC - if: matrix.nvhpc run: | docker exec nvhpc bash -c ' set -e @@ -223,72 +281,6 @@ jobs: cat /proc/cpuinfo | grep "model name" | head -1 ' - # ── Standard (non-NVHPC) setup ───────────────────────────────────── - - name: Setup MacOS - if: matrix.os == 'macos' && !matrix.nvhpc - run: | - brew update - brew upgrade || true - brew install coreutils python fftw hdf5 gcc@15 boost open-mpi lapack - echo "FC=gfortran-15" >> $GITHUB_ENV - echo "BOOST_INCLUDE=/opt/homebrew/include/" >> $GITHUB_ENV - - - name: Setup Ubuntu - if: matrix.os == 'ubuntu' && matrix.intel == false && !matrix.nvhpc - run: | - sudo apt update -y - sudo apt install -y cmake gcc g++ python3 python3-dev hdf5-tools \ - libfftw3-dev libhdf5-dev openmpi-bin libopenmpi-dev \ - libblas-dev liblapack-dev - - - name: Setup Ubuntu (Intel) - if: matrix.os == 'ubuntu' && matrix.intel == true - run: | - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" - sudo apt-get update - sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi intel-oneapi-mpi-devel - # Export only new/changed env vars from setvars.sh. - # `printenv >> $GITHUB_ENV` dumps all vars including shell internals - # with special characters that corrupt GITHUB_ENV parsing. - printenv | sort > /tmp/env_before - source /opt/intel/oneapi/setvars.sh - printenv | sort > /tmp/env_after - diff /tmp/env_before /tmp/env_after | grep '^>' | sed 's/^> //' >> $GITHUB_ENV - echo "FC=ifx" >> $GITHUB_ENV - echo "CC=icx" >> $GITHUB_ENV - echo "CXX=icpx" >> $GITHUB_ENV - echo "MPIFC=mpiifx" >> $GITHUB_ENV - echo "MPICC=mpiicx" >> $GITHUB_ENV - echo "MPICXX=mpiicpx" >> $GITHUB_ENV - - # ── Standard build + test ─────────────────────────────────────────── - - name: Build - if: '!matrix.nvhpc' - run: | - /bin/bash mfc.sh test -v --dry-run -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} $PRECISION $TEST_ALL - env: - TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }} - PRECISION: ${{ matrix.precision != '' && format('--{0}', matrix.precision) || '' }} - - - name: Test - if: '!matrix.nvhpc' - run: | - # Coverage-based test selection ENFORCED on PRs: runs only the tests whose - # recorded coverage overlaps the PR's changed files (conservative ladder in - # coverage.py — non-.fpp changes and .fpp files no test covers fall back to - # run-all). Pushes to master run the full suite (SELECT empty) as a backstop, - # and the NVHPC jobs below still run --test-all as a pre-merge full check. - SELECT=() - [ "${{ github.event_name }}" = "pull_request" ] && SELECT=(--select-enforce --changed-files "$CHANGED_FILES") - /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) "${SELECT[@]}" $TEST_ALL $TEST_PCT $PRECISION - env: - TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }} - TEST_PCT: ${{ matrix.debug == 'reldebug' && '-% 20' || '' }} - PRECISION: ${{ matrix.precision != '' && format('--{0}', matrix.precision) || '' }} - CHANGED_FILES: ${{ needs.file-changes.outputs.changed_files }} - # ── NVHPC build + test (via docker exec into long-lived container) ── # MFC_NVHPC_TEST_FLAGS carries --no-mpi: post_process segfaults under # mpirun in these containers, and every failure the lanes have produced @@ -298,7 +290,7 @@ jobs: # one place so the build and test steps cannot disagree: a no-MPI build # tested with MPI would run ppn>1 cases against a binary that has none. - name: Build (NVHPC) - if: matrix.nvhpc && matrix.target == 'cpu' + if: matrix.target == 'cpu' run: | docker exec nvhpc bash -c ' source /etc/nvhpc-env.sh @@ -306,7 +298,7 @@ jobs: ' - name: Build (NVHPC GPU) - if: matrix.nvhpc && matrix.target == 'gpu' + if: matrix.target == 'gpu' run: | docker exec nvhpc bash -c ' source /etc/nvhpc-env.sh @@ -330,7 +322,7 @@ jobs: # case-optimized 2D build elides that code entirely. OpenACC only, # since the two-pass IPO is disabled for OpenMP offload. - name: Build (NVHPC GPU, case-optimized) - if: matrix.nvhpc && matrix.target == 'gpu' + if: matrix.target == 'gpu' run: | docker exec nvhpc bash -c ' source /etc/nvhpc-env.sh @@ -339,7 +331,7 @@ jobs: ' - name: Test (NVHPC) - if: matrix.nvhpc && matrix.target == 'cpu' + if: matrix.target == 'cpu' run: | docker exec nvhpc bash -c ' source /etc/nvhpc-env.sh @@ -349,7 +341,7 @@ jobs: # ── Cleanup ───────────────────────────────────────────────────────── - name: Stop NVHPC container - if: always() && matrix.nvhpc + if: always() run: docker rm -f nvhpc || true self: From 4acb93ce2ec9a49b02d7c646f0bee773c2bd5a8b Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 5 Sep 2026 14:27:13 -0400 Subject: [PATCH 2/4] ci: declare permissions in test.yml so the new nvhpc job is scoped CodeQL flagged the `nvhpc` job this PR adds for missing permissions. The job itself is fine -- it checks out and pulls a container, nothing more -- but this branch is based on master, which has no `permissions:` block, so every job in test.yml still inherits the repository default. Add the workflow-level block here rather than relying on #1825 landing first, so this PR clears the alert on its own in either merge order. The block is byte-identical to the one #1825 adds to this file, so the two merge without conflict (verified with git merge-tree). Claude-Session: https://claude.ai/code/session_01XPqfEaUBG7ZaZVzeMWnKHd --- .github/workflows/test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ab0a5b89..4655d1eb0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,6 +12,13 @@ concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: ${{ github.event_name != 'push' }} +# Least-privilege default: no job in this workflow writes to the repo. +# pull-requests: read is required by dorny/paths-filter, which reads the PR's +# changed-file list through the API on pull_request events. +permissions: + contents: read + pull-requests: read + jobs: lint-gate: name: Lint Gate From 73edd7792ed4554e3508aeb9779164c6211062cd Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 5 Sep 2026 14:33:05 -0400 Subject: [PATCH 3/4] ci: scope pull-requests: read to the file-changes job Mirrors the same change on #1825 so the two branches keep an identical permissions region in test.yml and continue to merge in either order. The nvhpc job this PR adds needs only contents: read -- it checks out and pulls a container image, using no GitHub token -- so it now inherits exactly that rather than a workflow-wide pull-requests grant it has no use for. Same for the self-hosted Frontier and Phoenix lanes. Claude-Session: https://claude.ai/code/session_01XPqfEaUBG7ZaZVzeMWnKHd --- .github/workflows/test.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4655d1eb0..29f35917f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,11 +13,8 @@ concurrency: cancel-in-progress: ${{ github.event_name != 'push' }} # Least-privilege default: no job in this workflow writes to the repo. -# pull-requests: read is required by dorny/paths-filter, which reads the PR's -# changed-file list through the API on pull_request events. permissions: contents: read - pull-requests: read jobs: lint-gate: @@ -57,6 +54,13 @@ jobs: file-changes: name: Detect File Changes + # Job-level permissions replace the workflow default outright rather than + # merging with it, so contents must be restated here. paths-filter reads the + # PR's changed-file list via pulls.listFiles; this is the only job that needs + # it, so it is granted here instead of workflow-wide. + permissions: + contents: read + pull-requests: read runs-on: 'ubuntu-latest' outputs: checkall: ${{ steps.changes.outputs.checkall }} From 0e5f9fd499e464080a10d99f05b7715c876abc06 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 5 Sep 2026 14:47:10 -0400 Subject: [PATCH 4/4] ci: drop --test-all from the NVHPC cpu test step The cpu lane spent about 102 of its ~120 minutes in `mfc.sh test`, running the full suite once per NVHPC release, 15 times over. --test-all does two unrelated things (toolchain/mfc/test/test.py). At build time it adds post_process to the compiled binaries (line 389). At test time it re-runs pre_process+simulation+post_process a second time for every case and post-processes the silo output (line 735). Only the first is the compile coverage these lanes exist for; the second roughly doubles the runtime and duplicates checking the GNU, Intel and self-hosted lanes already do. So the flag moves out of the shared MFC_NVHPC_TEST_FLAGS and onto the build step alone. post_process is still compiled on all 15 releases -- no loss of compiler coverage, which is the point of the matrix -- it just is not re-run here. --no-mpi stays shared, since that one genuinely must match between build and test. The gpu lanes are untouched: they already pass --test-all explicitly and never referenced the env var. Claude-Session: https://claude.ai/code/session_01XPqfEaUBG7ZaZVzeMWnKHd --- .github/workflows/test.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 29f35917f..6759ca0ee 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -257,7 +257,7 @@ jobs: -e "FFLAGS=-tp=px -Kieee -noswitcherror" \ -e CFLAGS=-tp=px \ -e CXXFLAGS=-tp=px \ - -e "MFC_NVHPC_TEST_FLAGS=--test-all --no-mpi" \ + -e "MFC_NVHPC_TEST_FLAGS=--no-mpi" \ "$NVHPC_IMAGE" sleep infinity - name: Setup NVHPC @@ -296,16 +296,25 @@ jobs: # MFC_NVHPC_TEST_FLAGS carries --no-mpi: post_process segfaults under # mpirun in these containers, and every failure the lanes have produced # has been an MPI test (see PR #1822). The harness skips ppn>1 cases - # without MPI, so the rest of --test-all -- the compile coverage these - # lanes exist for -- is unaffected, as is MPI coverage elsewhere. Set in - # one place so the build and test steps cannot disagree: a no-MPI build - # tested with MPI would run ppn>1 cases against a binary that has none. + # without MPI, so compile coverage is unaffected, as is MPI coverage + # elsewhere. It stays in one place because a no-MPI build tested with + # MPI would run ppn>1 cases against a binary that has none. + # + # --test-all is deliberately NOT shared, and sits on the build only. + # It does two separate things (toolchain/mfc/test/test.py): at build + # time it adds post_process to the set of compiled binaries, and at + # test time it re-runs pre_process+simulation+post_process a second + # time for every case. The first is the compile coverage these lanes + # exist for; the second is roughly half the lane's ~120 min wall clock + # and duplicates runtime checking the GNU, Intel and self-hosted lanes + # already do. So: build with it, test without it. post_process is + # still compiled on all 15 releases, just not re-run here. - name: Build (NVHPC) if: matrix.target == 'cpu' run: | docker exec nvhpc bash -c ' source /etc/nvhpc-env.sh - /bin/bash mfc.sh test -v --dry-run -j $(nproc) $MFC_NVHPC_TEST_FLAGS + /bin/bash mfc.sh test -v --dry-run -j $(nproc) --test-all $MFC_NVHPC_TEST_FLAGS ' - name: Build (NVHPC GPU)