From e06d7f307c35d9ea7c709f476ce8c7b03d2fec19 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 4 Sep 2026 15:55:25 -0500 Subject: [PATCH 1/2] ci: run the NVHPC container lanes without MPI Every failure these lanes have produced has been an MPI test. post_process segfaults under mpirun on the GitHub-hosted runners -- "3D -> 2 MPI Ranks" (CE232828) and "MPI Consistency -> 3D -> Viscous" (0090B316) -- while the same tests pass on the GNU and Intel lanes and on the self-hosted clusters that run MPI at scale. The harness already skips ppn>1 cases when built without MPI (toolchain/mfc/test/test.py), and both failing tests are ppn=2, so --no-mpi drops exactly those and keeps the rest of --test-all. That is what these lanes are for: compile coverage across 15 NVHPC releases, where breaks have reached CI before. MPI runtime coverage is unaffected elsewhere. Only the cpu lanes change; the gpu lanes are build-only and never ran tests. --- .github/workflows/test.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 20ef3348b..9adbb006f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -289,12 +289,21 @@ jobs: CHANGED_FILES: ${{ needs.file-changes.outputs.changed_files }} # ── NVHPC build + test (via docker exec into long-lived container) ── + # --no-mpi: every failure these lanes have produced has been an MPI test. + # post_process segfaults under mpirun on the GitHub-hosted runners -- + # "3D -> 2 MPI Ranks" and "MPI Consistency -> 3D -> Viscous" -- while the + # same tests pass everywhere else, including the self-hosted clusters that + # actually run MPI at scale. The harness skips ppn>1 cases without MPI + # (test.py), so this drops exactly those and keeps the rest of --test-all, + # which is what these lanes exist for: compile coverage across 15 NVHPC + # releases. MPI runtime coverage stays on the GNU, Intel and self-hosted + # lanes. - name: Build (NVHPC) if: matrix.nvhpc && matrix.target == 'cpu' run: | docker exec nvhpc bash -c ' source /etc/nvhpc-env.sh - /bin/bash mfc.sh test -v --dry-run -j $(nproc) --test-all + /bin/bash mfc.sh test -v --dry-run -j $(nproc) --test-all --no-mpi ' - name: Build (NVHPC GPU) @@ -336,7 +345,7 @@ jobs: docker exec nvhpc bash -c ' source /etc/nvhpc-env.sh ulimit -s unlimited || ulimit -s 65536 || true - /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) --test-all + /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) --test-all --no-mpi ' # ── Cleanup ───────────────────────────────────────────────────────── From 28f72b26e936e02ab5c01a4eee60412cd39b3694 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Fri, 4 Sep 2026 16:18:08 -0500 Subject: [PATCH 2/2] ci: set the NVHPC test flags in one place Addresses review on #1822: --test-all --no-mpi was repeated in the build and test steps, so the two could drift. They now come from MFC_NVHPC_TEST_FLAGS, passed into the long-lived container at docker run so it expands inside the container, where these commands are evaluated. Divergence here is not cosmetic: a no-MPI build tested with MPI would run ppn>1 cases against a binary that has none. Also trims the rationale comment to the essentials and points at the PR for the incident detail. --- .github/workflows/test.yml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9adbb006f..8c57bb559 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -187,6 +187,7 @@ jobs: -e "FFLAGS=-tp=px -Kieee -noswitcherror" \ -e CFLAGS=-tp=px \ -e CXXFLAGS=-tp=px \ + -e "MFC_NVHPC_TEST_FLAGS=--test-all --no-mpi" \ "$NVHPC_IMAGE" sleep infinity - name: Setup NVHPC @@ -289,21 +290,19 @@ jobs: CHANGED_FILES: ${{ needs.file-changes.outputs.changed_files }} # ── NVHPC build + test (via docker exec into long-lived container) ── - # --no-mpi: every failure these lanes have produced has been an MPI test. - # post_process segfaults under mpirun on the GitHub-hosted runners -- - # "3D -> 2 MPI Ranks" and "MPI Consistency -> 3D -> Viscous" -- while the - # same tests pass everywhere else, including the self-hosted clusters that - # actually run MPI at scale. The harness skips ppn>1 cases without MPI - # (test.py), so this drops exactly those and keeps the rest of --test-all, - # which is what these lanes exist for: compile coverage across 15 NVHPC - # releases. MPI runtime coverage stays on the GNU, Intel and self-hosted - # lanes. + # MFC_NVHPC_TEST_FLAGS carries --no-mpi: post_process segfaults under + # mpirun in these containers, and every failure the lanes have produced + # has been an MPI test (see PR #1822). The harness skips ppn>1 cases + # without MPI, so the rest of --test-all -- the compile coverage these + # lanes exist for -- is unaffected, as is MPI coverage elsewhere. Set in + # one place so the build and test steps cannot disagree: a no-MPI build + # tested with MPI would run ppn>1 cases against a binary that has none. - name: Build (NVHPC) if: matrix.nvhpc && matrix.target == 'cpu' run: | docker exec nvhpc bash -c ' source /etc/nvhpc-env.sh - /bin/bash mfc.sh test -v --dry-run -j $(nproc) --test-all --no-mpi + /bin/bash mfc.sh test -v --dry-run -j $(nproc) $MFC_NVHPC_TEST_FLAGS ' - name: Build (NVHPC GPU) @@ -345,7 +344,7 @@ jobs: docker exec nvhpc bash -c ' source /etc/nvhpc-env.sh ulimit -s unlimited || ulimit -s 65536 || true - /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) --test-all --no-mpi + /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) $MFC_NVHPC_TEST_FLAGS ' # ── Cleanup ─────────────────────────────────────────────────────────