[CI probe, do not merge] CCE OpenACC EOS helper diagnostics

.claude/rules/common-pitfalls.md

@@ -60,6 +60,15 @@ covered in `docs/documentation/contributing.md`.
   QBMM/viscous and MHD HLLD, while both Lagrange bubble cases *complete* with out-of-tolerance
   answers. Measured 2026-08-29 on MI210. A compile-only check returns green, so any future attempt to
   drop these must run the tests, not just build.
+- Pass **scalars** into device routines, never an element of a device-resident array: a `declare create`
+  module array (`blkmod1(k,l,q)`, `gammas(i)`), an attached field (`q_prim_vf%vf(i)%sf(j,k,l)`), or a
+  dummy sized by a device global (`alpha_K(i)` with `dimension(num_fluids)`). Copy the element to a
+  local first and receive outputs into a local. CCE OpenACC compiles it clean and, unless it inlines
+  the callee, reads and writes the element at the wrong address: on PR #1811 a kernel wrote
+  `blkmod1(:,:,:)` through such a call and every cell stayed 0 while the host got 1.4 from the same
+  fields; every test through the path ended in `NaN(s) in timestep output` while CCE OpenMP, amdflang
+  and nvfortran were bit-identical. A leaf callee that gets inlined hides the bug, which is why the
+  parent PR passed. Expressions (`a/max(b, eps)`) are temporaries and are fine.
 - The same "call it from the loop body" rule covers `m_thermochem`: calling `get_species_*` from
   inside a `GPU_ROUTINE` rather than from the kernel gave CCE OpenMP a runtime
   `Memory access fault by GPU node-N ... Reason: Unknown` on the first step (exit 134), while every

.github/scripts/retry-build.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 # Provides retry_build(): 2-attempt loop.
-# On failure of attempt 1, nukes the entire build directory before attempt 2.
+# On failure of attempt 1, nukes the build directory before attempt 2, keeping
+# build/venv: a compute node cannot reinstall it (no route to PyPI), and a
+# failed reinstall is misread as a cluster-wide outage (#1813).
 # If RETRY_VALIDATE_CMD is set, runs it after a successful build; a non-zero
 # exit triggers the same nuke-and-retry, catching e.g. SIGILL from binaries
 # compiled on a different CPU architecture.
@@ -12,6 +14,10 @@
 # path without waiting on it; CI leaves it at the default.
 : "${MFC_BUILD_RETRY_DELAY:=30}"
 
+nuke_build() {
+    find build -mindepth 1 -maxdepth 1 ! -name venv -exec rm -rf {} + 2>/dev/null || true
+}
+
 retry_build() {
     local max_attempts=2
     local validate_cmd="${RETRY_VALIDATE_CMD:-}"
@@ -24,7 +30,7 @@ retry_build() {
                     echo "Post-build validation failed on attempt $attempt."
                     if [ $attempt -lt $max_attempts ]; then
                         echo "  Nuking build directory before retry..."
-                        rm -rf build 2>/dev/null || true
+                        nuke_build
                         sleep 5
                         attempt=$((attempt + 1))
                         continue
@@ -39,7 +45,7 @@ retry_build() {
         fi
         if [ $attempt -lt $max_attempts ]; then
             echo "  Build failed — nuking build directory before retry..."
-            rm -rf build 2>/dev/null || true
+            nuke_build
             sleep "$MFC_BUILD_RETRY_DELAY"
         else
             echo "Build failed after $max_attempts attempts."

.github/workflows/common/test.sh

@@ -74,9 +74,11 @@ fi
 # fall back to run-all). Pushes to master run the full suite as a backstop. Changed files
 # come from git detection (self-healing deepen) since the SLURM job doesn't receive the
 # paths-filter list.
-select_opts=""
-if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ]; then
-    select_opts="--select-enforce"
-fi
-
-./mfc.sh test -v --max-attempts 3 --no-build $select_opts -a -j $n_test_threads $rdma_opts $device_opts $build_opts $shard_opts -- -c $job_cluster
+# Probe: the reacting shocktube three times on one binary, one attempt each, so the
+# log carries three candidate values: identical means codegen, varying means a race.
+rc=0
+for pass_no in 1 2 3; do
+    echo "=== probe pass $pass_no ==="
+    ./mfc.sh test -v --max-attempts 1 --no-build --only 259B2428 -a -j $n_test_threads $rdma_opts $device_opts $build_opts $shard_opts -- -c $job_cluster || rc=1
+done
+exit $rc

.github/workflows/test.yml

@@ -68,11 +68,7 @@ jobs:
   github:
     name: ${{ matrix.nvhpc && format('NVHPC {0} ({1})', matrix.nvhpc, matrix.target) || format('Github ({0}, {1}, {2}, {3})', matrix.os, matrix.mpi, matrix.debug, matrix.intel && 'intel' || 'GNU') }}
     needs: [lint-gate, file-changes]
-    if: >-
-      !cancelled() &&
-      needs.lint-gate.result == 'success' &&
-      needs.file-changes.result == 'success' &&
-      needs.file-changes.outputs.checkall == 'true'
+    if: false
     strategy:
       matrix:
         os:    ['ubuntu', 'macos']
@@ -362,77 +358,12 @@ jobs:
     strategy:
       matrix:
         include:
-          # Phoenix (GT) — build+test combined in SLURM job
-          - runner:       'gt'
-            cluster:      'phoenix'
-            cluster_name: 'Georgia Tech | Phoenix (NVHPC)'
-            device: 'gpu'
-            interface: 'acc'
-          - runner:       'gt'
-            cluster:      'phoenix'
-            cluster_name: 'Georgia Tech | Phoenix (NVHPC)'
-            device: 'gpu'
-            interface: 'omp'
-          - runner:       'gt'
-            cluster:      'phoenix'
-            cluster_name: 'Georgia Tech | Phoenix (GNU)'
-            device: 'cpu'
-            interface: 'none'
-          # Frontier (ORNL) — CCE
-          - runner:       'frontier'
-            cluster:      'frontier'
-            cluster_name: 'Oak Ridge | Frontier (CCE)'
-            device: 'gpu'
-            interface: 'acc'
-            shard: '1/2'
-          - runner:       'frontier'
-            cluster:      'frontier'
-            cluster_name: 'Oak Ridge | Frontier (CCE)'
-            device: 'gpu'
-            interface: 'acc'
-            shard: '2/2'
           - runner:       'frontier'
             cluster:      'frontier'
             cluster_name: 'Oak Ridge | Frontier (CCE)'
             device: 'gpu'
             interface: 'omp'
-            shard: '1/2'
-          - runner:       'frontier'
-            cluster:      'frontier'
-            cluster_name: 'Oak Ridge | Frontier (CCE)'
-            device: 'gpu'
-            interface: 'omp'
-            shard: '2/2'
-          - runner:       'frontier'
-            cluster:      'frontier'
-            cluster_name: 'Oak Ridge | Frontier (CCE)'
-            device: 'cpu'
-            interface: 'none'
-          # Frontier AMD — build on login node, GPU tests sharded for batch partition
-          - runner:       'frontier'
-            cluster:      'frontier_amd'
-            cluster_name: 'Oak Ridge | Frontier (AMD)'
-            device: 'gpu'
-            interface: 'omp'
-            shard: '1/2'
-          - runner:       'frontier'
-            cluster:      'frontier_amd'
-            cluster_name: 'Oak Ridge | Frontier (AMD)'
-            device: 'gpu'
-            interface: 'omp'
-            shard: '2/2'
-          - runner:       'frontier'
-            cluster:      'frontier_amd'
-            cluster_name: 'Oak Ridge | Frontier (AMD)'
-            device: 'cpu'
-            interface: 'none'
-            shard: '1/2'
-          - runner:       'frontier'
-            cluster:      'frontier_amd'
-            cluster_name: 'Oak Ridge | Frontier (AMD)'
-            device: 'cpu'
-            interface: 'none'
-            shard: '2/2'
+            shard: ''
     runs-on:
       group:  phoenix
       labels: ${{ matrix.runner }}
@@ -532,7 +463,7 @@ jobs:
 
   case-optimization:
     name: "Case Opt | ${{ matrix.cluster_name }} (${{ matrix.device }}-${{ matrix.interface }})"
-    if: github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true' && github.event.pull_request.draft != true
+    if: false
     needs: [lint-gate, file-changes]
     # Frontier is non-blocking for the same reason as the self job above.
     continue-on-error: ${{ matrix.runner == 'frontier' }}

docs/documentation/case.md

@@ -732,6 +732,7 @@ To restart the simulation from $k$-th time step, see @ref running "Restarting Ca
 | `pi_inf_wrt`            | Logical | Add the liquid stiffness function to the database |
 | `pres_inf_wrt`          | Logical | Add the liquid stiffness to the formatted database	 |
 | `c_wrt`                 | Logical | Add the sound speed to the database	 |
+| `T_wrt`                 | Logical | Add each fluid's temperature to the database (needs `cv` > 0) |
 | `omega_wrt(i)`          | Logical | Add the $i$-direction vorticity to the database	 |
 | `schlieren_wrt`         | Logical | Add the numerical schlieren to the database|
 | `qm_wrt`                | Logical | Add the Q-criterion to the database|

docs/documentation/contributing.md

@@ -463,8 +463,8 @@ means supplying these, not grepping for `gammas`:
 | `f_pressure` / `s_compute_energy` | \f$p(e)\f$ and \f$E(p)\f$ |
 | `f_bulk_modulus` | \f$K(p)\f$ - every sound speed in MFC is \f$K/\rho\f$, differing only in how phases are mixed |
 | `s_compute_speed_of_sound` / `_avg` | that mixing: Wood's law, 6-equation, bubble-diluted |
-| `f_phase_internal_energy` | per-phase internal energy (6-equation model) |
-| `f_isentrope_exponent` / `f_isentrope_pressure` / `f_pressure_on_isentrope` | the isentrope \f$p + B = \textrm{const}\,\rho^n\f$ |
+| `s_phase_internal_energy` | per-phase internal energy (6-equation model) |
+| `f_isentrope_exponent` / `f_isentrope_pressure` | the isentrope \f$p + B = \textrm{const}\,\rho^n\f$ |
 | `f_sg_thermal` | the thermal law \f$p + B = (n-1)c_v\rho T\f$ |
 
 The first six are *mechanical* - they need only \f$p, \rho, e, c\f$. The last two are *caloric* and

examples/1D_isentropic_release/case.py

@@ -0,0 +1,72 @@
+"""
+Isentropic release of a fluid whose reference curve is an isentrope (JWL or Vinet): a Riemann problem
+between two states of the same fluid. Everything left of the contact keeps the left state's entropy,
+so the fan and the left star state must lie on the closed-form isentrope through (rho0, p0).
+"""
+
+import argparse
+import json
+import math
+
+parser = argparse.ArgumentParser(description="1D JWL isentropic release")
+parser.add_argument("--mfc", type=json.loads, default="{}", metavar="DICT")
+parser.add_argument("-N", type=int, default=400)
+parser.add_argument("--cfl", type=float, default=0.4)
+parser.add_argument("--eos", choices=["jwl", "vinet"], default="jwl", help="reference curve: both are isentropes")
+args = parser.parse_args()
+
+rho0, p0, rho_r, p_r = 1.0, 1.0, 0.3, 0.1
+if args.eos == "jwl":
+    fluid = {f"fluid_pp(1)%jwl_{k}": v for k, v in {"a": 6.0, "b": 0.15, "r1": 4.0, "r2": 1.0, "omega": 0.3, "rho0": rho0}.items()}
+else:
+    fluid = {f"fluid_pp(1)%vinet_{k}": v for k, v in {"k0": 2.0, "k0p": 4.0, "gruneisen": 0.3, "rho0": rho0}.items()}
+N, L, T_end = args.N, 1.0, 0.15
+c_max = math.sqrt((1.3 * p0 + 6.15) / rho0)  # generous bound on c + |u| for either fit
+dt = args.cfl * (L / N) / c_max
+Nt = math.ceil(T_end / dt)
+dt = T_end / Nt
+
+case = {
+    "run_time_info": "F",
+    "x_domain%beg": 0.0,
+    "x_domain%end": L,
+    "m": N - 1,
+    "n": 0,
+    "p": 0,
+    "dt": dt,
+    "t_step_start": 0,
+    "t_step_stop": Nt,
+    "t_step_save": Nt,
+    "num_patches": 2,
+    "model_eqns": 2,
+    "num_fluids": 1,
+    "time_stepper": 3,
+    "recon_type": "weno",
+    "weno_order": 5,
+    "weno_eps": 1.0e-16,
+    "mapped_weno": "T",
+    "riemann_solver": 2,
+    "wave_speeds": 1,
+    "avg_state": 2,
+    "bc_x%beg": -3,
+    "bc_x%end": -3,
+    "format": 1,
+    "precision": 2,
+    "prim_vars_wrt": "T",
+    "parallel_io": "F",
+    "fluid_pp(1)%eos": args.eos,
+    **fluid,
+}
+for pid, (x_c, rho, pres) in enumerate([(0.25, rho0, p0), (0.75, rho_r, p_r)], start=1):
+    case.update(
+        {
+            f"patch_icpp({pid})%geometry": 1,
+            f"patch_icpp({pid})%x_centroid": x_c,
+            f"patch_icpp({pid})%length_x": 0.5 * L,
+            f"patch_icpp({pid})%alpha_rho(1)": rho,
+            f"patch_icpp({pid})%alpha(1)": 1.0,
+            f"patch_icpp({pid})%vel(1)": 0.0,
+            f"patch_icpp({pid})%pres": pres,
+        }
+    )
+print(json.dumps(case))

examples/1D_mg_acoustic/case.py

@@ -0,0 +1,81 @@
+"""
+Right-moving acoustic pulse in a single Mie-Gruneisen fluid at its reference state.
+A rectangle patch carries a simple-wave perturbation (drho, dp = c^2 drho, du = c drho/rho0), so
+only the right-going characteristic is excited; the harness tracks the centroid of drho against
+the general analytic c. Patches rather than an analytic IC: an IC expression is compiled in, and
+every distinct one costs the test suite a full rebuild.
+"""
+
+import argparse
+import json
+import math
+
+parser = argparse.ArgumentParser(description="1D Mie-Gruneisen acoustic pulse")
+parser.add_argument("--mfc", type=json.loads, default="{}", metavar="DICT")
+parser.add_argument("-N", type=int, default=200)
+parser.add_argument("--cfl", type=float, default=0.4)
+parser.add_argument("--a", type=float, default=0.0, help="Gruneisen slope: Gamma_G = Gamma_0 + a mu")
+args = parser.parse_args()
+
+rho0, p0, c0, s, gruneisen = 1.0, 1.0, 1.0, 1.5, 0.4
+c = math.sqrt(c0**2 + (1.0 + gruneisen) * p0 / rho0 + args.a * p0 / (rho0 * gruneisen))  # the frozen speed at the reference state
+amp, x0, width = 1.0e-4, 0.25, 0.1
+N, L, T_end = args.N, 1.0, 0.3
+dt = args.cfl * (L / N) / c
+Nt = math.ceil(T_end / dt)
+dt = T_end / Nt
+
+print(
+    json.dumps(
+        {
+            "run_time_info": "F",
+            "x_domain%beg": 0.0,
+            "x_domain%end": L,
+            "m": N - 1,
+            "n": 0,
+            "p": 0,
+            "dt": dt,
+            "t_step_start": 0,
+            "t_step_stop": Nt,
+            "t_step_save": Nt,
+            "model_eqns": 2,
+            "num_fluids": 1,
+            "time_stepper": 3,
+            "recon_type": "weno",
+            "weno_order": 5,
+            "weno_eps": 1.0e-16,
+            "mapped_weno": "T",
+            "riemann_solver": 2,
+            "wave_speeds": 1,
+            "avg_state": 2,
+            "bc_x%beg": -3,
+            "bc_x%end": -3,
+            "format": 1,
+            "precision": 2,
+            "prim_vars_wrt": "T",
+            "parallel_io": "F",
+            "num_patches": 2,
+            "patch_icpp(1)%geometry": 1,
+            "patch_icpp(1)%x_centroid": 0.5,
+            "patch_icpp(1)%length_x": L,
+            "patch_icpp(1)%alpha_rho(1)": rho0,
+            "patch_icpp(1)%alpha(1)": 1.0,
+            "patch_icpp(1)%vel(1)": 0.0,
+            "patch_icpp(1)%pres": p0,
+            "patch_icpp(2)%geometry": 1,
+            "patch_icpp(2)%alter_patch(1)": "T",
+            "patch_icpp(2)%x_centroid": x0,
+            "patch_icpp(2)%length_x": width,
+            "patch_icpp(2)%alpha_rho(1)": rho0 + amp,
+            "patch_icpp(2)%alpha(1)": 1.0,
+            "patch_icpp(2)%vel(1)": c / rho0 * amp,
+            "patch_icpp(2)%pres": p0 + c**2 * amp,
+            "fluid_pp(1)%eos": "mie_gruneisen",
+            "fluid_pp(1)%mg_rho0": rho0,
+            "fluid_pp(1)%mg_c0": c0,
+            "fluid_pp(1)%mg_s": s,
+            "fluid_pp(1)%mg_gruneisen": gruneisen,
+            **({"fluid_pp(1)%mg_gruneisen_a": args.a} if args.a else {}),
+        }
+    )
+)