diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 196cbc6..0fbdcaa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,12 +28,18 @@ jobs: cxx: clang++ werror: ON capi: ON - # Warnings stay non-fatal on MSVC until /W4 output has been triaged - # on a Windows runner (same policy as the sibling *Tap repos). + # macOS gates on the float32 backend MuTap actually uses, which is + # OOURA — the root CMakeLists turns DspTap's Apple vDSP default back + # off, with the measurements behind that decision recorded there and + # in docs/itu-compliance.md (tap/MuTap#31). No flag needed here: the + # gate follows the product's default rather than overriding it, so + # this leg cannot drift away from what ships. - name: macOS AppleClang os: macos-latest werror: ON capi: ON + # Warnings stay non-fatal on MSVC until /W4 output has been triaged + # on a Windows runner (same policy as the sibling *Tap repos). # capi stays OFF on Windows: tools/capi carries no __declspec(dllexport) # (unlike DspTap's), so an MSVC build would link a DLL exporting nothing # — it would pass without gating anything. Turn this ON in the same @@ -47,6 +53,17 @@ jobs: with: submodules: recursive + # Chip identity matters on Apple: #31's draw was observed on an M1 VM + # and does NOT reproduce on Intel, so a bare "macOS" label is not enough + # to attribute a past result. + - name: Record host identity (macOS) + if: runner.os == 'macOS' + run: | + sysctl -n machdep.cpu.brand_string + sysctl -n hw.model + sw_vers + cc --version | head -2 + - name: Configure env: CC: ${{ matrix.cc }} @@ -57,12 +74,35 @@ jobs: -DMUTAP_WERROR=${{ matrix.werror }} -DMUTAP_BUILD_CAPI=${{ matrix.capi }} + # Record the backend actually built, so a log can never be misread about + # which float32 configuration produced its numbers. + - name: Record float32 FFT backend + if: runner.os == 'macOS' + run: grep '^TAP_DSP_FFT_ACCELERATE' build/CMakeCache.txt || echo 'TAP_DSP_FFT_ACCELERATE not in cache (default)' + - name: Build run: cmake --build build --config Release -j 4 - name: Test run: ctest --test-dir build -C Release --output-on-failure + # These three rows were the per-process bifurcation in #31. Now that the + # alignment fix has removed the draw, repeating them says something + # different but still worth recording: whether the remaining failures are + # STABLE. A row that fails 20/20 is a deterministic property of the + # backend difference; one that still wanders means a second source of + # variation survives and the diagnosis is incomplete. + # + # always(), because the whole point is to characterize a failure — and a + # failing Test step above would otherwise skip it, which is exactly what + # happened on the first run that needed it. + - name: Repeat the float32 rows (stability, not a draw) + if: always() && runner.os == 'macOS' + run: > + ctest --test-dir build -C Release --output-on-failure + --repeat until-fail:20 + -R 'itu_echo\.EchoStability|g168_adapted\.ToneStability|Float32Parity\.ToneRowWithNarrowbandGuard' + sanitizers: name: ASan + UBSan runs-on: ubuntu-latest diff --git a/CMakeLists.txt b/CMakeLists.txt index cccdd94..0c15d65 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,10 +6,36 @@ project(MuTap VERSION 0.1.0 LANGUAGES C CXX) # Apple vDSP float32 backends, formerly vendored here (include/mutap/fft.h + # third_party/{ooura,cmsis-dsp}). Now consumed from the tap/dsptap submodule as # tap::dsp, which carries the same numeric contract and selects the per-target -# backend itself (CMSIS on the bare-metal M55, vDSP on Apple, Ooura elsewhere). -# Override with -DTAP_DSP_FFT_CMSIS=OFF / -DTAP_DSP_FFT_ACCELERATE=OFF. +# backend itself (CMSIS on the bare-metal M55, Ooura elsewhere). +# Override with -DTAP_DSP_FFT_CMSIS=OFF / -DTAP_DSP_FFT_ACCELERATE=ON. +# +# APPLE float32 USES OOURA, NOT vDSP — DspTap defaults vDSP ON for Apple and +# MuTap deliberately turns it back off. Two measured reasons (tap/MuTap#31): +# +# 1. Accuracy. On spectra with exactly-empty bins — an on-bin tone, which is +# what the G.168 tone rows drive the chain with — the vDSP kernel selected +# by 64-byte buffer alignment is far less accurate than Ooura. Measured +# against a double-precision reference on Apple M1, MEDIAN per-bin relative +# error at N=2048: vDSP 0.65, Ooura 1.1e-07. Broadband and off-bin material +# show no such gap, which is why the peak-normalized parity gate never saw +# it. The compliance tone rows fail on that kernel and pass on Ooura. +# +# 2. Contract. Apple's own vDSP.h states the routines are "free to rearrange +# calculations for better performance", that "rounding errors will often be +# different when operations are rearranged", and that they are "not expected +# to conform to IEEE 754". Which kernel runs has been observed to depend on +# buffer alignment, and nothing documented prevents that from changing +# again. A certified compliance claim cannot rest on it. +# +# This is a MuTap policy choice, not a DspTap defect: vDSP remains the right +# default for consumers who want the ~3x and can tolerate that latitude. Setting +# it here (not FORCE) so an explicit -DTAP_DSP_FFT_ACCELERATE=ON still wins for +# anyone who wants to measure it. # ------------------------------------------------------------------------------ +set(TAP_DSP_FFT_ACCELERATE OFF CACHE BOOL + "Route the float32 FFT through Apple vDSP (MuTap defaults this OFF; see #31)") + add_subdirectory(submodules/dsptap) # ------------------------------------------------------------------------------ diff --git a/docs/itu-compliance.md b/docs/itu-compliance.md index 119b87e..b58a5d9 100644 --- a/docs/itu-compliance.md +++ b/docs/itu-compliance.md @@ -650,7 +650,72 @@ the four suites above (`test_itu_echo`, `test_itu_doubletalk`, `TYPED_TEST` over `` — float is the `/0` leg, double the `/1` leg — and **the whole battery clears every certified gate at float32 with the same margins as double** (64 rows × both precisions, -all green on the host CI legs). +all green on the host CI legs) — with one backend carved out below. + +### Which FFT backend each float32 number was measured on + +Precision is not the only axis the float32 path varies over: the float32 +real FFT has three backends, and "float32" alone does not name a +configuration. The certified numbers above are measured on **Ooura** +(Linux GCC, Linux Clang, Windows MSVC, **and macOS on both architectures** — +see below) and on **CMSIS Helium** and its **Ooura fallback** on the +Cortex-M55 leg — which together cover every deployment target that ships +today. All are deterministic across processes, which is the property that +makes a single run of a row count as evidence at all. + +**On Apple, MuTap uses Ooura for float32, not vDSP** — DspTap defaults vDSP +ON for Apple and MuTap's root `CMakeLists.txt` turns it back off. The +certified Apple numbers are therefore Ooura numbers, and the macOS CI leg +gates on that same default, so the certified configuration and the built +one cannot drift apart. + +Two measured reasons, both from [tap/MuTap#31](https://github.com/tap/MuTap/issues/31): + +**Accuracy on sparse spectra.** vDSP dispatches on buffer alignment, and +the kernel selected by 64-byte-aligned split-complex buffers is far less +accurate on spectra with exactly-empty bins — which is precisely what the +G.168 §7 tone row drives the chain with. Median per-bin relative error +against a double-precision reference, Apple M1: + +| material | vDSP (64-aligned) | vDSP (not aligned) | Ooura | +|---|---|---|---| +| broadband | 1.6e-07 | 1.2e-07 | 1.2e-07 | +| tone, off-bin | 1.3e-06 | 1.3e-06 | 6.4e-07 | +| **tone, on-bin** | **0.65** | 1.2e-07 | 1.1e-07 | + +The gap appears only on exactly-on-bin excitation; any leakage that lifts +the empty bins above the noise floor hides it, which is why the +peak-normalized `fft_backend_parity` gate never saw it. On that kernel +`g168_adapted.ToneStability` and +`Float32Parity.ToneRowWithNarrowbandGuard` fail; on Ooura they pass. + +**Contract.** Apple's `vDSP.h` states the routines are "free to rearrange +calculations for better performance", that "rounding errors will often be +different when operations are rearranged", and that they are "not expected +to conform to IEEE 754". Which kernel runs was observed to depend on buffer +alignment, and nothing documented prevents that from changing again. A +certified compliance claim cannot rest on it. + +This is a MuTap policy choice rather than a DspTap defect — vDSP stays the +right default for consumers who want the ~3× and can tolerate that +latitude. Anyone wanting to measure it can still build with +`-DTAP_DSP_FFT_ACCELERATE=ON`. + +Two consequences worth stating plainly, because the battery reported this +correctly for two weeks while it was read as CI flake: + +- A row that is run **once** per leg cannot certify a bimodal outcome. While + the alignment bug was live, the float32 rows on vDSP sampled a ~70/30 + draw, so a green there was a sample rather than a gate — which is how a + genuine compliance failure read as CI flake for two weeks. Repetition + (`--repeat until-fail:N`) is what makes a float32 result evidence, and it + is applied on the macOS leg. +- The argument this section makes for the precision axis — *if "double + passes ⇒ float passes" were sound, the tone row could not exist* — + transposes onto backends unchanged. float32-on-Ooura does not certify + float32-on-vDSP any more than double certifies float. That is why the + backend is named for every number above, and why the macOS gate points at + the backend that ships rather than the one that is convenient. Why both, rather than certify in double and infer float32: the two are not related by a uniform noise floor. The G.168 §7 tone row is the diff --git a/submodules/dsptap b/submodules/dsptap index 9cbfbca..ff6fc6c 160000 --- a/submodules/dsptap +++ b/submodules/dsptap @@ -1 +1 @@ -Subproject commit 9cbfbca6670bdc929bab2804c713b60dad8b1c0a +Subproject commit ff6fc6c5067f675316fd07a05e9c078dbc23b1bd