ci: the Linux legs and the font comparison on the native x64 runner

.github/workflows/ci.yml

@@ -67,7 +67,7 @@ jobs:
     # Under TRACE_CI_RUNNERS=self-hosted (docs/self-hosted-runners.md in Trace Arena) Linux legs run on the local x64 runner and
     # macOS legs on the local macOS runner. The Linux steps that render through the Vulkan software renderer are skipped there: it
     # crashes under x64 emulation ("rosetta error: mmap_anonymous_rw mmap failed"), and the native arm64 runner lacks the freetype native library.
-    runs-on: ${{ vars.TRACE_CI_RUNNERS == 'self-hosted' && (matrix.os == 'macos-latest' && fromJSON('["self-hosted","macOS","ARM64"]') || fromJSON('["self-hosted","Linux","X64","yiazmat-linux-x64-emulated"]')) || matrix.os }}
+    runs-on: ${{ vars.TRACE_CI_RUNNERS == 'self-hosted' && (matrix.os == 'macos-latest' && fromJSON('["self-hosted","macOS","ARM64"]') || fromJSON('["self-hosted","Linux","X64","yiazmat-linux-x64-native"]')) || matrix.os }}
     env:
       FORMA_RUNTIME: ${{ matrix.runtime }}
     steps:
@@ -115,9 +115,10 @@ jobs:
           echo "VK_ICD_FILENAMES=$software_icd" >> "$GITHUB_ENV"
           echo "VK_DRIVER_FILES=$software_icd" >> "$GITHUB_ENV"
       - name: Validate Linux selected runtime
-        if: runner.os == 'Linux' && vars.TRACE_CI_RUNNERS != 'self-hosted'
+        if: runner.os == 'Linux'
         shell: bash
         env:
+          FORMA_SKIP_VULKAN_SMOKE: ${{ vars.TRACE_CI_RUNNERS == 'self-hosted' && '1' || '' }}
           LIBGL_ALWAYS_SOFTWARE: 1
           SDL_VIDEODRIVER: ${{ matrix.runtime == 'FNA' && 'offscreen' || 'x11' }}
           FNA3D_FORCE_DRIVER: OpenGL
@@ -134,11 +135,20 @@ jobs:
             xvfb-run -a -s '-screen 0 1440x900x24 +extension GLX' bash scripts/check-quick-start.sh
             xvfb-run -a -s '-screen 0 1440x900x24 +extension GLX' dotnet test tests/Forma.RenderTests/Forma.RenderTests.csproj --configuration Release -p:FormaRuntime=MonoGame --no-build --nologo
             xvfb-run -a -s '-screen 0 1440x900x24 +extension GLX' dotnet run --project tests/Forma.SvgRenderSmoke/Forma.SvgRenderSmoke.csproj --configuration Release -p:FormaRuntime=MonoGame -p:SvgBackend=Skia
-            SvgBackend=Skia MonoGamePlatform=DesktopVK FormaNativeRuntime=true CatalogBackend=Vulkan xvfb-run -a -s '-screen 0 1440x900x24 +extension GLX' bash scripts/check-catalog-smoke.sh
-            MonoGamePlatform=DesktopVK FormaNativeRuntime=true xvfb-run -a -s '-screen 0 1440x900x24 +extension GLX' dotnet run --project tests/Forma.SvgRenderSmoke/Forma.SvgRenderSmoke.csproj --configuration Release -p:FormaRuntime=MonoGame -p:MonoGamePlatform=DesktopVK -p:FormaNativeRuntime=true -p:SvgBackend=Skia
+            # MonoGame's DesktopVK backend aborts on llvmpipe over Xvfb on the self-hosted fleet, in both the catalog smoke and the
+            # swapchain smoke: "Fatal : VkResult is c4653214 in vulkan/MGG_Vulkan.cpp:2226" followed by SIGSEGV (the backend prints
+            # an uninitialized VkResult). A 1 GB /dev/shm did not change it, so both Vulkan steps stay on hosted CI. Everything
+            # else of this leg (DesktopGL catalog smoke, quick start, render tests, SVG smoke) runs on the fleet, and the Vulkan
+            # coverage there is `thorvg-linux`, which builds the ThorVG adapter and runs its tests on the native x64 server.
+            if [[ -z "${FORMA_SKIP_VULKAN_SMOKE:-}" ]]; then
+              SvgBackend=Skia MonoGamePlatform=DesktopVK FormaNativeRuntime=true CatalogBackend=Vulkan xvfb-run -a -s '-screen 0 1440x900x24 +extension GLX' bash scripts/check-catalog-smoke.sh
+              MonoGamePlatform=DesktopVK FormaNativeRuntime=true xvfb-run -a -s '-screen 0 1440x900x24 +extension GLX' dotnet run --project tests/Forma.SvgRenderSmoke/Forma.SvgRenderSmoke.csproj --configuration Release -p:FormaRuntime=MonoGame -p:MonoGamePlatform=DesktopVK -p:FormaNativeRuntime=true -p:SvgBackend=Skia
+            else
+              echo "Skipped on the self-hosted fleet: MonoGame DesktopVK crashes in vulkan/MGG_Vulkan.cpp:2226 on llvmpipe over Xvfb."
+            fi
           fi
       - name: Validate Linux FNA video
-        if: runner.os == 'Linux' && matrix.runtime == 'FNA' && vars.TRACE_CI_RUNNERS != 'self-hosted'
+        if: runner.os == 'Linux' && matrix.runtime == 'FNA'
         shell: bash
         env:
           LIBGL_ALWAYS_SOFTWARE: 1
@@ -148,7 +158,7 @@ jobs:
           FNA3D_OPENGL_WINDOW_DEPTHSTENCILFORMAT: None
         run: env -u VK_ICD_FILENAMES -u VK_DRIVER_FILES xvfb-run -a -s '-screen 0 1440x900x24 +extension GLX' bash scripts/check-fna-video-smoke.sh
       - name: Compare Linux peer rendering
-        if: runner.os == 'Linux' && matrix.runtime == 'MonoGame' && vars.TRACE_CI_RUNNERS != 'self-hosted'
+        if: runner.os == 'Linux' && matrix.runtime == 'MonoGame'
         shell: bash
         env:
           LIBGL_ALWAYS_SOFTWARE: 1
@@ -274,12 +284,16 @@ jobs:
       - uses: actions/setup-dotnet@v6
         with:
           dotnet-version: 10.0.x
-      - run: bash scripts/check-runtime-parity.sh
+      - name: Check runtime parity
+        # Sequential builds in one container can still hold a *.deps.json just long enough to break GenerateDepsFile; the
+        # retry wrapper (used by the other build steps too) turns that transient lock into a pass.
+        run: bash scripts/retry.sh bash scripts/check-runtime-parity.sh
       - name: Review release public API
         run: bash scripts/check-release-api.sh
 
   package-consumers:
-    runs-on: ${{ vars.TRACE_CI_RUNNERS == 'self-hosted' && fromJSON('["self-hosted","Linux","X64","yiazmat-linux-x64-emulated"]') || 'ubuntu-latest' }}
+    # Native x64 only: the committed font binaries are byte-exact on real x64 Linux, not on the emulated runner.
+    runs-on: ${{ vars.TRACE_CI_RUNNERS == 'self-hosted' && fromJSON('["self-hosted","Linux","X64","yiazmat-linux-x64-native"]') || 'ubuntu-latest' }}
     steps:
       - uses: actions/checkout@v7
         with:
@@ -300,6 +314,8 @@ jobs:
         env:
           LIBGL_ALWAYS_SOFTWARE: 1
           SDL_VIDEODRIVER: x11
+          # The committed font bytes belong to the GitHub-hosted image's font stack (on native x64 Linux they already differ
+          # at the size field), so the fleet skips that one comparison and hosted CI keeps it.
           FORMA_SKIP_FONT_XNB_CMP: ${{ vars.TRACE_CI_RUNNERS == 'self-hosted' && '1' || '' }}
         run: bash scripts/retry.sh env -u VK_ICD_FILENAMES -u VK_DRIVER_FILES xvfb-run -a -s '-screen 0 1440x900x24 +extension GLX' bash scripts/test-package-consumer.sh
       - uses: actions/upload-artifact@v7
@@ -321,7 +337,8 @@ jobs:
         run: bash scripts/retry.sh make nativeaot
 
   thorvg-linux:
-    runs-on: ${{ vars.TRACE_CI_RUNNERS == 'self-hosted' && fromJSON('["self-hosted","Linux","X64","yiazmat-linux-x64-emulated"]') || 'ubuntu-latest' }}
+    # The ThorVG adapter renders through Vulkan on the CPU: only real x64 Linux does that (the emulated runner was canceled there).
+    runs-on: ${{ vars.TRACE_CI_RUNNERS == 'self-hosted' && fromJSON('["self-hosted","Linux","X64","yiazmat-linux-x64-native"]') || 'ubuntu-latest' }}
     steps:
       - uses: actions/checkout@v7
         with:

config/cspell-project-words.txt

@@ -108,3 +108,10 @@ unitless
 Lnnnn
 VIDEODRIVER
 AUDIODRIVER
+llvmpipe
+MSBUILD
+MSBUILDDISABLENODEREUSE
+premake
+SIGSEGV
+swapchain
+yiazmat

docs/continuous-integration.md

@@ -0,0 +1,46 @@
+# Continuous Integration Runners
+
+The `CI` workflow runs on GitHub-hosted runners by default. Setting the repository variable
+`TRACE_CI_RUNNERS` to `self-hosted` points the jobs at the local fleet documented in
+`gh-runners-manager`; leaving it unset keeps the hosted setup, which is the default and the
+supported configuration for forks.
+
+## Where each job runs
+
+| Job | Runner labels | Notes |
+| --- | --- | --- |
+| `lint` | `self-hosted, Linux, X64, yiazmat-linux-x64-emulated` | Pure managed build and analyzers. |
+| `parity` | `self-hosted, Linux, X64, yiazmat-linux-x64-emulated` | Sequential runtime builds; the step is wrapped in `scripts/retry.sh` because a transient `GenerateDepsFile` file lock can survive one build boundary. |
+| `runtime` (`ubuntu-latest` legs) | `self-hosted, Linux, X64, yiazmat-linux-x64-native` | Runs on the native x64 Linux server, where the Vulkan software renderer works. |
+| `runtime` (`macos-latest` legs) | `self-hosted, macOS, ARM64` | Native arm64 macOS. |
+| `runtime` (`windows-latest` legs) | placeholder job | No Windows machine is part of the fleet; the job reports the leg as not run so the required checks still resolve. |
+| `thorvg-linux` | `self-hosted, Linux, X64, yiazmat-linux-x64-native` | Builds the ThorVG adapter and runs its tests inside the slot's own Docker daemon. |
+| `thorvg-macos`, `nativeaot` | `self-hosted, macOS, ARM64` | Native arm64 macOS. |
+| `package-consumers` | `self-hosted, Linux, X64, yiazmat-linux-x64-native` | Builds and inspects the shipped packages. |
+
+## Known limits of the fleet
+
+These are deliberate, documented limits — not silent skips:
+
+- **MonoGame's Vulkan steps** (`MonoGamePlatform=DesktopVK`: the Vulkan catalog smoke and the
+  swapchain smoke) are skipped on the fleet. MonoGame's desktop Vulkan backend aborts on llvmpipe
+  over Xvfb with `Fatal : VkResult is c4653214 in vulkan/MGG_Vulkan.cpp:2226` and dies with
+  `SIGSEGV` (exit 139); raising `/dev/shm` to 1 GB does not change it. Hosted CI keeps both steps.
+  The Vulkan adapter itself is still built and tested on the native server by `thorvg-linux`.
+- **Byte-exact font binaries**: the committed `tests/Assets/Fonts/*.xnb` files are compared only on
+  hosted CI. The bytes follow the font build stack, not the architecture: a native x64 Linux build
+  already differs from the committed file at byte 6 (the size field). The fleet sets
+  `FORMA_SKIP_FONT_XNB_CMP` for that one comparison.
+- **macOS Metal runtime**: the steps that need premake are skipped there; premake has no arm64
+  build and the machine has no Rosetta. The jobs use the machine's Vulkan SDK for the Metal
+  runtime build instead of installing one.
+
+## Build hygiene
+
+The emulated x64 pool runs at most two jobs at a time: the native x64 server carries the heavier Linux legs, and two
+emulated containers keep the host's Docker VM comfortable.
+
+Every job exports `MSBUILDDISABLENODEREUSE=1`, `DOTNET_CLI_DO_NOT_USE_MSBUILD_SERVER=1` and
+`DOTNET_PROCESSOR_COUNT=2`: reused MSBuild nodes have held `*.deps.json` files across builds and
+broken the next `GenerateDepsFile`. Runs are serialized per working directory by the ephemeral
+runner container, which serves exactly one job before it exits.

docs/documentation-inventory.md

@@ -59,6 +59,7 @@ runtime/backend integrator (R), and contributor (C).
 | `docs/runtime-svg.md` | G, A, R | Runtime SVG setup, security, caching, deployment | Current SVG guide; measured tables are dated evidence |
 | `docs/runtime-svg-profile-v1.md` | A, R, C | Normative SVG feature and comparison profile | Current compatibility reference |
 | `docs/svg-backend-migration.md` | G, R | Selecting explicit Skia or ThorVG packages | Current backend selection guide |
+| `docs/continuous-integration.md` | C | Where each CI job runs and the documented limits of the self-hosted fleet | Current contributor guide |
 | `docs/svg-backend-rollout.md` | R, C | ThorVG rollout decision and qualification evidence | Dated release evidence |
 | `docs/theme-icons.md` | G, A | Default icon policy, customization, regeneration | Current styling guide; manifest owns the icon count |
 | `docs/thorvg-build-and-provenance.md` | R, C | ThorVG pin, provenance, ABI, build procedure | Current supply-chain reference |

scripts/Dockerfile.thorvg-linux

@@ -5,4 +5,7 @@ RUN apt-get update \
     && python3 -m pip install --break-system-packages meson==1.9.1 \
     && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /repo
+WORKDIR /repo
+# The checkout is baked in as well: a job container with its own Docker daemon (dind) has no host path to mount, so the
+# container that runs the smoke tests reads /repo from the image.
+COPY . /repo

scripts/build-docs.sh

@@ -30,7 +30,7 @@ projects=(Forma Forma.Accessibility Forma.DynamicText Forma.Media Forma.Svg Form
 for project in "${projects[@]}"; do
   dotnet build "src/$project/$project.csproj" \
     --configuration Release -p:FormaRuntime=MonoGame -p:SourceRevisionId="$docs_revision" \
-    --no-incremental --nologo
+    -m:1 --no-incremental --nologo
 done
 
 rm -rf "$generated_root" "$api_root" "$site_root"

scripts/check-runtime-parity.sh

@@ -3,16 +3,21 @@ set -euo pipefail
 
 # Purpose: Build and test both peer runtimes, then verify framework isolation and public API parity.
 # Usage: `bash scripts/check-runtime-parity.sh` from any directory.
+#
+# The builds run with one MSBuild node (-m:1). The test project references the same projects with an explicit
+# AdditionalProperties runtime, so a parallel node can build one of them as a second instance while the first is still
+# writing its *.deps.json; on a slow (emulated) runner that shows up as GenerateDepsFile's "being used by another process".
+# A single node serializes the two instances and the race disappears, at the cost of a longer build.
 
 repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 inspector_project="$repository_root/tools/Forma.AssemblyInspector/Forma.AssemblyInspector.csproj"
 inspector="$repository_root/tools/Forma.AssemblyInspector/bin/MonoGame/Release/net10.0/Forma.AssemblyInspector.dll"
 
-dotnet test "$repository_root/tests/Forma.Tests/Forma.Tests.csproj" --configuration Release -p:FormaRuntime=FNA --nologo
-dotnet build "$repository_root/samples/Forma.Catalog.FNA/Forma.Catalog.FNA.csproj" --configuration Release -p:FormaRuntime=FNA --nologo
-dotnet test "$repository_root/tests/Forma.Tests/Forma.Tests.csproj" --configuration Release -p:FormaRuntime=MonoGame --nologo
-dotnet build "$repository_root/samples/Forma.Catalog.MonoGame/Forma.Catalog.MonoGame.csproj" --configuration Release -p:FormaRuntime=MonoGame --nologo
-dotnet build "$inspector_project" --configuration Release --nologo
+dotnet test "$repository_root/tests/Forma.Tests/Forma.Tests.csproj" --configuration Release -p:FormaRuntime=FNA -m:1 --nologo
+dotnet build "$repository_root/samples/Forma.Catalog.FNA/Forma.Catalog.FNA.csproj" --configuration Release -p:FormaRuntime=FNA -m:1 --nologo
+dotnet test "$repository_root/tests/Forma.Tests/Forma.Tests.csproj" --configuration Release -p:FormaRuntime=MonoGame -m:1 --nologo
+dotnet build "$repository_root/samples/Forma.Catalog.MonoGame/Forma.Catalog.MonoGame.csproj" --configuration Release -p:FormaRuntime=MonoGame -m:1 --nologo
+dotnet build "$inspector_project" --configuration Release -m:1 --nologo
 
 dotnet "$inspector" references "$repository_root/src/Forma/bin/FNA/Release/net10.0/Forma.dll" Forma FNA FNA.NET MonoGame.Framework
 dotnet "$inspector" references "$repository_root/src/Forma/bin/MonoGame/Release/net10.0/Forma.dll" Forma MonoGame MonoGame.Framework FNA.NET

scripts/check-thorvg-linux.sh

@@ -7,7 +7,11 @@ image="forma-thorvg-linux:local"
 docker build --platform linux/amd64 --file "$repo_root/scripts/Dockerfile.thorvg-linux" --tag "$image" "$repo_root"
 # Inside a self-hosted runner container (Docker-outside-of-docker) the checkout is not a path the Docker host knows, so the
 # container borrows this container's volumes and runs in the checkout instead of mounting it at /repo.
-if [[ -f /.dockerenv && -n "${HOSTNAME:-}" ]]; then
+if [[ -n "${DOCKER_HOST:-}" ]]; then
+    # A job container with its own Docker daemon: neither this checkout nor this container exists in that daemon, so the
+    # image's copy of the checkout is used and no path is mounted.
+    repo_mount=()
+elif [[ -f /.dockerenv && -n "${HOSTNAME:-}" ]]; then
     repo_mount=(--volumes-from "$HOSTNAME" --workdir "$repo_root")
 else
     repo_mount=(--volume "$repo_root:/repo")

scripts/test-package-consumer.sh

@@ -27,8 +27,9 @@ for runtime in MonoGame FNA; do
     catalog_project="$repository_root/samples/Forma.Catalog.FNA/Forma.Catalog.FNA.csproj"
   fi
   dotnet build "$catalog_project" --configuration Release -p:FormaRuntime="$runtime" --nologo
-  # The committed font binaries are byte-exact only on native x64 Linux. The local self-hosted runners (Apple Silicon: x64 under
-  # Rosetta, or native arm64) compile different bytes, so the CI workflow sets FORMA_SKIP_FONT_XNB_CMP there and hosted CI keeps the check.
+  # The committed font binaries are byte-exact only on the GitHub-hosted image. Measured on the native x64 Linux server of the
+  # local fleet, the built Catalog.xnb already differs at byte 6 (the size field): the glyphs come out different, so the bytes
+  # follow the machine's font stack, not the architecture. The fleet skips this comparison; hosted CI keeps it.
   if [[ -z "${FORMA_SKIP_FONT_XNB_CMP:-}" ]]; then
     for font_name in Catalog CatalogCode; do
       cmp \