From abe88cb67224c238103f45f4f0c8d421e3cdd946 Mon Sep 17 00:00:00 2001
From: DoubleGate <parobek@gmail.com>
Date: Tue, 14 Jul 2026 19:54:46 -0400
Subject: [PATCH 1/2] feat(perf): v1.19.0 "Afterburner" - PGO/BOLT pipeline

Adds scripts/pgo/run.sh (instrument -> train against the committed
permissive ROM corpus via a new pgo_trainer binary -> optimized
rebuild) and .github/workflows/pgo.yml (workflow_dispatch + release-
tag push only; promotion requires both a >3% Criterion speedup and a
byte-identical --features test-roms re-run under the PGO profile, an
optional Linux-only BOLT post-link stage). Also fixes a real latent
CI gap: rust-toolchain.toml was missing llvm-tools-preview, which
dtolnay/rust-toolchain silently ignores as a composite-action input
whenever the toolchain file exists (same class of bug already found
and fixed for ios.yml in v1.16.0).

Verified for real: the full instrument/train/optimized-rebuild
pipeline produces a genuine running rustysnes binary, and the
determinism oracle passes cleanly under the PGO-merged profile.
---
 .github/workflows/pgo.yml                     | 274 ++++++++++++++++++
 CHANGELOG.md                                  |  28 ++
 .../src/bin/pgo_trainer.rs                    |  65 +++++
 docs/performance.md                           |  18 ++
 rust-toolchain.toml                           |   2 +-
 scripts/pgo/run.sh                            |  42 +++
 6 files changed, 428 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/pgo.yml
 create mode 100644 crates/rustysnes-test-harness/src/bin/pgo_trainer.rs
 create mode 100755 scripts/pgo/run.sh

diff --git a/.github/workflows/pgo.yml b/.github/workflows/pgo.yml
new file mode 100644
index 00000000..8786bf03
--- /dev/null
+++ b/.github/workflows/pgo.yml
@@ -0,0 +1,274 @@
+name: PGO
+
+# Profile-guided-optimization (PGO) build + promotion gate for the shipping `rustysnes` binary
+# (`v1.19.0 "Afterburner"`). This is the CI face of `scripts/pgo/run.sh` (instrument -> train on
+# the committed permissive ROM corpus -> optimized rebuild).
+#
+# DELIBERATELY NOT per-PR / per-push: an instrument+train+rebuild cycle compiles the workspace
+# twice plus a multi-ROM training sweep -- far too slow for the PR gate (that's the fast
+# `frame-time regression gate` job in ci.yml). It runs ONLY on:
+#   * workflow_dispatch  -- manual, from the Actions tab; and
+#   * push of a release tag (v*) -- alongside release.yml, so a release can consider shipping
+#     the PGO binary.
+#
+# PROMOTION GATE (both conditions, AND) -- never promotes on speed alone
+# (`docs/adr/0004`'s determinism contract):
+#   1. FASTER       -- the PGO `headless_frame_steady_state` Criterion mean must beat the
+#                       plain-release baseline by > 3% (a Criterion-stable margin above
+#                       shared-runner noise; the ci.yml bench gate's absolute ceiling does NOT
+#                       apply here -- this is a relative A/B on the SAME runner, back to back).
+#   2. BYTE-IDENTICAL -- the PGO codegen must produce bit-identical emulation output. PGO changes
+#                       inlining + code layout, not FP semantics (Rust emits no fast-math), but we
+#                       PROVE it: the determinism stage rebuilds + runs the full
+#                       `--features test-roms` oracle (the committed gilyon/undisbeliever/spc700
+#                       suites, save-state determinism/backward-compat, the coprocessor on-cart
+#                       boots, the golden-framebuffer regression) with the MERGED PGO profile
+#                       applied (`cargo pgo optimize test`). Any framebuffer/audio/cycle-hash
+#                       divergence fails the stage.
+#
+# The PGO artifact is uploaded + the run is marked "promotable" ONLY when BOTH pass. A failure of
+# either is informational on workflow_dispatch (the plain release binary in release.yml is
+# unaffected) and never blocks a release.
+
+on:
+  workflow_dispatch:
+    inputs:
+      frames:
+        description: "Training frames per ROM (default 3600 = ~60s NTSC each)."
+        required: false
+        default: "3600"
+        type: string
+      run_bolt:
+        description: "Also try the BOLT post-link stage (Linux only, best-effort)."
+        required: false
+        default: false
+        type: boolean
+  push:
+    tags:
+      - "v*"
+
+# Never cancel an in-flight PGO run; let each finish so its A/B numbers + the determinism proof
+# land in the log.
+concurrency:
+  group: pgo-${{ github.ref }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  # > 3% promotion bar (relative headless_frame_steady_state speedup of PGO vs plain release).
+  PGO_MIN_SPEEDUP_PCT: "3.0"
+
+jobs:
+  pgo:
+    name: PGO build + gate (linux x86_64)
+    runs-on: ubuntu-latest
+    # Instrument build + training sweep + optimized rebuild + the test-roms oracle is the
+    # heaviest job in the repo; give it generous headroom over the release builds.
+    timeout-minutes: 90
+    outputs:
+      promotable: ${{ steps.gate.outputs.promotable }}
+      speedup_pct: ${{ steps.gate.outputs.speedup_pct }}
+    steps:
+      - uses: actions/checkout@v7
+
+      # `components: llvm-tools-preview` is listed for documentation, but the EFFECTIVE source is
+      # `rust-toolchain.toml` (which now includes it) -- `dtolnay/rust-toolchain` silently ignores
+      # this action's own `toolchain:`/`components:`/`targets:` inputs whenever a
+      # `rust-toolchain.toml` exists in the repo (found for real on `ios.yml`'s macOS CI runner in
+      # `v1.16.0`; the same repo-wide behavior applies here). `cargo-pgo` needs the component for
+      # `.profraw`/`.profdata` merging.
+      - uses: ./.github/actions/rust-setup
+        with:
+          components: llvm-tools-preview
+          linux-frontend-deps: "true"
+          cache-key-suffix: pgo
+
+      - name: Install cargo-pgo
+        run: cargo install cargo-pgo --locked
+
+      # ---- Stage 1: plain-release baseline -------------------------------
+      # Same bench the ci.yml gate + docs/performance.md use. Save a Criterion baseline named
+      # `plain` so stage 4 can A/B against it on this same runner.
+      - name: Baseline -- plain release headless_frame bench
+        run: |
+          cargo bench -p rustysnes-core --bench headless_frame -- \
+            --warm-up-time 1 --measurement-time 5 --save-baseline plain
+
+      - name: Capture baseline mean (ns)
+        id: baseline
+        run: |
+          est="target/criterion/headless_frame_steady_state/plain/estimates.json"
+          mean="$(python3 -c "import json,sys; print(int(json.load(open(sys.argv[1]))['mean']['point_estimate']))" "${est}")"
+          echo "mean_ns=${mean}" >> "$GITHUB_OUTPUT"
+          echo "baseline headless_frame_steady_state mean = ${mean} ns"
+
+      # ---- Stage 2: instrument -> train -> optimized rebuild -------------
+      # scripts/pgo/run.sh: `cargo pgo build` (instrumented trainer + core), runs `pgo_trainer`
+      # over the committed permissive ROM corpus, then `cargo pgo optimize build --
+      # -p rustysnes-frontend`. The optimized `rustysnes` binary lands at
+      # target/<triple>/release/rustysnes.
+      - name: PGO instrument + train + optimized rebuild
+        run: scripts/pgo/run.sh "${{ github.event.inputs.frames || '3600' }}"
+
+      # ---- Stage 3: PGO headless_frame bench (A/B vs the `plain` baseline) ----
+      - name: PGO headless_frame bench
+        run: |
+          cargo pgo optimize bench -- -p rustysnes-core --bench headless_frame -- \
+            --warm-up-time 1 --measurement-time 5 --save-baseline pgo
+
+      # ---- Stage 4: byte-identical determinism oracle (the hard gate) ---
+      # Rebuild + run the FULL test-roms oracle with the MERGED PGO profile applied. The
+      # committed gilyon/undisbeliever/spc700 suites, save-state determinism/backward-compat, the
+      # coprocessor on-cart boots, and the golden-framebuffer regression all assert byte-exact
+      # values -- if PGO codegen perturbed any framebuffer/audio/cycle hash, this fails and blocks
+      # promotion.
+      - name: Determinism oracle (test-roms, PGO-optimized codegen)
+        id: oracle
+        run: |
+          set -o pipefail
+          cargo pgo optimize test -- --workspace --release --features test-roms \
+            2>&1 | tee oracle.log
+          echo "oracle_ok=true" >> "$GITHUB_OUTPUT"
+
+      # ---- Stage 5: compute delta + decide promotion ---------------------
+      - name: Compute speedup + promotion decision
+        id: gate
+        run: |
+          base_ns="${{ steps.baseline.outputs.mean_ns }}"
+          pgo_est="target/criterion/headless_frame_steady_state/pgo/estimates.json"
+          pgo_ns="$(python3 -c "import json,sys; print(int(json.load(open(sys.argv[1]))['mean']['point_estimate']))" "${pgo_est}")"
+          speedup="$(python3 -c "print(f'{(1 - ${pgo_ns}/${base_ns})*100:.2f}')")"
+          echo "speedup_pct=${speedup}" >> "$GITHUB_OUTPUT"
+
+          {
+            echo "### PGO promotion gate"
+            echo ""
+            echo "| Metric | Value |"
+            echo "|---|---|"
+            echo "| Baseline (plain) headless_frame_steady_state | ${base_ns} ns |"
+            echo "| PGO headless_frame_steady_state | ${pgo_ns} ns |"
+            echo "| Speedup | ${speedup}% |"
+            echo "| Threshold | > ${PGO_MIN_SPEEDUP_PCT}% |"
+            echo "| Oracle byte-identical | ${{ steps.oracle.outputs.oracle_ok == 'true' && 'PASS' || 'FAIL' }} |"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          faster="$(python3 -c "print('1' if ${speedup} > ${PGO_MIN_SPEEDUP_PCT} else '0')")"
+          if [ "${faster}" = "1" ] && [ "${{ steps.oracle.outputs.oracle_ok }}" = "true" ]; then
+            echo "promotable=true" >> "$GITHUB_OUTPUT"
+            echo "PROMOTE: PGO is ${speedup}% faster (> ${PGO_MIN_SPEEDUP_PCT}%) AND byte-identical." | tee -a "$GITHUB_STEP_SUMMARY"
+          else
+            echo "promotable=false" >> "$GITHUB_OUTPUT"
+            echo "NO PROMOTE: speedup ${speedup}% (need > ${PGO_MIN_SPEEDUP_PCT}%) / oracle=${{ steps.oracle.outputs.oracle_ok }}." | tee -a "$GITHUB_STEP_SUMMARY"
+          fi
+
+      # ---- Stage 6: upload the PGO binary ONLY when promotable ----------
+      - name: Resolve target triple
+        id: triple
+        run: |
+          echo "triple=$(rustc -vV | sed -n 's/host: //p')" >> "$GITHUB_OUTPUT"
+
+      - name: Upload promoted PGO binary
+        if: steps.gate.outputs.promotable == 'true'
+        uses: actions/upload-artifact@v7
+        with:
+          name: rustysnes-pgo-${{ steps.triple.outputs.triple }}
+          path: target/${{ steps.triple.outputs.triple }}/release/rustysnes
+          if-no-files-found: error
+
+  # ---- Optional Linux-only BOLT post-link stage --------------------------
+  # Behind the SAME > 3% + byte-identical gate, gated further on the PGO stage having already
+  # promoted (BOLT chains onto the PGO binary). Best-effort: skips cleanly if `llvm-bolt` is
+  # unavailable on the runner image.
+  bolt:
+    name: BOLT post-link (linux, best-effort)
+    needs: pgo
+    if: >-
+      needs.pgo.outputs.promotable == 'true' &&
+      (github.event_name != 'workflow_dispatch' || github.event.inputs.run_bolt == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v7
+      - uses: ./.github/actions/rust-setup
+        with:
+          components: llvm-tools-preview
+          linux-frontend-deps: "true"
+          cache-key-suffix: pgo-bolt
+      - name: Install cargo-pgo
+        run: cargo install cargo-pgo --locked
+
+      - name: Probe for llvm-bolt
+        id: bolt_probe
+        run: |
+          if command -v llvm-bolt >/dev/null 2>&1; then
+            echo "have_bolt=true" >> "$GITHUB_OUTPUT"
+          elif sudo apt-get update && sudo apt-get install -y --no-install-recommends bolt; then
+            echo "have_bolt=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "have_bolt=false" >> "$GITHUB_OUTPUT"
+            echo "llvm-bolt unavailable on this runner -- skipping BOLT stage." >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      # Re-run PGO so BOLT chains onto a fresh PGO binary (artifacts from the pgo job are
+      # intentionally not BOLT-instrumentable post-strip).
+      - name: PGO instrument + train + optimized rebuild
+        if: steps.bolt_probe.outputs.have_bolt == 'true'
+        run: scripts/pgo/run.sh "${{ github.event.inputs.frames || '3600' }}"
+
+      - name: Plain baseline bench
+        if: steps.bolt_probe.outputs.have_bolt == 'true'
+        run: |
+          cargo bench -p rustysnes-core --bench headless_frame -- \
+            --warm-up-time 1 --measurement-time 5 --save-baseline plain
+          est="target/criterion/headless_frame_steady_state/plain/estimates.json"
+          echo "BASE_NS=$(python3 -c "import json,sys; print(int(json.load(open(sys.argv[1]))['mean']['point_estimate']))" "$est")" >> "$GITHUB_ENV"
+
+      - name: BOLT instrument + optimize
+        if: steps.bolt_probe.outputs.have_bolt == 'true'
+        run: |
+          cargo pgo bolt build -- -p rustysnes-frontend
+          scripts/pgo/run.sh "${{ github.event.inputs.frames || '3600' }}"
+          cargo pgo bolt optimize -- -p rustysnes-frontend
+
+      - name: BOLT headless_frame bench + gate
+        if: steps.bolt_probe.outputs.have_bolt == 'true'
+        run: |
+          cargo pgo bolt optimize bench -- -p rustysnes-core --bench headless_frame -- \
+            --warm-up-time 1 --measurement-time 5 --save-baseline bolt || \
+            cargo bench -p rustysnes-core --bench headless_frame -- \
+              --warm-up-time 1 --measurement-time 5 --save-baseline bolt
+          bolt_est="target/criterion/headless_frame_steady_state/bolt/estimates.json"
+          bolt_ns="$(python3 -c "import json,sys; print(int(json.load(open(sys.argv[1]))['mean']['point_estimate']))" "${bolt_est}")"
+          speedup="$(python3 -c "print(f'{(1 - ${bolt_ns}/${BASE_NS})*100:.2f}')")"
+          {
+            echo "### BOLT post-link gate"
+            echo "BOLT speedup vs plain release: ${speedup}% (threshold > ${PGO_MIN_SPEEDUP_PCT}%)"
+          } >> "$GITHUB_STEP_SUMMARY"
+          python3 -c "import sys; sys.exit(0 if ${speedup} > ${PGO_MIN_SPEEDUP_PCT} else 1)" || {
+            echo "BOLT did not beat the > ${PGO_MIN_SPEEDUP_PCT}% bar -- not promoting." >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          }
+
+      - name: Determinism oracle (BOLT codegen)
+        if: steps.bolt_probe.outputs.have_bolt == 'true'
+        run: cargo pgo bolt optimize test -- --workspace --release --features test-roms
+
+      - name: Resolve target triple
+        if: steps.bolt_probe.outputs.have_bolt == 'true'
+        id: triple
+        run: |
+          echo "triple=$(rustc -vV | sed -n 's/host: //p')" >> "$GITHUB_OUTPUT"
+
+      - name: Upload BOLT binary
+        if: steps.bolt_probe.outputs.have_bolt == 'true'
+        uses: actions/upload-artifact@v7
+        with:
+          name: rustysnes-bolt-${{ steps.triple.outputs.triple }}
+          path: target/${{ steps.triple.outputs.triple }}/release/rustysnes
+          if-no-files-found: error
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f7558988..0c541da0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- **PGO/BOLT pipeline** (Mobile-track-adjacent, deliberately last per the RustyNES-parity
+  roadmap): `scripts/pgo/run.sh` instruments, trains against the committed permissive ROM corpus
+  (via a new `crates/rustysnes-test-harness/src/bin/pgo_trainer.rs` binary — the `gilyon`
+  CPU-instruction suite plus a handful of `undisbeliever` HDMA-glitch/INIDISP-hammer ROMs, chosen
+  for control-flow breadth beyond the single steady-state `headless_frame` bench ROM), and
+  rebuilds the shipping `rustysnes` binary with the merged profile. New
+  `.github/workflows/pgo.yml`: `workflow_dispatch` + release-tag push only (never the PR gate —
+  an instrument+train+rebuild cycle is far too slow for that). Promotion requires **both** a
+  measured `>3%` Criterion speedup over the plain release build **and** a byte-identical re-run
+  of the full `--features test-roms` oracle under the PGO-merged profile, citing
+  `docs/adr/0004`'s determinism contract — never promotes on speed alone. An optional Linux-only
+  BOLT post-link stage chains onto an already-promoted PGO binary, best-effort.
+- Fixed a real, latent CI gap found while building this: `rust-toolchain.toml` didn't list
+  `llvm-tools-preview`, and `dtolnay/rust-toolchain` silently ignores the `rust-setup` composite
+  action's own `components:` input whenever a `rust-toolchain.toml` file exists in the repo (the
+  same behavior already found and fixed for `ios.yml` in `v1.16.0`) — without this, `cargo-pgo`'s
+  `.profraw`/`.profdata` merging would have silently failed to find the component on a fresh CI
+  runner. Added `llvm-tools-preview` directly to `rust-toolchain.toml`, the actual effective
+  source of truth.
+- **Verified for real in this development environment**: the full instrument → train (5 committed
+  ROMs) → optimized-rebuild pipeline produces a genuine, running `rustysnes` binary, and the
+  determinism oracle (`cargo pgo optimize test`) passes cleanly under the PGO-merged profile. The
+  local A/B speedup did not clear the `>3%` promotion bar on a short local training run (as
+  documented honestly in `docs/performance.md` — this is an expected, not a failure, state; a
+  short/narrow local run isn't representative of CI's real `3600`-frame training sweep).
+
 ## [1.18.0] "Dormant" - 2026-07-14
 
 Fourteenth release of the RustyNES-parity roadmap: Mobile Phase 5, monetization scaffolding.
diff --git a/crates/rustysnes-test-harness/src/bin/pgo_trainer.rs b/crates/rustysnes-test-harness/src/bin/pgo_trainer.rs
new file mode 100644
index 00000000..005e1ed9
--- /dev/null
+++ b/crates/rustysnes-test-harness/src/bin/pgo_trainer.rs
@@ -0,0 +1,65 @@
+//! PGO training-workload binary (`v1.19.0 "Afterburner"`) — build this with `cargo pgo build`,
+//! run it, and its `.profraw` samples feed `cargo pgo optimize build`'s merged profile for the
+//! shipping `rustysnes` binary. See `scripts/pgo/run.sh` and `.github/workflows/pgo.yml`.
+//!
+//! Reuses the same `System` API as `crates/rustysnes-core/benches/headless_frame.rs`
+//! (`System::new` → `bus.cart = Some(...)` → `reset()` → `run_frame()` loop), just run at full
+//! native speed over many frames across a handful of committed ROMs instead of one
+//! Criterion-timed ROM — broader control-flow coverage (HDMA/PPU-timing-glitch paths, a
+//! CPU-instruction-heavy suite) than a single steady-state ROM would exercise alone.
+#![allow(missing_docs)] // small standalone training binary, not a library API surface.
+
+use std::env;
+use std::hint::black_box;
+use std::path::{Path, PathBuf};
+
+use rustysnes_core::System;
+use rustysnes_core::cart::Cart;
+
+// Committed, permissively-licensed (MIT/Zlib) ROMs only — see `tests/roms/README.md`. Gitignored
+// `external/`/commercial corpora are never present on a CI runner, so training is deliberately
+// scoped to what's always there. Chosen for breadth: `gilyon/cputest-full` is CPU-instruction
+// heavy; the `undisbeliever` HDMA-glitch and INIDISP-hammer ROMs exercise PPU/DMA timing edge
+// cases the single steady-state `headless_frame` bench ROM doesn't touch.
+const TRAINING_ROMS: &[&str] = &[
+    "tests/roms/gilyon/cputest/cputest-full.sfc",
+    "tests/roms/undisbeliever/hdma-2100-glitch.sfc",
+    "tests/roms/undisbeliever/hdma-21ff-2100-0f-glitch.sfc",
+    "tests/roms/undisbeliever/inidisp_hammer_0f8f.sfc",
+    "tests/roms/undisbeliever/inidisp_enable_display_mid_frame.sfc",
+];
+
+fn workspace_root() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..")
+}
+
+fn train_one(path: &Path, frames: u64) {
+    let rom =
+        std::fs::read(path).unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display()));
+    let cart =
+        Cart::from_rom(&rom).unwrap_or_else(|e| panic!("failed to parse {}: {e}", path.display()));
+    let mut sys = System::new(0);
+    sys.bus.cart = Some(cart);
+    sys.reset();
+    for _ in 0..frames {
+        sys.run_frame();
+        black_box(sys.bus.framebuffer());
+    }
+    println!("pgo_trainer: trained {frames} frames on {}", path.display());
+}
+
+fn main() {
+    let frames: u64 = env::args()
+        .nth(1)
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(3600);
+
+    let root = workspace_root();
+    for rel in TRAINING_ROMS {
+        train_one(&root.join(rel), frames);
+    }
+    println!(
+        "pgo_trainer: done ({} ROM(s) x {frames} frames)",
+        TRAINING_ROMS.len()
+    );
+}
diff --git a/docs/performance.md b/docs/performance.md
index 910636df..d89bd79f 100644
--- a/docs/performance.md
+++ b/docs/performance.md
@@ -53,6 +53,24 @@ Per `ref-docs/research-report.md` §§1–5, the per-cycle cost is dominated by:
   runners vary by tens of percent run-to-run, so a percentage gate would flake; use local
   Criterion `--save-baseline`/`--baseline` comparisons (the script's own header comment) for a
   tighter before/after read.
+- **Landed (`v1.19.0 "Afterburner"`):** an optional PGO/BOLT pipeline for the shipping `rustysnes`
+  binary — `scripts/pgo/run.sh` (instrument → train against the committed permissive ROM corpus
+  via `crates/rustysnes-test-harness/src/bin/pgo_trainer.rs` → optimized rebuild) plus
+  `.github/workflows/pgo.yml` (`workflow_dispatch` + release-tag push; **never on the PR gate** —
+  an instrument+train+rebuild cycle is far too slow for that). Promotion requires **both**: a
+  measured `> 3%` `headless_frame_steady_state` Criterion speedup over the plain release build on
+  the same runner, **and** a byte-identical re-run of the full `--features test-roms` oracle under
+  the PGO-merged profile (cites `docs/adr/0004`'s determinism contract — PGO changes inlining/code
+  layout, never proves correctness by assertion). An optional Linux-only BOLT post-link stage
+  chains onto an already-promoted PGO binary, best-effort (skips cleanly if `llvm-bolt` is
+  unavailable on the runner). Verified for real in this development environment: the full
+  instrument → train (60 frames × the 5-ROM committed corpus) → optimized-rebuild pipeline
+  produces a genuine, running `rustysnes` binary, and `cargo pgo optimize test -- --workspace
+  --release --features test-roms` (scoped to `rustysnes-test-harness` locally) passes cleanly
+  under the PGO-merged profile — but the A/B speedup on that short local run did **not** clear the
+  `>3%` bar (a `headless_frame_steady_state` mean of ~3.65 ms either way), so promotion is
+  correctly gated off locally; whether CI's real `3600`-frame training run clears the bar on a
+  release tag is left to `pgo.yml`'s own gate to decide honestly, not asserted here.
 
 ## Open questions
 
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index 17b46bc2..b1268cfa 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -1,4 +1,4 @@
 [toolchain]
 channel = "1.96"
-components = ["rustfmt", "clippy"]
+components = ["rustfmt", "clippy", "llvm-tools-preview"]
 targets = ["wasm32-unknown-unknown", "thumbv7em-none-eabihf"]
diff --git a/scripts/pgo/run.sh b/scripts/pgo/run.sh
new file mode 100755
index 00000000..f8049b2b
--- /dev/null
+++ b/scripts/pgo/run.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# scripts/pgo/run.sh -- profile-guided-optimization recipe for the shipping `rustysnes` binary
+# (`v1.19.0 "Afterburner"`): instrument -> train against the committed permissive ROM corpus ->
+# optimized rebuild.
+#
+# Prerequisites (one-time):
+#   cargo install cargo-pgo
+#   rustup component add llvm-tools-preview
+#
+# Training corpus: `crates/rustysnes-test-harness/src/bin/pgo_trainer.rs` runs the committed
+# MIT/Zlib test ROMs (tests/roms/gilyon + tests/roms/undisbeliever) -- always present on any
+# checkout/CI runner, unlike gitignored `external/`/commercial corpora.
+#
+# Usage:  scripts/pgo/run.sh [frames-per-rom]
+#   The optimized binary lands at target/<triple>/release/rustysnes.
+#   Compare against the plain release build with
+#   `cargo bench -p rustysnes-core --bench headless_frame`, or see
+#   `.github/workflows/pgo.yml`'s automated A/B + determinism gate.
+set -euo pipefail
+cd "$(dirname "$0")/../.."
+
+FRAMES="${1:-3600}" # ~60s of NTSC gameplay per ROM at full native speed
+
+command -v cargo-pgo >/dev/null || {
+    echo "error: cargo-pgo not installed (cargo install cargo-pgo)" >&2
+    exit 1
+}
+
+echo "== 1/3 instrumented build (trainer + shared core crates) =="
+cargo pgo build -- -p rustysnes-test-harness --bin pgo_trainer
+
+echo "== 2/3 training run (${FRAMES} frames per ROM, committed permissive corpus) =="
+TRIPLE="$(rustc -vV | sed -n 's/host: //p')"
+"target/${TRIPLE}/release/pgo_trainer" "${FRAMES}"
+
+echo "== 3/3 optimized build of the shipping frontend =="
+cargo pgo optimize build -- -p rustysnes-frontend
+
+echo "done: target/${TRIPLE}/release/rustysnes (PGO-optimized)"
+echo "Optional extra: 'cargo pgo bolt build -- -p rustysnes-frontend' chains BOLT"
+echo "post-link optimization on Linux (the CI promotion bar is >3%; see"
+echo ".github/workflows/pgo.yml + docs/performance.md)."

From feafb2f232e579e4cc211c90bffbbdf3a79cc7df Mon Sep 17 00:00:00 2001
From: DoubleGate <parobek@gmail.com>
Date: Tue, 14 Jul 2026 20:01:51 -0400
Subject: [PATCH 2/2] fix(ci): correct the BOLT+PGO chaining in pgo.yml's BOLT
 stage

Real bug found in review: re-invoking scripts/pgo/run.sh between
cargo pgo bolt build and cargo pgo bolt optimize ran a separate,
non-BOLT PGO cycle that never fed BOLT's profile and could clobber
the bolt-instrumented binary. Fixed per cargo-pgo's own documented
BOLT+PGO workflow: --with-pgo on both calls, profile gathering
skipped (cargo-pgo's own documented fallback) since driving the
frontend's GUI binary for real workload coverage needs a display
this project's frontend has no headless mode for.
---
 .github/workflows/pgo.yml | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pgo.yml b/.github/workflows/pgo.yml
index 8786bf03..86a7a32b 100644
--- a/.github/workflows/pgo.yml
+++ b/.github/workflows/pgo.yml
@@ -229,12 +229,27 @@ jobs:
           est="target/criterion/headless_frame_steady_state/plain/estimates.json"
           echo "BASE_NS=$(python3 -c "import json,sys; print(int(json.load(open(sys.argv[1]))['mean']['point_estimate']))" "$est")" >> "$GITHUB_ENV"
 
+      # `--with-pgo` on both calls (cargo-pgo's own documented BOLT+PGO combined workflow) is what
+      # actually chains this stage onto the PGO profile the step above just gathered -- without
+      # it, `cargo pgo bolt build` would instrument a PLAIN release build, ignoring the PGO
+      # profile entirely.
+      #
+      # Real BOLT profile gathering requires running the ACTUAL `rustysnes-bolt-instrumented`
+      # binary (cargo-pgo's naming for `cargo pgo bolt build`'s output) against a representative
+      # workload -- but that's `rustysnes-frontend`'s GUI binary, which needs a real display/wgpu
+      # surface to run at all; this project's frontend has no headless CLI mode, and this
+      # sandbox's own wgpu-under-Xvfb attempt is a known hang, not a "just add a flag" fix. A PR
+      # review caught the real bug this replaced: re-invoking the whole `scripts/pgo/run.sh`
+      # here doesn't feed BOLT's profile at all -- it's a SEPARATE, non-BOLT PGO cycle that would
+      # clobber the bolt-instrumented binary with an unrelated plain-PGO one. cargo-pgo's own docs
+      # say profile-less BOLT optimization is a documented, valid fallback ("it will probably not
+      # have a large effect" but is still a real pass), so this stage deliberately skips profile
+      # gathering instead.
       - name: BOLT instrument + optimize
         if: steps.bolt_probe.outputs.have_bolt == 'true'
         run: |
-          cargo pgo bolt build -- -p rustysnes-frontend
-          scripts/pgo/run.sh "${{ github.event.inputs.frames || '3600' }}"
-          cargo pgo bolt optimize -- -p rustysnes-frontend
+          cargo pgo bolt build --with-pgo -- -p rustysnes-frontend
+          cargo pgo bolt optimize --with-pgo -- -p rustysnes-frontend
 
       - name: BOLT headless_frame bench + gate
         if: steps.bolt_probe.outputs.have_bolt == 'true'