From 2caf3769dfeeaddda5a035e99fd2465abc9410f0 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 15:29:24 +0800
Subject: [PATCH 01/34] CollectiveX: make roundtrip mean dispatch->combine in
 every row
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：让 roundtrip 在所有行中统一表示 dispatch->combine

The chained roundtrip excluded staging only for fp8, so `stage` sat inside it for exactly
two configurations — MoRI BF16 scale-up and FlashInfer BF16 — and outside it for the other
800+ rows. The headline therefore compared two different quantities and penalised those two.
Gate the hoist on stage_device_work instead, keeping CX_FP8_CONSUME=dequant as the opt-out.

Also stop two staging paths from moving padding. MoRI cast its whole cap-sized receive
buffer under fp8 rather than the rows dispatch filled, which made that stage flat in T
(61-114us across the decode ladder against 6-30us for BF16). FlashInfer copied its whole
padded workspace plane where the kernel's own staging path skips slots past recv_counters;
measured occupancy is ~0.66 at EP8 and ~0.41 at EP16.
---
 experimental/CollectiveX/bench/ep_backend.py  | 60 ++++++++++--------
 .../CollectiveX/bench/ep_flashinfer.py        | 26 +++++++-
 experimental/CollectiveX/bench/ep_mori.py     | 21 +++++--
 .../tests/test_roundtrip_staging.py           | 63 +++++++++++++------
 4 files changed, 117 insertions(+), 53 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_backend.py b/experimental/CollectiveX/bench/ep_backend.py
index 2741e719a..b3a7d171a 100644
--- a/experimental/CollectiveX/bench/ep_backend.py
+++ b/experimental/CollectiveX/bench/ep_backend.py
@@ -132,27 +132,30 @@ class EPBackend(abc.ABC):
     fp8_consume = os.environ.get("CX_FP8_CONSUME", "native")
 
     @property
-    def stages_fp8_natively(self) -> bool:
-        """Whether the chained roundtrip should skip the per-iteration `stage()`.
-
-        Gated on precision, NOT on `stage_device_work` alone. The two are equivalent for
-        deepep-v2 and uccl, but MoRI sets `stage_device_work = self._fp8 or not
-        self._external_input`, so its scale-up kernels (IntraNode/IntraNodeLL) report True
-        for BF16 as well -- and their `stage()` really does run a device copy into the
-        registered combine-input buffer. That copy is not an fp8 dequant and nothing in this
-        change's evidence says it should leave the timed region, so BF16 keeps executing it
-        inline every iteration and its numbers are unmoved.
-
-        (Whether MoRI's registered-buffer copy is a production cost or a harness artefact is
-        a real open question -- a native integration may have the expert GEMM write straight
-        into that buffer -- but it is a separate question from fp8 consumption, it applies to
-        both precisions equally, and it needs its own evidence.)
+    def stage_excluded_from_roundtrip(self) -> bool:
+        """Whether the chained roundtrip skips the per-iteration `stage()`.
+
+        `roundtrip` must mean the same thing in every row, or it cannot be compared across
+        backends. It means dispatch -> combine: the transport, staging excluded. So the
+        answer is yes whenever `stage()` does device work, regardless of precision.
+
+        This was previously gated on precision as well, which left `stage` inside the
+        roundtrip for exactly two configurations -- MoRI BF16 scale-up and FlashInfer BF16 --
+        and transport-only for all 800+ other rows, so the headline compared two different
+        quantities and penalised those two.
+
+        Gated on `stage_device_work` rather than applied blanket: where `stage()` is a bare
+        pointer assignment there is nothing to lift, and hoisting anyway would hand the
+        low-latency backends a VIEW into their double-buffered receive, whose parity flips on
+        each timed re-dispatch -- combine would then read the stale-parity buffer.
+
+        `CX_FP8_CONSUME=dequant` still opts an fp8 run back into the inline stage, because
+        that switch exists to model a stack that really does dequantise between the two
+        collectives (see `fp8_consume`).
         """
-        return (
-            self.precision == "fp8"
-            and self.stage_device_work
-            and self.fp8_consume == "native"
-        )
+        if not self.stage_device_work:
+            return False
+        return not (self.precision == "fp8" and self.fp8_consume == "dequant")
 
     def __init_subclass__(cls, **kwargs):
         super().__init_subclass__(**kwargs)
@@ -368,12 +371,17 @@ def benchmark_roundtrip(self, problem, warmup, iters):
 
         self.warm(problem, warmup)
         staged = None
-        if self.stages_fp8_natively:
-            # Materialise the expert-output stand-in ONCE, untimed. A native fp8 stack has no
-            # separate conversion between dispatch and combine, so the chained measurement
-            # must not contain one. Routing is fixed for a ladder point, so the same staged
-            # tensor is valid for every iteration (for MoRI it IS the registered combine
-            # buffer, already filled).
+        if self.stage_excluded_from_roundtrip:
+            # Materialise the expert-output stand-in ONCE, untimed, so the chained
+            # measurement is dispatch -> combine and nothing else. Routing is fixed for a
+            # ladder point, so the same staged tensor is valid for every iteration (for MoRI
+            # it IS the registered combine buffer, already filled; for FlashInfer it is the
+            # workspace combine region, which dispatch cannot clobber because that region
+            # sits past the end of every dispatch receive plane).
+            #
+            # Read the staged payload back through `combine_input_attr` rather than
+            # constructing one: nccl-ep keeps an `nccl.ep.Tensor` wrapper there, and
+            # `run_roundtrip` restores that same object, so no type ever changes hands.
             handle = self.dispatch(problem)
             self.stage(problem, handle)
             staged = getattr(handle, self.combine_input_attr)
diff --git a/experimental/CollectiveX/bench/ep_flashinfer.py b/experimental/CollectiveX/bench/ep_flashinfer.py
index 22a2eb1ac..8977fa109 100644
--- a/experimental/CollectiveX/bench/ep_flashinfer.py
+++ b/experimental/CollectiveX/bench/ep_flashinfer.py
@@ -186,6 +186,21 @@ def _combine_buffer(self, h):
             h.tokens, h.recv_x.shape[-1], h.recv_x.dtype
         )
 
+    def _filled_slot_index(self, p, h):
+        """Row indices of the receive slots dispatch actually filled, resolved once per rung.
+
+        `_valid_rows` gives a boolean mask, and indexing with one needs the match count on the
+        host, which would put a device read inside the timed stage. Routing is fixed for a
+        ladder point, so resolve it to an integer index on first use -- which is always the
+        untimed `warm()` pass -- and cache it on the problem, the way `warm` already caches
+        `recv_tokens` for the same reason.
+        """
+        index = getattr(p, "flashinfer_filled_slots", None)
+        if index is None:
+            index = self._valid_rows(h).nonzero(as_tuple=True)[0]
+            p.flashinfer_filled_slots = index
+        return index
+
     def stage(self, p, h):
         """Materialise the combine payload in the workspace region the API designates.
 
@@ -194,9 +209,18 @@ def stage(self, p, h):
         staging copy. Copying here rather than handing `combine` a caller-owned tensor keeps
         that copy out of the combine measurement, where production does not pay it; it is
         still executed and reported, as `stage`.
+
+        Only the filled slots are copied, which is what the kernel's own staging path does --
+        `moeA2APrepareCombineKernel` returns early on `token_idx >= recv_counters[source]`.
+        Copying the whole plane moved 1/occupancy times too much: measured occupancy is
+        ~0.66 at EP8 and ~0.41 at EP16, i.e. 1.5x and 2.4x over-copy. The slots left
+        untouched are never read -- combine addresses peers exclusively through the
+        `topk_send_indices` recorded at dispatch, and an unfilled slot has none.
         """
         buffer = self._combine_buffer(h)
-        buffer.copy_(h.recv_x)
+        filled = self._filled_slot_index(p, h)
+        flat_buffer = buffer.view(-1, buffer.shape[-1])
+        flat_buffer[filled] = h.recv_x.view(-1, h.recv_x.shape[-1])[filled]
         h.combine_input = buffer
 
     def combine(self, p, h):
diff --git a/experimental/CollectiveX/bench/ep_mori.py b/experimental/CollectiveX/bench/ep_mori.py
index cb3c43951..9a671c9ee 100644
--- a/experimental/CollectiveX/bench/ep_mori.py
+++ b/experimental/CollectiveX/bench/ep_mori.py
@@ -267,16 +267,25 @@ def stage(self, p, h):
         rows = getattr(p, "recv_tokens", None)
         if not isinstance(rows, int) or rows < 0 or rows > h.dispatch_output.size(0):
             raise RuntimeError("MoRI receive count was not validated before staging")
-        # FP8: dispatch delivered an e4m3 payload; dequantize it to the BF16 combine sends.
-        h.combine_input = (
-            h.dispatch_output.to(torch.bfloat16) if self._fp8 else h.dispatch_output
-        )
         if self._external_input:
+            # The kernel reads the padded plane directly here, so all of it must be BF16.
+            h.combine_input = (
+                h.dispatch_output.to(torch.bfloat16) if self._fp8 else h.dispatch_output
+            )
             return None
+        # Zero-copy path: combine only ever reads the `rows` slots dispatch filled, so cast
+        # and copy just those. `dispatch_output` is sized to the buffer cap times the world,
+        # independent of the token count, so casting all of it made this stage flat in T --
+        # 61-114us across the whole decode ladder against 6-30us for BF16, ~99.8% of it
+        # padding at T=1.
+        source = h.dispatch_output[:rows]
+        if self._fp8:
+            # FP8: dispatch delivered an e4m3 payload; dequantize it to the BF16 combine sends.
+            source = source.to(torch.bfloat16)
         buffer = self.op.get_registered_combine_input_buffer(
-            torch.bfloat16, hidden_dim=h.combine_input.size(1)
+            torch.bfloat16, hidden_dim=h.dispatch_output.size(1)
         )
-        buffer[:rows, :].copy_(h.combine_input[:rows, :])
+        buffer[:rows, :].copy_(source)
         h.combine_input = buffer
 
     def combine(self, p, h):
diff --git a/experimental/CollectiveX/tests/test_roundtrip_staging.py b/experimental/CollectiveX/tests/test_roundtrip_staging.py
index b6338fc50..d9f2ec59f 100644
--- a/experimental/CollectiveX/tests/test_roundtrip_staging.py
+++ b/experimental/CollectiveX/tests/test_roundtrip_staging.py
@@ -64,8 +64,9 @@ def test_staged_input_keeps_the_conversion_out_of_the_chain(self):
         self.assertNotIn("stage", b.calls)
 
     def test_without_staged_input_the_stage_runs_inline(self):
-        # The fp8 `dequant` model takes this path, and so does BF16 — free for the adapters
-        # whose receive buffer is already the combine input, real work for mori/flashinfer-ep.
+        # The fp8 `dequant` model takes this path, as does any backend whose `stage` is a bare
+        # pointer assignment. mori/flashinfer-ep no longer do: their real device copy is
+        # hoisted, so the chained roundtrip is dispatch -> combine for every row.
         b = _StubBackend(stage_device_work=True, fp8_consume="dequant")
         b.run_roundtrip(object())
         self.assertEqual(b.calls, ["dispatch", "stage", "combine(staged-by-stage)"])
@@ -84,43 +85,65 @@ def test_default_models_the_native_path(self):
         self.assertEqual(ep_backend.EPBackend.fp8_consume, "native")
 
 
-class NativeStagingGate(unittest.TestCase):
-    """`stage_device_work` does NOT imply fp8, so the gate must check precision.
+class RoundtripStagingGate(unittest.TestCase):
+    """`roundtrip` must mean dispatch -> combine in EVERY row, or it is not comparable.
 
-    MoRI sets `stage_device_work = self._fp8 or not self._external_input`, so its scale-up
-    kernels report True for BF16 too, and their stage() does a real copy into the registered
-    combine-input buffer. Gating on stage_device_work alone silently lifted that copy out of
-    the BF16 timed region -- a precision-asymmetric change of exactly the kind this file
-    exists to prevent.
+    The gate was previously precision-dependent, which left `stage` inside the roundtrip for
+    exactly two configurations -- MoRI BF16 scale-up and FlashInfer BF16 -- and transport-only
+    for every other, so the headline compared two different quantities. It is now gated on
+    `stage_device_work` alone, with `CX_FP8_CONSUME=dequant` as the sole opt-out.
     """
 
-    def test_bf16_with_device_staging_keeps_the_stage_inline(self):
+    def test_bf16_with_device_staging_now_lifts_the_copy_out(self):
+        # MoRI BF16 scale-up and FlashInfer BF16: a real device copy, previously charged to
+        # the chained roundtrip and to nothing else's.
         mori_intranode_bf16 = _StubBackend(
             stage_device_work=True, fp8_consume="native", precision="bf16"
         )
-        self.assertFalse(mori_intranode_bf16.stages_fp8_natively)
-        mori_intranode_bf16.run_roundtrip(object())
+        self.assertTrue(mori_intranode_bf16.stage_excluded_from_roundtrip)
+        mori_intranode_bf16.run_roundtrip(object(), staged="pre-materialised")
         self.assertEqual(
-            mori_intranode_bf16.calls, ["dispatch", "stage", "combine(staged-by-stage)"]
+            mori_intranode_bf16.calls, ["dispatch", "combine(pre-materialised)"]
         )
 
     def test_fp8_with_device_staging_lifts_the_conversion_out(self):
         self.assertTrue(
             _StubBackend(
                 stage_device_work=True, fp8_consume="native", precision="fp8"
-            ).stages_fp8_natively
+            ).stage_excluded_from_roundtrip
         )
 
-    def test_the_hatch_and_no_op_stages_never_take_the_fast_path(self):
-        self.assertFalse(  # CX_FP8_CONSUME=dequant restores the inline stage
+    def test_a_no_op_stage_is_never_hoisted(self):
+        # deepep-v2 / uccl-ep / nccl-ep at BF16: `stage` is a pointer assignment, so there is
+        # nothing to lift -- and hoisting anyway would hand the low-latency backends a view
+        # into their double-buffered receive, whose parity flips on each re-dispatch.
+        self.assertFalse(
             _StubBackend(
-                stage_device_work=True, fp8_consume="dequant", precision="fp8"
-            ).stages_fp8_natively
+                stage_device_work=False, fp8_consume="native", precision="bf16"
+            ).stage_excluded_from_roundtrip
         )
-        self.assertFalse(  # nccl-ep / bf16 deepep-v2: nothing to lift
+        self.assertFalse(
             _StubBackend(
                 stage_device_work=False, fp8_consume="native", precision="fp8"
-            ).stages_fp8_natively
+            ).stage_excluded_from_roundtrip
+        )
+
+    def test_the_dequant_hatch_restores_the_inline_stage(self):
+        # CX_FP8_CONSUME=dequant models a stack that really does convert between the two
+        # collectives, so that run wants the stage back inside the chain.
+        backend = _StubBackend(
+            stage_device_work=True, fp8_consume="dequant", precision="fp8"
+        )
+        self.assertFalse(backend.stage_excluded_from_roundtrip)
+        backend.run_roundtrip(object())
+        self.assertEqual(backend.calls, ["dispatch", "stage", "combine(staged-by-stage)"])
+
+    def test_the_hatch_does_not_apply_to_bf16(self):
+        # The hatch is about fp8 consumption; a BF16 row has no conversion to model.
+        self.assertTrue(
+            _StubBackend(
+                stage_device_work=True, fp8_consume="dequant", precision="bf16"
+            ).stage_excluded_from_roundtrip
         )
 
 

From 807e13b3946531ddd48441e196afd34d1a664fce Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 15:48:29 +0800
Subject: [PATCH 02/34] CollectiveX: mark the roundtrip contract change and
 correct its docs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：标记 roundtrip 契约变更并修正相关文档

methodology.md described the headline as containing expert-output staging, which is now the
opposite of what it measures; three docstrings and one comment still described the old gate,
including the test module's own premise. Bump the sweep version so the durable store can tell
the two measurement generations apart, and emit stage_excluded_from_roundtrip per row: without
it a MoRI BF16 row measured before this change is byte-indistinguishable from one measured
after, while meaning something different.

Also scope the dequant hatch's historical-reproduction claim, which no longer holds for MoRI
fp8 now that its stage casts only the rows dispatch filled.
---
 experimental/CollectiveX/bench/ep_backend.py  | 24 ++++++++++++-------
 experimental/CollectiveX/bench/ep_harness.py  |  8 +++++++
 experimental/CollectiveX/configs/sweep.json   |  2 +-
 experimental/CollectiveX/docs/methodology.md  |  5 +++-
 .../tests/test_roundtrip_staging.py           |  6 +++--
 5 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_backend.py b/experimental/CollectiveX/bench/ep_backend.py
index b3a7d171a..2d14db945 100644
--- a/experimental/CollectiveX/bench/ep_backend.py
+++ b/experimental/CollectiveX/bench/ep_backend.py
@@ -110,8 +110,12 @@ class EPBackend(abc.ABC):
     #
     # `dequant` is a VERIFICATION HATCH, not a second metric: never a sweep axis, never a
     # default. It is retained because it costs nothing (BF16 needs the same staged-is-None
-    # branch) and because it reproduces historical numbers exactly for regression checks --
-    # measured 302.0us against 302.5us in run 30177021271 at T=1.
+    # branch) and because it reproduces historical numbers for regression checks on the
+    # backends whose stage this repo has not since changed -- measured 302.0us against 302.5us
+    # in run 30177021271 at T=1 for deepep-v2/uccl-ep. It no longer reproduces MoRI fp8, whose
+    # stage now casts only the rows dispatch filled rather than the whole padded plane; and no
+    # env combination reproduces pre-hoist BF16 MoRI/FlashInfer roundtrips, because the hatch
+    # is fp8-only by design.
     #
     # A second measured mode is unnecessary because the mismatched-config cost is DERIVABLE
     # from what every run already emits:
@@ -341,11 +345,11 @@ def warm(self, problem, count):
     def run_roundtrip(self, problem, staged=None):
         """One chained round trip; returns combined activations.
 
-        `staged` supplies a pre-materialised combine input so the conversion pass stays out of
-        the timed region (see `fp8_consume`). When it is None the stage runs inline, which is
-        the `dequant` fp8 model and the BF16 path -- free for the adapters whose
-        received buffer is already the combine input, real device work for the ones that
-        must place it (mori, flashinfer-ep; both declare `stage_device_work`).
+        `staged` supplies a pre-materialised combine input so staging stays out of the timed
+        region, which is the default for every backend that does device work there (see
+        `stage_excluded_from_roundtrip`). It is None only when `stage()` is a bare pointer
+        assignment -- deepep-v2, uccl-ep and nccl-ep at BF16, where there is nothing to lift --
+        or under the `CX_FP8_CONSUME=dequant` hatch, which wants the conversion back in the chain.
         """
         handle = self.dispatch(problem)
         if staged is None:
@@ -380,8 +384,10 @@ def benchmark_roundtrip(self, problem, warmup, iters):
             # sits past the end of every dispatch receive plane).
             #
             # Read the staged payload back through `combine_input_attr` rather than
-            # constructing one: nccl-ep keeps an `nccl.ep.Tensor` wrapper there, and
-            # `run_roundtrip` restores that same object, so no type ever changes hands.
+            # constructing one, so whatever the adapter put there round-trips unchanged. No
+            # current backend needs that -- nccl-ep is the one whose attribute holds a
+            # non-torch wrapper, and its `stage()` does no device work so it never reaches
+            # here -- but constructing a tensor instead would silently break it if it did.
             handle = self.dispatch(problem)
             self.stage(problem, handle)
             staged = getattr(handle, self.combine_input_attr)
diff --git a/experimental/CollectiveX/bench/ep_harness.py b/experimental/CollectiveX/bench/ep_harness.py
index ba34e135f..a25ae1248 100644
--- a/experimental/CollectiveX/bench/ep_harness.py
+++ b/experimental/CollectiveX/bench/ep_harness.py
@@ -1204,6 +1204,14 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
             # pick this per installed library version (flashinfer-ep does), so without it
             # a wheel bump silently changes the arithmetic behind `passed` with no trace.
             "combine_reduction": getattr(backend, "combine_reduction", "domain-fp32"),
+            # Whether `roundtrip` excludes expert-output staging. It always does now, unless
+            # the CX_FP8_CONSUME=dequant hatch is set, but it did not always: rows measured
+            # before that change carried the staging copy inside the chain for MoRI BF16 and
+            # FlashInfer BF16 only. Without this field those rows are indistinguishable from
+            # these ones while measuring a different quantity.
+            "stage_excluded_from_roundtrip": bool(
+                getattr(backend, "stage_excluded_from_roundtrip", False)
+            ),
             # See EPBackend.maturity: a "candidate" row measures the library, not a deployment.
             "maturity": getattr(backend, "maturity", None) or "unknown",
             "name": backend.name,
diff --git a/experimental/CollectiveX/configs/sweep.json b/experimental/CollectiveX/configs/sweep.json
index 22c873e9f..89313eab8 100644
--- a/experimental/CollectiveX/configs/sweep.json
+++ b/experimental/CollectiveX/configs/sweep.json
@@ -1,5 +1,5 @@
 {
-  "version": 1,
+  "version": 2,
   "suite": "ep-core",
   "modes": {
     "normal": ["decode", "prefill"],
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 8108c3fd4..56d20a7b3 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -115,7 +115,10 @@ Adapters may not generate routing or reinterpret one quantity as the other.
 
 Normal mode uses `layout-and-dispatch-v1`: dispatch timing includes layout plus communication, and
 combine returns activation payload through an unweighted rank-sum path. Expert-output staging is
-outside isolated combine timing and inside the measured paired roundtrip. Each component declares
+outside isolated combine timing AND outside the measured paired roundtrip, so `roundtrip` means
+dispatch then combine — the transport — in every row. It is reported as its own `stage` component
+wherever it does device work. The one exception is the `CX_FP8_CONSUME=dequant` verification hatch,
+which puts the conversion back inside the chain on purpose. Each component declares
 availability, origin, and sample count. A paired-only API reports null isolated components.
 `isolated_sum` is derived. The artifact records the mode so a reader can keep distinct measurement
 contracts separate.
diff --git a/experimental/CollectiveX/tests/test_roundtrip_staging.py b/experimental/CollectiveX/tests/test_roundtrip_staging.py
index d9f2ec59f..3906cbbd7 100644
--- a/experimental/CollectiveX/tests/test_roundtrip_staging.py
+++ b/experimental/CollectiveX/tests/test_roundtrip_staging.py
@@ -1,8 +1,10 @@
 #!/usr/bin/env python3
 """Contract for what the chained roundtrip measures.
 
-`stage` exists only for FP8 (`stage_device_work = self._fp8`), so charging it to the chained
-roundtrip compares FP8 and BF16 through structurally different pipelines. Real stacks decide
+`stage` is not an FP8-only cost: deepep-v2 and uccl-ep set `stage_device_work = self._fp8`, but
+MoRI sets `self._fp8 or not self._external_input` and FlashInfer sets it unconditionally, so BF16
+rows on those two do real device work there. Charging it to the chained roundtrip therefore made
+`roundtrip` mean different things in different rows. Real stacks decide
 this on quant-format match: SGLang's DeepEP dispatcher contains no dequant at all, and vLLM
 returns the dispatched fp8 + scales untouched when `block_k == DEEPEP_QUANT_BLOCK_SIZE`,
 dequantising only as a mismatch fallback. These tests pin both models.

From 8dde2ecd7e238f78a4db86c26d15d049a97b497f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 15:52:14 +0800
Subject: [PATCH 03/34] CollectiveX: publish the entry-skew bracket beside the
 headline
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：在头条指标旁发布 entry-skew 区间

Entry stagger is charged to the cross-rank MAX, and how much depends on the code path AND the
precision rather than only the fleet: on identical h200 low-latency decode cells the spread is
~9.3us for deepep-v2 and uccl-ep at BF16 (shared legacy Buffer path) against ~2.6us for
nccl-ep, and collapses to ~2.8us for those two under FP8 where the in-kernel quantise makes
dispatch self-align the ranks. So MAX taxes some rows more than others and the term cannot be
subtracted in a principled way.

Keep MAX as the headline — a layer is not done until its slowest rank is, and MIN would launder
stagger the backend itself causes — but emit the bracket the harness already measures, and state
the rule: rank on roundtrip p50 only where MAX and MIN agree, never on p99 of MAX for multi-node
decode, and read isolated components as residual-wait diagnostics.
---
 experimental/CollectiveX/docs/methodology.md | 21 +++++++++++-
 experimental/CollectiveX/summarize.py        | 34 ++++++++++++++++----
 2 files changed, 48 insertions(+), 7 deletions(-)

diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 56d20a7b3..7e3f90406 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -120,7 +120,26 @@ dispatch then combine — the transport — in every row. It is reported as its
 wherever it does device work. The one exception is the `CX_FP8_CONSUME=dequant` verification hatch,
 which puts the conversion back inside the chain on purpose. Each component declares
 availability, origin, and sample count. A paired-only API reports null isolated components.
-`isolated_sum` is derived. The artifact records the mode so a reader can keep distinct measurement
+`isolated_sum` is derived.
+
+Headline latency is the p50 of the per-iteration cross-rank MAX: a layer is not finished until
+its slowest rank is, so MAX is the completion cost, and it charges inter-rank entry stagger to
+whichever component the ranks entered unevenly. How much stagger there is depends on the code
+path AND the precision, not only on the fleet: on identical h200 low-latency decode cells the
+per-iteration spread is ~9.3 us for deepep-v2 and uccl-ep at BF16 (they share the legacy
+`Buffer` path) against ~2.6 us for nccl-ep, and it collapses to ~2.8 us for those same two under
+FP8, where the kernel quantises in-kernel and the heavier dispatch self-aligns the ranks. So the
+term is not subtractable in any principled way, and MAX alone taxes some rows more than others.
+
+Every row therefore also carries `cross_rank_min_us` (the same iterations reduced with MIN — the
+skew-excluded floor) and `cross_rank_spread_us` (per-iteration MAX minus MIN). Read MAX and MIN
+as a bracket. Two cells whose MAX gap is smaller than the larger contender's spread are not
+separated by the data: rank on roundtrip p50 and call a winner only where MAX and MIN agree on
+the ordering. Do not rank on p99 of MAX for multi-node decode cells, where it is dominated by
+worst-rank stalls rather than transport — p99 of MIN is the synchronized-cost tail beside it. The
+isolated components inherit the preceding operation's per-rank exit stagger, so treat them as
+residual-wait diagnostics rather than per-operation costs; the paired roundtrip is the
+comparable quantity. The artifact records the mode so a reader can keep distinct measurement
 contracts separate.
 
 Every measured component uses one fixed timing profile, defined once in `configs/sweep.json`
diff --git a/experimental/CollectiveX/summarize.py b/experimental/CollectiveX/summarize.py
index 5785d6757..64a464408 100644
--- a/experimental/CollectiveX/summarize.py
+++ b/experimental/CollectiveX/summarize.py
@@ -43,11 +43,32 @@ def _identity(document: dict) -> tuple[str, str, str, str, str, str, int, str]:
     )
 
 
-def _headline(document: dict) -> tuple[int | str, float | str, float | str]:
+def _headline(document: dict) -> tuple:
+    """Headline row, with the skew bracket beside it.
+
+    `p50`/`p99` are the cross-rank MAX: a layer is not finished until its slowest rank is, so
+    MAX is the completion cost. But entry stagger is charged to it, and how much is a property
+    of the BACKEND, not just the fleet — on identical h200 low-latency cells the per-iteration
+    spread is 9.2us for deepep-v2 and uccl-ep (which share a kernel family) against 2.0us for
+    nccl-ep, flat across the whole ladder. So MAX alone taxes some backends more than others,
+    and two cells whose MAX gap is smaller than that spread are not separated by the data.
+    `min50` is the same iterations reduced with MIN — the skew-excluded floor — and `skew` is
+    the per-iteration MAX-MIN. Read the pair as a bracket, not the two ends as rival metrics.
+    """
     rows = document["measurement"]["rows"]
     row = next((item for item in rows if item["tokens_per_rank"] == 64), rows[len(rows) // 2])
     latency = row["components"]["roundtrip"]["percentiles_us"]
-    return row["tokens_per_rank"], latency["p50"], latency["p99"]
+
+    def percentile(block: str, name: str) -> float | str:
+        # Absent on rows measured before the skew diagnostics were emitted.
+        component = (row.get(block) or {}).get(name) or row.get(block) or {}
+        return (component.get("percentiles_us") or {}).get("p50", "-")
+
+    return (
+        row["tokens_per_rank"], latency["p50"], latency["p99"],
+        percentile("cross_rank_min_us", "roundtrip"),
+        percentile("cross_rank_spread_us", ""),
+    )
 
 
 def render(documents: list[dict]) -> str:
@@ -63,16 +84,17 @@ def render(documents: list[dict]) -> str:
         )
         lines.append("")
     lines += [
-        "| ver | sku | backend | mode | precision | suite | phase | routing | ep | outcome | T* | p50 us | p99 us |",
-        "|--:|---|---|---|---|---|---|---|--:|---|--:|--:|--:|",
+        "| ver | sku | backend | mode | precision | suite | phase | routing | ep | outcome "
+        "| T* | p50 us | p99 us | min50 us | skew us |",
+        "|--:|---|---|---|---|---|---|---|--:|---|--:|--:|--:|--:|--:|",
     ]
     for document in documents:
         sku, backend, suite, routing, mode, phase, ep, precision = _identity(document)
-        token, p50, p99 = _headline(document)
+        token, p50, p99, min50, skew = _headline(document)
         lines.append(
             f"| {document['version']} | {sku} | `{backend}` | {mode} | {precision} | {suite} | "
             f"{phase} | {routing} | {ep} | "
-            f"{document['outcome']['status']} | {token} | {p50} | {p99} |"
+            f"{document['outcome']['status']} | {token} | {p50} | {p99} | {min50} | {skew} |"
         )
     if not documents:
         lines.append("\n> No valid native outcome documents found.")

From a7e8884a696791aaa6f230c12a0aff92b2cfcec5 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 16:02:29 +0800
Subject: [PATCH 04/34] CollectiveX: size nccl-ep HT combine to its receive
 count, align MoRI's LL cap
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：将 nccl-ep HT combine 按实际接收数取切片，并对齐 MoRI 的 LL 容量

nccl-ep HT combine's staging copy is sized by the tensor it is handed -- upstream reads
num_tokens = x->sizes[0] -- not by the group's buffer, so passing the whole ladder-max receive
plane made it copy max(ladder) * world rows on every call regardless of T. That put a
rung-independent floor under the measurement: ~55-80us on a decode leg against ~470-1295us on a
prefill leg whose ladder maximum is 16x larger, while the per-token slope stayed within 12%.
Slice to the count the metadata exchange already reports, which is what upstream's own ep_test
does. LL keeps the full padded plane; its kernel asserts that shape.

MoRI had no buffer_cap override, so its low-latency ladder ran to T=512 while every other
backend stopped at 256 -- a top rung no cross-vendor comparison could use. 256 is also vLLM's
DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP.
---
 experimental/CollectiveX/bench/ep_mori.py     | 20 +++++++++--
 experimental/CollectiveX/bench/ep_nccl.py     | 34 ++++++++++++++++---
 .../CollectiveX/tests/test_ep_nccl_handle.py  | 23 +++++++++++++
 3 files changed, 70 insertions(+), 7 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_mori.py b/experimental/CollectiveX/bench/ep_mori.py
index 9a671c9ee..f0f40509b 100644
--- a/experimental/CollectiveX/bench/ep_mori.py
+++ b/experimental/CollectiveX/bench/ep_mori.py
@@ -127,6 +127,17 @@ def __init__(self, args, rank, world_size, local_rank, device):
         # Stash the __init__-only locals the moved create_buffer body reads back.
         self._gpus_per_node = gpus_per_node
 
+    def buffer_cap(self, args):
+        if self.mode == "low-latency":
+            # 256 tokens/rank, matching deepep-v2, uccl-ep and nccl-ep, so every backend's
+            # low-latency ladder ends at the same rung. MoRI imposes no bound of its own here
+            # -- this adapter previously allowed 512, which published a T=512 decode rung no
+            # other backend has, so the top of the ladder was unusable for any cross-vendor
+            # comparison. 256 is also vLLM's DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP,
+            # i.e. the capacity a deployment actually configures.
+            return 256
+        return None
+
     def create_buffer(self, spec):
         args, world_size, rank = self.args, self.world_size, self.rank
         gpus_per_node = self._gpus_per_node
@@ -151,8 +162,13 @@ def create_buffer(self, spec):
                 f"MoRI realized {realized_qps} QPs per PE; {self.num_qps} required"
             )
 
-        # MoRI preallocates one communicator buffer for the case's entire ladder.
-        self._cap = max(512, spec.max_tokens_per_rank)
+        # MoRI preallocates one communicator buffer for the case's entire ladder. 256 matches
+        # the other three backends' low-latency cap and vLLM's own
+        # DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP; the previous 512 was this adapter's
+        # invention -- no MoRI bound requires it -- and it published a T=512 low-latency rung
+        # that no other backend has, so no cross-vendor comparison could use the top of the
+        # ladder. Normal mode still takes the ladder maximum, which is larger.
+        self._cap = max(256, spec.max_tokens_per_rank)
         # quant_type stays "none" for both precisions: dispatch precision is carried by
         # the passed tensor dtype (caller-prequantized e4m3 under FP8, BF16 otherwise),
         # and "none" keeps combine a genuine BF16 send. data_type is deprecated upstream
diff --git a/experimental/CollectiveX/bench/ep_nccl.py b/experimental/CollectiveX/bench/ep_nccl.py
index 5a616add2..61cbbca2e 100644
--- a/experimental/CollectiveX/bench/ep_nccl.py
+++ b/experimental/CollectiveX/bench/ep_nccl.py
@@ -309,7 +309,7 @@ def _ensure_handle(self, p):
             h.handle = self._handle
             torch.cuda.synchronize()
             if not self._ll:
-                h.count = int(h.recv_total.item())
+                self._bind_ht_recv_count(h)
             self._bound = h
         else:
             h.handle = self._handle
@@ -317,6 +317,26 @@ def _ensure_handle(self, p):
         p._nccl = h
         return h
 
+    def _bind_ht_recv_count(self, h):
+        """Read HT's received-token count and pre-wrap the combine input at that size.
+
+        The combine call's staging copy is sized by the tensor it is HANDED -- upstream reads
+        `num_tokens = x->sizes[0]` and copies that many rows into the group's IPC staging --
+        not by the group's buffer. Handing it the whole ladder-max receive plane therefore made
+        HT combine copy `max(ladder) * world` rows on every call regardless of T, putting a
+        rung-independent floor under it: ~55-80us on a decode leg (ladder max 512) against
+        ~470-1295us on a prefill leg (max 8192), while the per-token slope stayed within 12%.
+        Slicing is free (a contiguous leading-dim view) and is what upstream's own ep_test does
+        when it sizes the combine input to the actual receive count.
+
+        Both callers are untimed -- handle creation and rebind, which only run on a shape change
+        -- so the `.item()` read never lands in a measured window.
+        """
+        h.count = int(h.recv_total.item())
+        # A rank that received nothing still needs a non-empty tensor for the shape checks; the
+        # routing map decides what combine reads, so the extra row cannot reach the output.
+        h.combine_in_t = self._t(self._recv_x[: max(h.count, 1)])
+
     def _rebind(self, h):
         """Point the single handle at h's routing (collective; untimed callers only).
 
@@ -332,7 +352,7 @@ def _rebind(self, h):
         )
         torch.cuda.synchronize()
         if not self._ll:
-            h.count = int(h.recv_total.item())
+            self._bind_ht_recv_count(h)
         self._bound = h
 
     # ---- transport contract ------------------------------------------------------------------
@@ -375,8 +395,11 @@ def dispatch(self, p):
 
     def stage(self, p, h):
         # BF16 combine input is the received buffer itself; no device work (value correctness
-        # is exercised only through the oracle's combine_transformed path).
-        h.combine_input_t = self._recv_x_t
+        # is exercised only through the oracle's combine_transformed path). LL takes the full
+        # padded plane -- its kernel asserts that shape -- while HT takes only the rows this
+        # routing actually received, so its staging copy scales with T (see
+        # `_bind_ht_recv_count`).
+        h.combine_input_t = self._recv_x_t if self._ll else h.combine_in_t
 
     def combine(self, p, h):
         stream = self._stream()
@@ -489,7 +512,8 @@ def combine_transformed(self, p, h, transformed):
         self._recv_x[: transformed.shape[0]].copy_(transformed.to(self._recv_x.dtype))
         stream = self._stream()
         h.handle.combine(
-            CombineInputs(tokens=self._recv_x_t),
+            # Same sliced input the timed path uses, so the two cannot diverge in shape.
+            CombineInputs(tokens=h.combine_in_t),
             CombineOutputs(tokens=h.out_t),
             config=self._combine_cfg,
             stream=stream,
diff --git a/experimental/CollectiveX/tests/test_ep_nccl_handle.py b/experimental/CollectiveX/tests/test_ep_nccl_handle.py
index 943625ee7..09055abaa 100644
--- a/experimental/CollectiveX/tests/test_ep_nccl_handle.py
+++ b/experimental/CollectiveX/tests/test_ep_nccl_handle.py
@@ -96,6 +96,10 @@ def backend(ll=True):
     b.args = types.SimpleNamespace(hidden=16)
     b._t = lambda x: x
     b._stream = lambda: 0
+    # create_buffer always runs before the first _ensure_handle, so the HT receive plane
+    # exists by then: `_bind_ht_recv_count` slices it to the received-token count. A list
+    # stands in for the tensor because `_t` is identity here and only the slice is exercised.
+    b._recv_x = list(range(64))
     return b
 
 
@@ -146,6 +150,25 @@ def test_ll_never_passes_layout_info_on_rebind(self):
         b._ensure_handle(problem(2))
         self.assertEqual([info for _, info in b._ep_group.handle.updates], [None])
 
+    def test_ht_combine_input_is_sliced_to_the_received_count(self):
+        """HT combine's staging copy is sized by the tensor it is handed, not by the group.
+
+        Passing the whole ladder-max receive plane made combine copy max(ladder) * world rows
+        on every call regardless of T, which put a rung-independent floor under it. LL must keep
+        the full padded plane -- its kernel asserts that shape.
+        """
+        b = backend(ll=False)
+        h = b._ensure_handle(problem(1))
+        # 7 is what the stubbed `torch.zeros(...).item()` reports as the received count.
+        self.assertEqual(h.count, 7)
+        self.assertEqual(h.combine_in_t, list(range(7)))
+        # The point of the fix: the slice, not the whole 64-row plane.
+        self.assertLess(len(h.combine_in_t), len(b._recv_x))
+
+        ll = backend(ll=True)
+        ll_h = ll._ensure_handle(problem(1))
+        self.assertFalse(hasattr(ll_h, "combine_in_t"))
+
     def test_ht_rebind_carries_that_problems_counters(self):
         """HT re-runs the metadata exchange into the rebound problem's own counter tensors."""
         b = backend(ll=False)

From 1c50e472c6b2c0210515742ce6450b5ec8166a6a Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 16:05:40 +0800
Subject: [PATCH 05/34] CollectiveX: state why nccl-ep excludes routing and
 what MoRI's config represents
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：说明 nccl-ep 为何不计入路由绑定，以及 MoRI 配置代表什么

nccl-ep binds routing with a collective whose cost scales with group capacity rather than token
count, so charging it per iteration would import a ladder-max-proportional term into dispatch —
the artifact just removed from combine. It is bound during warm-up, matching NVIDIA's own
ep_bench; low-latency has nothing to exclude because its update returns immediately and the
kernel reads cached routing inside the timed dispatch.

MoRI's pinned MANUAL launch config is what vLLM and SGLang actually run, so its numbers describe
the engine-integrated configuration rather than MoRI's peak. Record the delta to its own tuned
tables (~1.3-1.4x combine at T=256 on gfx950) as context, and why AUTO would not reproduce them.
---
 experimental/CollectiveX/docs/methodology.md | 24 ++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 7e3f90406..14b02d86f 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -57,7 +57,14 @@ request NCCL Device API LSA and fail closed unless the realized LSA team covers
 x86 EP16 scale-out uses the hybrid path with GIN and requires two logical scale-out domains
 represented by two physical RDMA ranks, with eight scale-up ranks per domain. GB EP16 remains MNNVL
 scale-up and uses LSA. MoRI EP8 uses the direct IntraNode kernel on every CDNA SKU; its EP16 InterNodeV1 path is
-configured but unsupported (transport-layer combine corruption, ROCm/mori#475) and never dispatched. UCCL-EP is a drop-in, API-identical DeepEP replacement that keeps the legacy `Buffer`
+configured but unsupported (transport-layer combine corruption, ROCm/mori#475) and never dispatched.
+MoRI runs under its MANUAL launch mode with a pinned launch config, because that is what the engines
+run: neither vLLM nor SGLang sets `MORI_EP_LAUNCH_CONFIG_MODE`, and both pin block_num 80 with
+rdma_block_num 0. These numbers therefore describe the engine-integrated configuration, not MoRI's
+peak -- its own shipped tuning tables reach roughly 1.3-1.4x faster combine at T=256 on gfx950 with a
+config no engine selects, and AUTO would not reproduce those tables anyway (there is no BF16 gfx950
+dispatch rule and no gfx950 IntraNodeLL combine table, so AUTO falls back to hard-coded defaults and
+couples the result to whichever MoRI revision is pinned). UCCL-EP is a drop-in, API-identical DeepEP replacement that keeps the legacy `Buffer`
 `dispatch`/`combine` (unweighted rank-sum) but routes it over CPU-proxy GPUDirect RDMA on plain
 `libibverbs` — no NVSHMEM/IBGDA — with software message ordering, atomics, and flow control; its
 scale-up is single-node `cudaIpc` over NVLink/XGMI (so the scale-up domain is one physical node,
@@ -139,7 +146,20 @@ the ordering. Do not rank on p99 of MAX for multi-node decode cells, where it is
 worst-rank stalls rather than transport — p99 of MIN is the synchronized-cost tail beside it. The
 isolated components inherit the preceding operation's per-rank exit stagger, so treat them as
 residual-wait diagnostics rather than per-operation costs; the paired roundtrip is the
-comparable quantity. The artifact records the mode so a reader can keep distinct measurement
+comparable quantity.
+
+One backend's timed window omits a cost the others pay, deliberately. nccl-ep binds routing with
+`ncclEpUpdateHandle`, a collective whose cost scales with the group's token capacity rather than with
+the token count, so charging it per iteration would import a ladder-max-proportional term into
+dispatch -- the same shape of artifact that sizing HT's combine input to the ladder maximum used to
+put under combine. It is therefore bound during the untimed warm-up, which is also what NVIDIA's own
+`ep_bench` does (CUDA events around dispatch and combine only, handle update outside the loop). In
+low-latency mode there is nothing to exclude: `ncclEpUpdateHandle` returns immediately and the kernel
+reads the cached routing inside the timed dispatch. Every other backend pays its layout per timed
+call -- uccl-ep calls `get_dispatch_layout` inside dispatch; deepep-v2, MoRI and FlashInfer pass
+routing on every call -- and those costs scale with tokens, so they belong in the window.
+
+The artifact records the mode so a reader can keep distinct measurement
 contracts separate.
 
 Every measured component uses one fixed timing profile, defined once in `configs/sweep.json`

From 20960ad453f1f71c72fde14cea21ac42842c077c Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 16:11:59 +0800
Subject: [PATCH 06/34] CollectiveX: cache nccl-ep's low-latency gate wrapper
 per handle
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：按 handle 缓存 nccl-ep 低延迟门控权重包装器

LL applies the routing gate in its combine kernel, and the adapter built a fresh nccl.ep.Tensor
wrapper for the weights on every timed combine while its dispatch path cached every wrapper it
uses. Building one costs a torch resolve, an np.asarray and a cybind allocation, and time_us
charges host work inside the measured window, so this was a per-call tax no other backend paid —
measured as roughly 3-4us of nccl-ep's small-T floor. Wrap it once in _ensure_handle with the
rest, and use the same object on the oracle path so the two cannot diverge.
---
 experimental/CollectiveX/bench/ep_nccl.py     | 11 +++++++++--
 .../CollectiveX/tests/test_ep_nccl_handle.py  | 19 +++++++++++++++++++
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_nccl.py b/experimental/CollectiveX/bench/ep_nccl.py
index 61cbbca2e..6c2d7aa4d 100644
--- a/experimental/CollectiveX/bench/ep_nccl.py
+++ b/experimental/CollectiveX/bench/ep_nccl.py
@@ -280,6 +280,13 @@ def _ensure_handle(self, p):
         )
         if not self._ll:
             h.in_weights_t = self._t(p.topk_weights)
+        else:
+            # LL applies the gate in its COMBINE kernel, not on dispatch. Wrap the weights here
+            # with every other per-handle wrapper rather than per timed combine: building one
+            # costs a torch resolve, an np.asarray and a cybind allocation, and `time_us`
+            # charges host work inside the window, so a fresh wrapper per iteration was a
+            # per-call tax no other backend pays.
+            h.combine_weights_t = self._t(p.topk_weights)
         # combined output is restored to original token order: [num_tokens, hidden].
         h.out = torch.empty((p.T, self.args.hidden), dtype=torch.bfloat16, device=self.device)
         h.out_t = self._t(h.out)
@@ -408,7 +415,7 @@ def combine(self, p, h):
             # source token's gate (CombineOutputs.topk_weights) before the FP32 accumulation.
             h.handle.combine(
                 CombineInputs(tokens=h.combine_input_t),
-                CombineOutputs(tokens=h.out_t, topk_weights=self._t(p.topk_weights)),
+                CombineOutputs(tokens=h.out_t, topk_weights=h.combine_weights_t),
                 config=self._combine_cfg,
                 stream=stream,
             )
@@ -492,7 +499,7 @@ def _ll_combine_transformed(self, p, h, transformed):
         stream = self._stream()
         h.handle.combine(
             CombineInputs(tokens=self._t(combine_buf)),
-            CombineOutputs(tokens=h.out_t, topk_weights=self._t(p.topk_weights)),
+            CombineOutputs(tokens=h.out_t, topk_weights=h.combine_weights_t),
             config=self._combine_cfg,
             stream=stream,
         )
diff --git a/experimental/CollectiveX/tests/test_ep_nccl_handle.py b/experimental/CollectiveX/tests/test_ep_nccl_handle.py
index 09055abaa..f68bb75a2 100644
--- a/experimental/CollectiveX/tests/test_ep_nccl_handle.py
+++ b/experimental/CollectiveX/tests/test_ep_nccl_handle.py
@@ -150,6 +150,25 @@ def test_ll_never_passes_layout_info_on_rebind(self):
         b._ensure_handle(problem(2))
         self.assertEqual([info for _, info in b._ep_group.handle.updates], [None])
 
+    def test_ll_gate_wrapper_is_built_once_per_handle(self):
+        """LL applies the gate in combine, so its weights wrapper must be cached like the rest.
+
+        Building one per timed combine costs a torch resolve, an np.asarray and a cybind
+        allocation, and `time_us` charges host work inside the window — a per-call tax no other
+        backend pays. HT never needs it: FWD forbids input weights on its combine.
+        """
+        ll = backend(ll=True)
+        pa = problem(1)
+        h = ll._ensure_handle(pa)
+        self.assertTrue(hasattr(h, "combine_weights_t"))
+        self.assertEqual(h.combine_weights_t, "w1")
+        # Re-entering the SAME problem -- the timed loop's steady state -- reuses the handle and
+        # therefore the wrapper; a fresh problem object legitimately builds its own.
+        self.assertIs(ll._ensure_handle(pa).combine_weights_t, h.combine_weights_t)
+
+        ht = backend(ll=False)
+        self.assertFalse(hasattr(ht._ensure_handle(problem(1)), "combine_weights_t"))
+
     def test_ht_combine_input_is_sliced_to_the_received_count(self):
         """HT combine's staging copy is sized by the tensor it is handed, not by the group.
 

From f17d37b731f65aaf5ffd5017bcaf138933d78b65 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 16:35:26 +0800
Subject: [PATCH 07/34] CollectiveX: match MoRI's production warps, and surface
 what makes rows incomparable
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：对齐 MoRI 生产环境 warp 数，并在报表中暴露不可比因素

MoRI's combine ran 8 warps while dispatch ran 16; vLLM and SGLang both pin ONE
warp_num_per_block of 16 that applies to both phases, so 16 matches the engine-integrated
config exactly. 16 is also the kernel ceiling (kMaxWarpGroups 8 x kWarpsPerGroup 2, unguarded
groupData[8] indexed by warpId/2), so it cannot go higher.

Emit the copy counts behind the byte figures, keyed on the combine contract rather than on mode:
deepep-v2, uccl-ep and nccl-ep low-latency receive one copy per (token, expert), but MoRI's
IntraNodeLL deduplicates by destination rank, so a blanket "low-latency uses assignment bytes"
rule would overstate MoRI by ~1.5x at EP8. routed_copies stays canonical.

Surface topology and wire basis in the summary table. Two things cannot be fixed by changing
what we time and were previously invisible: GB's "EP8" is two 4-GPU trays inside a 72-GPU MNNVL
domain rather than 8 GPUs in one node, and two low-latency rows at the same token count move
~1.5x different combine traffic depending on which basis their kernels use.
---
 experimental/CollectiveX/bench/ep_harness.py | 22 ++++++++++++
 experimental/CollectiveX/bench/ep_mori.py    | 10 ++++--
 experimental/CollectiveX/summarize.py        | 38 +++++++++++++++++---
 3 files changed, 64 insertions(+), 6 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_harness.py b/experimental/CollectiveX/bench/ep_harness.py
index a25ae1248..3b5ebc453 100644
--- a/experimental/CollectiveX/bench/ep_harness.py
+++ b/experimental/CollectiveX/bench/ep_harness.py
@@ -1066,6 +1066,20 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
             backend.dispatch_value_bytes, backend.dispatch_scale_bytes_per_copy,
         )
         combine_bytes = logical_byte_provenance(rstats["routed_copies"], args.hidden)
+        # Second byte basis, for the backends whose wire really does carry one copy per
+        # (token, expert) rather than per (token, dest-rank). Which basis applies is a property
+        # of the RECEIVE, not of the mode: deepep-v2, uccl-ep and nccl-ep low-latency receive
+        # per assignment, but MoRI's IntraNodeLL genuinely deduplicates, so keying this on
+        # "low-latency" would overstate MoRI by the dedup factor (~1.5x at EP8). Key it on the
+        # combine contract instead, which already encodes the distinction. `routed_copies`
+        # stays the canonical comparable basis; this is emitted alongside so a reader can
+        # convert a low-latency row onto the wire basis without guessing the factor.
+        assignment_copies = int(sum(rstats["expert_assignments_per_rank"]))
+        wire_basis = (
+            "per-assignment"
+            if backend.combine_weight_semantics == "weighted-kernel-sum"
+            else "rank-deduplicated"
+        )
         roundtrip_bytes = {
             field: dispatch_bytes[field] + combine_bytes[field] for field in dispatch_bytes
         }
@@ -1104,6 +1118,14 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
                 "dispatch": dispatch_bytes,
                 "roundtrip": roundtrip_bytes,
                 "stage": stage_bytes,
+                # Copy counts behind the byte figures above, so a reader can rebase them.
+                # `routed` is what they use; `assignments` is the per-(token, expert) count,
+                # and `wire` names which of the two this backend's kernels actually move.
+                "copies": {
+                    "routed": int(rstats["routed_copies"]),
+                    "assignments": assignment_copies,
+                    "wire": wire_basis,
+                },
             },
             "receive": {
                 "max": recv_max,
diff --git a/experimental/CollectiveX/bench/ep_mori.py b/experimental/CollectiveX/bench/ep_mori.py
index f0f40509b..25bfd37dd 100644
--- a/experimental/CollectiveX/bench/ep_mori.py
+++ b/experimental/CollectiveX/bench/ep_mori.py
@@ -80,9 +80,15 @@ def __init__(self, args, rank, world_size, local_rank, device):
         # default; `kernel_type` kwarg omitted); scale-out EP16 uses InterNodeV1, whose
         # required enum member is an image-lineage check.
         # (kernel, generation label, (block_num, rdma_block_num, dispatch_warps, combine_warps))
+        # Scale-up matches the config the engines pin: vLLM and SGLang both set block_num 80,
+        # rdma_block_num 0 and ONE warp_num_per_block of 16 that applies to dispatch and combine
+        # alike, and neither sets MORI_EP_LAUNCH_CONFIG_MODE, so production runs MANUAL with
+        # these numbers. 16 is also the kernel's hard ceiling (kMaxWarpGroups 8 x kWarpsPerGroup
+        # 2, with groupData[8] indexed by warpId/2 and no upstream guard), so it cannot go
+        # higher. The scale-out tuple is left as-is: EP16 is walled and never dispatched.
         kernel_name, self.kernel_generation, blocks = (
             ("InterNodeV1", "inter-node-v1", (96, 64, 8, 8)) if scale_out
-            else ("IntraNode", "intranode", (80, 0, 16, 8))
+            else ("IntraNode", "intranode", (80, 0, 16, 16))
         )
         if self.mode == "low-latency":
             # LOW-LATENCY (decode) mode: IntraNodeLL, the scale-up low-latency kernel. It is
@@ -105,7 +111,7 @@ def __init__(self, args, rank, world_size, local_rank, device):
                     "is out of scope; see platform_config ll_backends)"
                 )
             kernel_name, self.kernel_generation, blocks = (
-                "IntraNodeLL", "intranode-ll", (80, 0, 16, 8)
+                "IntraNodeLL", "intranode-ll", (80, 0, 16, 16)
             )
         self._kernel_type = None
         if kernel_name != "IntraNode":
diff --git a/experimental/CollectiveX/summarize.py b/experimental/CollectiveX/summarize.py
index 64a464408..664e437c3 100644
--- a/experimental/CollectiveX/summarize.py
+++ b/experimental/CollectiveX/summarize.py
@@ -43,6 +43,35 @@ def _identity(document: dict) -> tuple[str, str, str, str, str, str, int, str]:
     )
 
 
+def _topology(document: dict) -> str:
+    """Scale-up shape, because the same `ep` label is not the same hardware.
+
+    GB200/GB300 run 4 GPUs per node inside a 72-GPU MNNVL domain, so their "EP8" spans two
+    trays over a rack fabric while every other SKU's EP8 is 8 GPUs in one node over NVLink or
+    XGMI. Nothing in the timing can fix that; printing it stops a reader comparing the two as
+    though they were the same configuration.
+    """
+    topology = document.get("topology") or {}
+    per_node = topology.get("gpus_per_node")
+    domain = topology.get("scale_up_domain")
+    nodes = topology.get("nodes")
+    if per_node is None or domain is None:
+        return "-"
+    return f"{nodes}x{per_node}/d{domain}"
+
+
+def _wire_basis(document: dict) -> str:
+    """Which copy basis this backend's kernels actually move.
+
+    Low-latency deepep-v2/uccl-ep/nccl-ep receive one copy per (token, expert); MoRI's
+    IntraNodeLL deduplicates by destination rank. At the same token count that is ~1.5x
+    different combine traffic at EP8, so two low-latency rows are not doing equal work.
+    """
+    rows = document["measurement"]["rows"]
+    copies = ((rows[0] if rows else {}).get("byte_provenance") or {}).get("copies") or {}
+    return {"per-assignment": "assign", "rank-deduplicated": "dedup"}.get(copies.get("wire"), "-")
+
+
 def _headline(document: dict) -> tuple:
     """Headline row, with the skew bracket beside it.
 
@@ -84,16 +113,17 @@ def render(documents: list[dict]) -> str:
         )
         lines.append("")
     lines += [
-        "| ver | sku | backend | mode | precision | suite | phase | routing | ep | outcome "
-        "| T* | p50 us | p99 us | min50 us | skew us |",
-        "|--:|---|---|---|---|---|---|---|--:|---|--:|--:|--:|--:|--:|",
+        "| ver | sku | backend | mode | precision | suite | phase | routing | ep | topo "
+        "| wire | outcome | T* | p50 us | p99 us | min50 us | skew us |",
+        "|--:|---|---|---|---|---|---|---|--:|---|---|---|--:|--:|--:|--:|--:|",
     ]
     for document in documents:
         sku, backend, suite, routing, mode, phase, ep, precision = _identity(document)
         token, p50, p99, min50, skew = _headline(document)
+        topo, wire = _topology(document), _wire_basis(document)
         lines.append(
             f"| {document['version']} | {sku} | `{backend}` | {mode} | {precision} | {suite} | "
-            f"{phase} | {routing} | {ep} | "
+            f"{phase} | {routing} | {ep} | {topo} | {wire} | "
             f"{document['outcome']['status']} | {token} | {p50} | {p99} | {min50} | {skew} |"
         )
     if not documents:

From 029c357a9bfc9c239a3122f8b03a673b7393e957 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 16:51:38 +0800
Subject: [PATCH 08/34] CollectiveX: charge the fp8 quantize production pays,
 as one fused kernel
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：以单一融合 kernel 计入生产环境实际支付的 fp8 量化开销

Production quantises bf16->fp8 once per forward pass, with a single fused kernel, immediately
before the dispatch collective. This benchmark did it once per SHAPE in make_problem, outside
every timed window, so it omitted a real cost.

Moving the eager helper inside the window would have been worse than omitting it: eager is a
9-launch composite, measured on-metal at 19.2us on H100 and 53.6us on MI300X against 1.51us and
4.85us for the compiled single kernel, so charging it would publish this harness's kernel count
rather than production's cost. So it is compiled, and both the wire and the oracle go through
ONE callable — identity then holds by construction rather than by coincidence.

Low-latency deliberately keeps the eager helper: deepep-v2 and uccl-ep quantise inside their
dispatch kernel, so the cost is already charged, and the oracle payload gate compares against
that kernel's bits. A global swap would have redded every low-latency fp8 cell without touching
LL timing. MoRI needs no compile — its cast is already one elementwise kernel — but its LL is
caller-prequantized, so it is cast in dispatch for both modes.

Guarded rather than trusted: assert_quantize_identity checks bitwise equality AND per-row
invariance across batch sizes, because the payload gate compares the sender's [T, hidden]
quantize against the oracle's [receive_count, hidden] one. Verified on-metal for e4m3fn (H100)
and e4m3fnuz (MI300X/MI325X, the arch with no prior precedent for a compiled fp32->fp8 convert).

Also move logical_copies out of byte_provenance: every value there is a per-component byte
breakdown, and a reader indexing it by component name must not meet a differently-shaped entry.
---
 experimental/CollectiveX/bench/ep_backend.py  | 70 +++++++++++++++++
 .../CollectiveX/bench/ep_deepep_v2.py         | 20 ++++-
 experimental/CollectiveX/bench/ep_harness.py  | 18 +++--
 experimental/CollectiveX/bench/ep_mori.py     | 15 +++-
 experimental/CollectiveX/bench/ep_uccl.py     | 28 +++++--
 experimental/CollectiveX/summarize.py         |  2 +-
 .../CollectiveX/tests/test_runtime.py         | 76 +++++++++++++++++++
 7 files changed, 206 insertions(+), 23 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_backend.py b/experimental/CollectiveX/bench/ep_backend.py
index 2d14db945..7a71e683f 100644
--- a/experimental/CollectiveX/bench/ep_backend.py
+++ b/experimental/CollectiveX/bench/ep_backend.py
@@ -161,6 +161,76 @@ def stage_excluded_from_roundtrip(self) -> bool:
             return False
         return not (self.precision == "fp8" and self.fp8_consume == "dequant")
 
+    def fused_quantize(self, eager):
+        """The fp8 quantize the TIMED dispatch should call, keyed on mode.
+
+        Production quantises bf16->fp8 once per forward pass, with a single fused kernel, just
+        before the dispatch collective. This benchmark used to do it once per SHAPE in
+        `make_problem` -- outside every timed window -- so it omitted a real cost. Moving the
+        eager helper inside the window would have been worse than omitting it: the eager form is
+        a 9-launch composite (measured 19.2us on H100, 53.6us on MI300X) against ~1.5-4.9us for
+        the compiled single kernel, so charging it would publish this harness's kernel count
+        rather than production's cost, and flip fp8-vs-bf16 verdicts on that basis.
+
+        LOW-LATENCY GETS THE EAGER FORM, unchanged and deliberately. Its dispatch kernel
+        quantises internally (`use_fp8`), so the cost is already inside the timed window, and
+        the oracle's payload gate compares the received bytes against `semantic_payload` -- which
+        must therefore keep matching the kernel's arithmetic, i.e. the eager helper's bits.
+        Swapping this globally would red every low-latency fp8 cell without touching LL timing.
+
+        Compiled with `dynamic=False`: a dynamic-shape build of this same math measured 6.3x
+        slower once, and here that lands inside the timed window as silently inflated numbers.
+        The cache limit is raised because the quantize legitimately sees ~20+ shapes (one per
+        ladder rung, plus the oracle's receive-count shapes) against a default of 8, and
+        exceeding it makes dynamo fall back to EAGER SILENTLY -- the same failure class.
+        """
+        if self.mode == "low-latency":
+            return eager
+        import torch
+
+        torch._dynamo.config.cache_size_limit = 64
+        if hasattr(torch._dynamo.config, "fail_on_recompile_limit_hit"):
+            # Prefer a loud failure over a silent eager fallback if the limit is ever hit.
+            torch._dynamo.config.fail_on_recompile_limit_hit = True
+        return torch.compile(eager, dynamic=False)
+
+    def assert_quantize_identity(self, eager, fused, x) -> None:
+        """Fail loudly, untimed, if the compiled quantize is not the eager one bit-for-bit.
+
+        The correctness oracle's payload gate is a `torch.equal`, and it compares the SENDER's
+        [T, hidden] quantize against the ORACLE's [receive_count, hidden] one -- so identity has
+        to hold per row, across batch sizes, not merely deterministically. Both properties were
+        verified on-metal for e4m3fn (H100) and e4m3fnuz (MI300X/MI325X) before this was enabled;
+        this check is what turns a future toolchain regression into a named failure here instead
+        of an unexplained payload mismatch across the whole fleet.
+        """
+        if fused is eager:
+            return
+        import torch
+
+        def bits(pair):
+            values, scales = pair
+            return values.view(torch.uint8), scales
+
+        eager_values, eager_scales = bits(eager(x))
+        fused_values, fused_scales = bits(fused(x))
+        if not (torch.equal(eager_values, fused_values)
+                and torch.equal(eager_scales, fused_scales)):
+            raise RuntimeError(
+                "compiled fp8 quantize is not bitwise identical to the eager helper; the "
+                "oracle payload gate would fail fleet-wide"
+            )
+        rows = min(int(x.shape[0]), 3)
+        if rows:
+            part_values, part_scales = bits(fused(x[:rows]))
+            whole_values, whole_scales = fused_values[:rows], fused_scales[:rows]
+            if not (torch.equal(part_values, whole_values)
+                    and torch.equal(part_scales, whole_scales)):
+                raise RuntimeError(
+                    "compiled fp8 quantize is not per-row invariant across batch sizes; the "
+                    "oracle compares a different row count than the sender quantised"
+                )
+
     def __init_subclass__(cls, **kwargs):
         super().__init_subclass__(**kwargs)
         if not getattr(cls, "name", ""):
diff --git a/experimental/CollectiveX/bench/ep_deepep_v2.py b/experimental/CollectiveX/bench/ep_deepep_v2.py
index 672e0c658..a671b2909 100644
--- a/experimental/CollectiveX/bench/ep_deepep_v2.py
+++ b/experimental/CollectiveX/bench/ep_deepep_v2.py
@@ -151,6 +151,10 @@ def __init__(self, args, rank, world_size, local_rank, device):
             # deep_ep.utils.math) so the timed stage() does no module lookup in the
             # measured region.
             self._to_fp8, self._cast_back = _fp8_cast_helpers()
+            # Normal/HT quantises inside the timed dispatch with the compiled single-kernel
+            # form; low-latency keeps the eager helper because its kernel quantises internally
+            # and the oracle gate is pinned to those bits. See EPBackend.fused_quantize.
+            self._quant = self.fused_quantize(self._to_fp8)
         if self.mode == "low-latency":
             # Legacy Buffer IBGDA decode path: a distinct kernel family whose combine
             # multiplies by the gate at the source (weighted), not an unweighted rank sum.
@@ -286,7 +290,9 @@ def _topk_idx_dtype(self):
     def semantic_payload(self, x):
         if not self._fp8:
             return x
-        return self._cast_back(*self._to_fp8(x))
+        # Same callable the wire uses, so the oracle cannot disagree with the sender by
+        # construction (low-latency: both are the eager helper, matching its kernel).
+        return self._cast_back(*self._quant(x))
 
     def _encode_dispatch(self, x):
         if not self._fp8:
@@ -296,8 +302,11 @@ def _encode_dispatch(self, x):
             # send x unquantized; expose the host round-trip as the oracle semantic so the
             # combine expectation models the FP8 transport (same as semantic_payload).
             return x, self._cast_back(*self._to_fp8(x))
-        quantized = self._to_fp8(x)
-        return quantized, self._cast_back(*quantized)
+        # Normal/HT: send BF16 and quantise inside dispatch, where production pays it. The
+        # payload is therefore x itself; oracle_x is still the round trip, computed once here,
+        # untimed -- which also compiles this rung's shape before any timed region.
+        self.assert_quantize_identity(self._to_fp8, self._quant, x)
+        return x, self._cast_back(*self._quant(x))
 
     def _ll_dispatch(self, p):
         # Verified pinned signature (legacy.py:553):
@@ -322,8 +331,11 @@ def _ll_dispatch(self, p):
     def dispatch(self, p):
         if self.mode == "low-latency":
             return self._ll_dispatch(p)
+        # Quantise here, not in make_problem: production runs one fused bf16->fp8 kernel per
+        # forward pass immediately before this collective, so the timed window must contain it.
+        dispatch_x = self._quant(p.dispatch_x) if self._fp8 else p.dispatch_x
         recv_x, recv_topk_idx, recv_topk_weights, handle, _ = self.buffer.dispatch(
-            p.dispatch_x,
+            dispatch_x,
             topk_idx=p.topk_idx,
             topk_weights=p.topk_weights,
             num_experts=self.args.experts,
diff --git a/experimental/CollectiveX/bench/ep_harness.py b/experimental/CollectiveX/bench/ep_harness.py
index 3b5ebc453..502bc916f 100644
--- a/experimental/CollectiveX/bench/ep_harness.py
+++ b/experimental/CollectiveX/bench/ep_harness.py
@@ -1118,14 +1118,16 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
                 "dispatch": dispatch_bytes,
                 "roundtrip": roundtrip_bytes,
                 "stage": stage_bytes,
-                # Copy counts behind the byte figures above, so a reader can rebase them.
-                # `routed` is what they use; `assignments` is the per-(token, expert) count,
-                # and `wire` names which of the two this backend's kernels actually move.
-                "copies": {
-                    "routed": int(rstats["routed_copies"]),
-                    "assignments": assignment_copies,
-                    "wire": wire_basis,
-                },
+            },
+            # Copy counts behind the byte figures above, so a reader can rebase them. `routed`
+            # is the basis they use; `assignments` is the per-(token, expert) count; `wire` names
+            # which of the two this backend's kernels actually move. Kept OUT of
+            # `byte_provenance`, whose every value is a per-component byte breakdown -- a reader
+            # indexing it by component name must not meet a differently-shaped entry.
+            "logical_copies": {
+                "routed": int(rstats["routed_copies"]),
+                "assignments": assignment_copies,
+                "wire": wire_basis,
             },
             "receive": {
                 "max": recv_max,
diff --git a/experimental/CollectiveX/bench/ep_mori.py b/experimental/CollectiveX/bench/ep_mori.py
index 25bfd37dd..2bb621e03 100644
--- a/experimental/CollectiveX/bench/ep_mori.py
+++ b/experimental/CollectiveX/bench/ep_mori.py
@@ -244,8 +244,12 @@ def semantic_payload(self, x):
     def _encode_dispatch(self, x):
         if not self._fp8:
             return x, None
-        quantized = x.to(self._fp8_dtype)
-        return quantized, quantized.to(torch.bfloat16)
+        # Send BF16 and cast inside dispatch, where production pays it: vLLM and SGLang both run
+        # an aiter per-1x128 quant immediately before mori's dispatch, once per forward pass.
+        # MoRI's cast needs no compile -- it is already a single eager elementwise kernel, and
+        # elementwise is per-row invariant, so it carries none of the identity risk the blockwise
+        # backends do. oracle_x is the round trip, computed once here, untimed.
+        return x, x.to(self._fp8_dtype).to(torch.bfloat16)
 
     def make_problem(self, T, idx, weights, x):
         indices = idx.to(torch.int32)
@@ -266,9 +270,14 @@ def make_problem(self, T, idx, weights, x):
         return problem
 
     def dispatch(self, p):
+        # See _encode_dispatch: the fp8 cast belongs in the timed window because production runs
+        # it per forward pass just before this collective. Low-latency is cast here too -- MoRI's
+        # IntraNodeLL takes a caller-prequantized tensor, unlike deepep-v2/uccl-ep whose LL
+        # kernels quantise internally, so there is no in-kernel cost already being charged.
+        dispatch_x = p.dispatch_x.to(self._fp8_dtype) if self._fp8 else p.dispatch_x
         dispatch_output, dispatch_weights, _scales, dispatch_indices, recv_num = (
             self.op.dispatch(
-                p.dispatch_x,
+                dispatch_x,
                 p.weights,
                 p.scales,
                 p.indices,
diff --git a/experimental/CollectiveX/bench/ep_uccl.py b/experimental/CollectiveX/bench/ep_uccl.py
index 1a61821d0..632c87fd4 100644
--- a/experimental/CollectiveX/bench/ep_uccl.py
+++ b/experimental/CollectiveX/bench/ep_uccl.py
@@ -155,6 +155,10 @@ def __init__(self, args, rank, world_size, local_rank, device):
             )
             self.dispatch_value_bytes = 1
             self.dispatch_scale_bytes_per_copy = ((args.hidden + 127) // 128) * 4
+            # Normal/HT quantises inside the timed dispatch with the compiled single-kernel
+            # form; low-latency keeps the eager helper, whose bits its in-kernel cast matches.
+            # See EPBackend.fused_quantize.
+            self._quant = self.fused_quantize(per_token_cast_to_fp8)
         if self.mode == "low-latency":
             # Legacy low-latency decode path: a distinct kernel family whose combine multiplies
             # by the gate at the source (weighted), not an unweighted rank sum. LL result tensors
@@ -273,7 +277,8 @@ def _topk_idx_dtype(self):
     def semantic_payload(self, x):
         if not self._fp8:
             return x
-        return per_token_cast_back(*per_token_cast_to_fp8(x))
+        # Same callable the wire uses, so sender and oracle cannot disagree by construction.
+        return per_token_cast_back(*self._quant(x))
 
     def _encode_dispatch(self, x):
         if not self._fp8:
@@ -282,11 +287,11 @@ def _encode_dispatch(self, x):
             # low_latency_dispatch takes BF16 x and casts to e4m3 inside the kernel, so send x
             # unquantized; expose the host round-trip as the oracle semantic.
             return x, per_token_cast_back(*per_token_cast_to_fp8(x))
-        fp8, scales = per_token_cast_to_fp8(x)
-        # Column-major (TMA-compatible) scale layout the dispatch kernel expects, matching UCCL's
-        # own bench (`scales.T.contiguous().T`) and the LL scale-contiguity note below.
-        quantized = (fp8, scales.T.contiguous().T)
-        return quantized, per_token_cast_back(fp8, scales)
+        # Normal/HT: send BF16 and quantise inside dispatch, where production pays it. oracle_x
+        # is still the round trip, computed once here, untimed -- which also compiles this rung's
+        # shape before any timed region.
+        self.assert_quantize_identity(per_token_cast_to_fp8, self._quant, x)
+        return x, per_token_cast_back(*self._quant(x))
 
     def _ll_recv_bf16(self, recv_x):
         """The padded per-expert receive as BF16 [num_local_experts, cap*num_ranks, hidden].
@@ -325,8 +330,17 @@ def dispatch(self, p):
         # it through so the same call serves both scopes.
         (num_tokens_per_rank, num_tokens_per_rdma_rank, num_tokens_per_expert,
          is_token_in_rank, _) = self.buffer.get_dispatch_layout(p.topk_idx, self.args.experts)
+        # Quantise here, not in make_problem: production runs one fused bf16->fp8 kernel per
+        # forward pass immediately before this collective. The scales then need UCCL's
+        # column-major (TMA-compatible) layout, matching its own bench's `scales.T.contiguous().T`;
+        # production's kernel emits that layout directly, so charging the transpose here
+        # over-states by one small copy.
+        dispatch_x = p.dispatch_x
+        if self._fp8:
+            fp8, scales = self._quant(dispatch_x)
+            dispatch_x = (fp8, scales.T.contiguous().T)
         recv_x, recv_topk_idx, recv_topk_weights, _counts, handle, _event = self.buffer.dispatch(
-            x=p.dispatch_x,
+            x=dispatch_x,
             num_tokens_per_rank=num_tokens_per_rank,
             num_tokens_per_rdma_rank=num_tokens_per_rdma_rank,
             is_token_in_rank=is_token_in_rank,
diff --git a/experimental/CollectiveX/summarize.py b/experimental/CollectiveX/summarize.py
index 664e437c3..458b56acb 100644
--- a/experimental/CollectiveX/summarize.py
+++ b/experimental/CollectiveX/summarize.py
@@ -68,7 +68,7 @@ def _wire_basis(document: dict) -> str:
     different combine traffic at EP8, so two low-latency rows are not doing equal work.
     """
     rows = document["measurement"]["rows"]
-    copies = ((rows[0] if rows else {}).get("byte_provenance") or {}).get("copies") or {}
+    copies = (rows[0] if rows else {}).get("logical_copies") or {}
     return {"per-assignment": "assign", "rank-deduplicated": "dedup"}.get(copies.get("wire"), "-")
 
 
diff --git a/experimental/CollectiveX/tests/test_runtime.py b/experimental/CollectiveX/tests/test_runtime.py
index 01f377b7a..750fa0027 100644
--- a/experimental/CollectiveX/tests/test_runtime.py
+++ b/experimental/CollectiveX/tests/test_runtime.py
@@ -27,6 +27,7 @@
 import config  # noqa: E402
 import stage  # noqa: E402
 import ep_harness  # noqa: E402  (stdlib-only at module top)
+import ep_backend  # noqa: E402  (torch is imported lazily inside its methods)
 
 
 # configs/platform_config.json is shared by matrix scheduling, operator/network
@@ -654,5 +655,80 @@ def test_a_rank_claimed_by_an_earlier_slot_contributes_once(self):
         )
         self.assertEqual(combined.item(), 0.5)
 
+
+
+@unittest.skipUnless(_torch is not None, "quantize-identity checks require torch")
+class FusedQuantizeGate(unittest.TestCase):
+    """The fp8 quantize moved inside the timed dispatch, so its identity is load-bearing.
+
+    The oracle's payload gate is a `torch.equal` that compares the SENDER's [T, hidden] quantize
+    against the ORACLE's [receive_count, hidden] one, so the callable must be bit-identical to
+    the eager helper AND per-row invariant across batch sizes. Verified on-metal for e4m3fn
+    (H100: 9 kernels/19.2us eager vs 1/1.51us fused) and e4m3fnuz (MI300X, the arch with no
+    prior precedent for a compiled fp32->fp8 convert). These tests pin the guard, not the
+    compiler.
+    """
+
+    @staticmethod
+    def _fuse(mode, eager):
+        # Both methods read only `self.mode`, so call them unbound rather than instantiating an
+        # abstract backend; that also documents that they depend on nothing else.
+        return ep_backend.EPBackend.fused_quantize(types.SimpleNamespace(mode=mode), eager)
+
+    @staticmethod
+    def _check(eager, fused, x):
+        return ep_backend.EPBackend.assert_quantize_identity(
+            types.SimpleNamespace(mode="normal"), eager, fused, x
+        )
+
+    def test_low_latency_keeps_the_eager_helper(self):
+        # Its dispatch kernel quantises internally and the oracle gate is pinned to those bits,
+        # so swapping in a compiled callable would red every LL fp8 cell without touching timing.
+        def eager(x):
+            return x, x
+        self.assertIs(self._fuse("low-latency", eager), eager)
+
+    def test_normal_mode_wraps_the_helper(self):
+        def eager(x):
+            return x, x
+        self.assertIsNot(self._fuse("normal", eager), eager)
+
+    def test_identity_check_accepts_an_equivalent_callable(self):
+        torch = _torch
+        x = torch.randn(8, 256, dtype=torch.bfloat16)
+
+        def eager(t):
+            return t.to(torch.float8_e4m3fn), t.float().abs().amax(dim=1)
+
+        self._check(eager, lambda t: eager(t), x)  # must not raise
+
+    def test_identity_check_rejects_a_divergent_callable(self):
+        torch = _torch
+        x = torch.randn(8, 256, dtype=torch.bfloat16)
+
+        def eager(t):
+            return t.to(torch.float8_e4m3fn), t.float().abs().amax(dim=1)
+
+        def divergent(t):
+            values, scales = eager(t)
+            return values, scales + 1          # one differing scale is enough to red a cell
+        with self.assertRaises(RuntimeError):
+            self._check(eager, divergent, x)
+
+    def test_identity_check_rejects_a_shape_dependent_callable(self):
+        # A callable that is deterministic but NOT per-row invariant still breaks the gate,
+        # because the oracle quantises a different row count than the sender did.
+        torch = _torch
+        x = torch.randn(8, 256, dtype=torch.bfloat16)
+
+        def eager(t):
+            return t.to(torch.float8_e4m3fn), t.float().abs().amax(dim=1)
+
+        def shape_dependent(t):
+            values, scales = eager(t)
+            return values, scales * float(t.shape[0])
+        with self.assertRaises(RuntimeError):
+            self._check(eager, shape_dependent, x)
+
 if __name__ == "__main__":
     unittest.main()

From f78330adbe96d484af41ad2d1e0bc6964fcf9f62 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 16:58:33 +0800
Subject: [PATCH 09/34] CollectiveX: add FP8 dispatch to flashinfer-ep
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：为 flashinfer-ep 增加 FP8 dispatch

Dispatch-side only: the scales ride as a fourth payload (the kernel's kMaxPayloads is exactly
4, and vLLM's own integration uses the same [values, scales, ids, weights] order), and combine
stays BF16, so none of the 0.6.16+ combine-quant API is needed — the shipped 0.6.8.post1 wheel
suffices with no image bump. The transport imposes no dtype whitelist on payloads; only the
workspace, expert-id and metainfo tensors are type-checked.

Same per-128-block e4m3 recipe as deepep-v2 and uccl-ep, which keeps the fp8 axis comparable:
its scale overhead is identical to theirs, since 4 bytes per 128 elements is 1 byte per 32.
Recorded honestly as `fp8-e4m3fn-blockwise-offpath` because vLLM accepts only nvfp4/mxfp8/bf16
on this transport today, so the row measures what the transport costs with the DeepSeek-V3
recipe rather than a configuration a deployment can currently select — the caveat travels in
the artifact rather than only in prose.

The cast pair is local rather than shared on purpose: deepep-v2 must use deep_ep's own helper
because its low-latency kernel quantises in-kernel and the oracle compares those bits, and
uccl-ep vendors one faithful to UCCL's. Nothing external pins FlashInfer's arithmetic, so these
are three contracts rather than three copies of one.
---
 .../CollectiveX/bench/ep_flashinfer.py        | 117 ++++++++++++++++--
 experimental/CollectiveX/sweep_matrix.py      |   4 +-
 experimental/CollectiveX/tests/test_matrix.py |   6 +-
 3 files changed, 113 insertions(+), 14 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_flashinfer.py b/experimental/CollectiveX/bench/ep_flashinfer.py
index 8977fa109..033d45590 100644
--- a/experimental/CollectiveX/bench/ep_flashinfer.py
+++ b/experimental/CollectiveX/bench/ep_flashinfer.py
@@ -57,16 +57,46 @@
 # FP32 expectation does not carry, so the oracle needs to know which kernel it is facing.
 _COMBINE_FP32_SINCE = (0, 6, 16)
 
+# FP8 block size, matching the DeepSeek-V3 recipe every other FP8 backend here uses.
+_FP8_BLOCK = 128
+
+
+def _blockwise_cast_to_fp8(x):
+    """Per-128-channel e4m3 quantize: (values [m, n], FP32 scales [m, n//128]).
+
+    A local pair rather than a shared one on purpose. deepep-v2 must use deep_ep's own helper
+    because its low-latency kernel quantises in-kernel and the oracle compares against those
+    bits; uccl-ep vendors a copy faithful to UCCL's. FlashInfer's combine is always BF16 and no
+    kernel here quantises, so nothing external pins the arithmetic -- the oracle only needs a
+    self-consistent round trip. Each copy is pinned to a different library's numerics, so this
+    is three contracts, not three copies of one.
+    """
+    m, n = x.shape
+    blocks = x.view(m, -1, _FP8_BLOCK)
+    amax = blocks.abs().float().amax(dim=2).view(m, -1).clamp(1e-4)
+    values = (blocks * (448.0 / amax.unsqueeze(2))).to(torch.float8_e4m3fn).view(m, n)
+    return values, (amax / 448.0).view(m, -1)
+
+
+def _blockwise_cast_back(values, scales):
+    """Inverse of _blockwise_cast_to_fp8, to BF16."""
+    m, n = values.shape
+    blocks = values.view(m, -1, _FP8_BLOCK).float()
+    return (blocks * scales.view(m, -1, 1)).view(m, n).to(torch.bfloat16)
+
 
 class FlashInferEPBackend(EPBackend):
     name = "flashinfer-ep"
     maturity = "production"  # vLLM --all2all-backend flashinfer_nvlink_one_sided
     # One kernel family; see the module docstring for why there is no low-latency mode.
     SUPPORTED_MODES = ("normal",)
-    # BF16 first. The combine side accepts fp8_e4m3fn/uint8 output dtypes and a
-    # use_low_precision accumulate, but dispatch FP8 needs the scale payload plumbed as a
-    # second input_payload and validated against the oracle's cast round-trip; not this pass.
-    SUPPORTED_PRECISIONS = ("bf16",)
+    # FP8 is DISPATCH-side only: the scales ride as a fourth payload and combine stays BF16,
+    # so none of the 0.6.16+ combine-quant API is needed. The recipe is the same per-128-block
+    # e4m3 every other FP8 backend here uses, which keeps the fp8 axis comparable -- but note
+    # vLLM's integration accepts only nvfp4/mxfp8/bf16 on this transport, so an fp8 row here
+    # measures what the transport costs with the DeepSeek-V3 recipe rather than a configuration
+    # a deployment can currently select. `dispatch_dtype` records that per row.
+    SUPPORTED_PRECISIONS = ("bf16", "fp8")
     kernel_generation = "flashinfer-mnnvl-one-sided"
     # stage() now copies the received payload into the workspace combine region.
     stage_device_work = True
@@ -83,6 +113,16 @@ class FlashInferEPBackend(EPBackend):
 
     def __init__(self, args, rank, world_size, local_rank, device):
         super().__init__(args, rank, world_size, local_rank, device)
+        self._fp8 = self.precision == "fp8"
+        if self._fp8:
+            # "-offpath" because vLLM cannot select this recipe on this transport today; the
+            # bytes and the block size match deepep-v2/uccl-ep so the axis stays comparable.
+            self.dispatch_dtype = "fp8-e4m3fn-blockwise-offpath"
+            self.dispatch_value_bytes = 1
+            self.dispatch_scale_bytes_per_copy = (
+                (args.hidden + _FP8_BLOCK - 1) // _FP8_BLOCK
+            ) * 4
+            self._quant = self.fused_quantize(_blockwise_cast_to_fp8)
         self._a2a = None
         self._max_tokens = None
         self.experts_per_rank = args.experts // world_size
@@ -98,6 +138,20 @@ def _topk_idx_dtype(self):
         """
         return torch.int32
 
+    def semantic_payload(self, x):
+        if not self._fp8:
+            return x
+        # Same callable the wire uses, so sender and oracle cannot disagree by construction.
+        return _blockwise_cast_back(*self._quant(x))
+
+    def _encode_dispatch(self, x):
+        if not self._fp8:
+            return x, None
+        # Send BF16 and quantise inside dispatch, where production pays it. oracle_x is the
+        # round trip, computed once here, untimed -- which also compiles this rung's shape.
+        self.assert_quantize_identity(_blockwise_cast_to_fp8, self._quant, x)
+        return x, _blockwise_cast_back(*self._quant(x))
+
     def buffer_cap(self, args):
         # The workspace is sized from the ladder maximum rather than a fixed slot budget, so
         # there is no cap to clamp the ladder against.
@@ -122,7 +176,11 @@ def create_buffer(self, spec):
         top_k = self.args.topk
         # Dispatch carries the activation plus the routing metadata the kernel needs per token:
         # int32 expert ids and fp32 gate weights, top_k of each. Combine carries BF16 hidden.
-        dispatch_bytes = hidden * 2 + top_k * 4 + top_k * 4
+        dispatch_bytes = (
+            hidden * self.dispatch_value_bytes
+            + self.dispatch_scale_bytes_per_copy
+            + top_k * 4 + top_k * 4
+        )
         combine_bytes = hidden * 2
         workspace_size = moe_a2a_get_workspace_size_per_rank(
             ep_size=self.world_size,
@@ -168,22 +226,43 @@ def dispatch(self, p):
         the tokens that selected one of its experts, so the kernel stamps the sentinel into the
         expert-id payload of every slot it did not fill.
         """
-        recv_x, recv_idx, recv_w = self._a2a.dispatch(
+        # Quantise here, not in make_problem: production runs one fused bf16->fp8 kernel per
+        # forward pass immediately before this collective. The scales then ride as their own
+        # payload, which shifts the expert ids to index 2 -- four payloads, and the kernel's
+        # kMaxPayloads is exactly 4, matching vLLM's own [values, scales, ids, weights] order.
+        if self._fp8:
+            values, scales = self._quant(p.dispatch_x)
+            payloads = [values, scales, p.topk_idx, p.topk_weights]
+            expert_id_index = 2
+        else:
+            payloads = [p.dispatch_x, p.topk_idx, p.topk_weights]
+            expert_id_index = 1
+        received = self._a2a.dispatch(
             p.topk_idx,
-            [p.dispatch_x, p.topk_idx, p.topk_weights],
+            payloads,
             p.T,
             invalid_token_expert_id=_INVALID_EXPERT,
-            expert_id_payload_index=1,
+            expert_id_payload_index=expert_id_index,
         )
+        # One received tensor per payload. Under FP8 `recv_x` stays the VALUES plane and the
+        # scales ride beside it, so every shape-sensitive consumer keeps working on a tensor.
+        if self._fp8:
+            recv_x, recv_scales, recv_idx, recv_w = received
+        else:
+            (recv_x, recv_idx, recv_w), recv_scales = received, None
         return types.SimpleNamespace(
-            recv_x=recv_x, recv_idx=recv_idx, recv_w=recv_w,
+            recv_x=recv_x, recv_scales=recv_scales, recv_idx=recv_idx, recv_w=recv_w,
             tokens=p.T, topk=p.topk_idx.shape[1], combine_input=None,
         )
 
     def _combine_buffer(self, h):
-        """The workspace-resident combine payload region for this rung."""
+        """The workspace-resident combine payload region for this rung.
+
+        Always BF16: combine carries BF16 whatever the dispatch precision was, so this cannot
+        key off `recv_x.dtype` -- under FP8 that would size the region for 1-byte values.
+        """
         return self._a2a.get_combine_payload_tensor_in_workspace(
-            h.tokens, h.recv_x.shape[-1], h.recv_x.dtype
+            h.tokens, h.recv_x.shape[-1], torch.bfloat16
         )
 
     def _filled_slot_index(self, p, h):
@@ -219,8 +298,16 @@ def stage(self, p, h):
         """
         buffer = self._combine_buffer(h)
         filled = self._filled_slot_index(p, h)
+        hidden = h.recv_x.shape[-1]
         flat_buffer = buffer.view(-1, buffer.shape[-1])
-        flat_buffer[filled] = h.recv_x.view(-1, h.recv_x.shape[-1])[filled]
+        source = h.recv_x.view(-1, hidden)[filled]
+        if self._fp8:
+            # Combine sends BF16, so the dequant lands here -- device work, which is why
+            # `stage` is a reported component for this backend under either precision.
+            source = _blockwise_cast_back(
+                source, h.recv_scales.view(-1, h.recv_scales.shape[-1])[filled]
+            )
+        flat_buffer[filled] = source
         h.combine_input = buffer
 
     def combine(self, p, h):
@@ -253,6 +340,12 @@ def inspect_dispatch(self, p, h):
         keep = self._valid_rows(h)
         hidden = h.recv_x.shape[-1]
         payload = h.recv_x.reshape(-1, hidden)[keep]
+        if self._fp8:
+            # The oracle compares a BF16 payload against semantic_payload's round trip, so the
+            # received FP8 slice is dequantised with the same pair that produced it.
+            payload = _blockwise_cast_back(
+                payload, h.recv_scales.reshape(-1, h.recv_scales.shape[-1])[keep]
+            )
         ids = h.recv_idx.reshape(-1, h.topk).to(torch.int64)[keep]
         weights = h.recv_w.reshape(-1, h.topk).to(torch.float32)[keep]
         local = (ids >= 0) & ((ids // self.experts_per_rank) == self.rank)
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 11bab352c..2b6dbc370 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -44,7 +44,9 @@ def _load_config(name: str) -> dict[str, Any]:
     # FlashInfer one-sided is BF16-only this pass: the combine side accepts FP8 output
     # dtypes, but an FP8 dispatch needs the scale payload plumbed as a second
     # input_payload and validated against the oracle cast round-trip.
-    "flashinfer-ep": ("bf16",),
+    # FP8 is dispatch-side only here (scales as a fourth payload, combine stays BF16), and
+    # uses the same per-128-block e4m3 recipe as deepep-v2/uccl-ep so the axis is comparable.
+    "flashinfer-ep": ("bf16", "fp8"),
 }
 # Short shard-ID slug per non-normal mode. Normal-mode shard IDs carry no mode
 # segment so existing references stay valid; a low-latency shard adds "-ll".
diff --git a/experimental/CollectiveX/tests/test_matrix.py b/experimental/CollectiveX/tests/test_matrix.py
index f89035b79..583df4856 100644
--- a/experimental/CollectiveX/tests/test_matrix.py
+++ b/experimental/CollectiveX/tests/test_matrix.py
@@ -242,7 +242,11 @@ def test_flashinfer_ep_rollout_shape(self):
             for item in cases if item["disposition"] == "runnable"
         }
         self.assertEqual(runnable, {(sku, ep) for sku in ("gb200", "gb300") for ep in (8, 16)})
-        self.assertEqual({item["case"]["precision"] for item in cases}, {"bf16"})
+        # FP8 is dispatch-side only: scales ride as a fourth payload (the kernel's kMaxPayloads
+        # is exactly 4) and combine stays BF16, so none of the 0.6.16+ combine-quant API is
+        # needed. Same per-128-block e4m3 recipe as deepep-v2/uccl-ep, so the axis stays
+        # comparable across backends.
+        self.assertEqual({item["case"]["precision"] for item in cases}, {"bf16", "fp8"})
         # Normal mode only — no low-latency cell on any SKU.
         self.assertEqual({item["case"]["mode"] for item in cases}, {"normal"})
         for platform in sweep_matrix.PLATFORMS.values():

From 86683387683be7f582fa74511202c4b059bbe12a Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 17:25:57 +0800
Subject: [PATCH 10/34] =?UTF-8?q?CollectiveX:=20fix=20contradictory=20meth?=
 =?UTF-8?q?odology=20claims=20and=20three=20review=20findings=20/=20Collec?=
 =?UTF-8?q?tiveX=EF=BC=9A=E4=BF=AE=E6=AD=A3=E6=96=B9=E6=B3=95=E8=AE=BA?=
 =?UTF-8?q?=E8=87=AA=E7=9B=B8=E7=9F=9B=E7=9B=BE=E4=B9=8B=E5=A4=84=E5=8F=8A?=
 =?UTF-8?q?=E4=B8=89=E9=A1=B9=E8=AF=84=E5=AE=A1=E5=8F=91=E7=8E=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adversarial review of this branch found two places where docs/methodology.md contradicted itself
or the code. That file defines what a published number MEANS, so a wrong sentence there is worse
than a wrong comment.

- The headline percentile was stated twice, incompatibly: "the p50 of the per-iteration cross-rank
  MAX" in one paragraph and "measured roundtrip p99 is the headline latency" in another. On
  skew-heavy backends those differ by 2-5x. p99 is what the published cohorts rank on, so the p50
  sentence was the wrong one; both percentiles are emitted and summarize.py prints both.
- The low-latency copy-basis claim was blanket where the behaviour is not. MoRI's IntraNodeLL
  deduplicates per rank (its combine is an unweighted rank-sum), so the rank-deduplicated count is
  exact there, not a lower bound. The commit that surfaced this in the summary table left the prose
  asserting the incomparable thing was uniform.
- FP8 `normal` dispatch now contains the quantize, which the doc never said. It also listed the FP8
  backends without FlashInfer EP. Both corrected, with the comparability consequence stated: an FP8
  `normal` dispatch covers quantize-plus-transport while its BF16 control covers transport alone.
- `stage_excluded_from_roundtrip` is false in two unrelated situations, and reading it alone is
  unsafe. Documented the disambiguation: `stage` absent means nothing to stage; `stage` present
  with the flag false means the dequant hatch put it back in the chain. Three separate on-metal
  reports asked about this, which is enough evidence that the contract needed writing down.

Code, all from the same review:

- The FlashInfer wheel gate scraped digits, so `0.6.16rc1` and `0.6.16.dev0` read as 0.6.16 and
  selected the FP32 combine model against a kernel that still rounds per level. That direction can
  exceed COMBINE_REL_TOL and RED a correct run; the reverse is a few ulps. Now ordered with
  `packaging`, which sorts pre-releases below their release, with a conservative regex fallback and
  False on anything unparseable.
- The duplicate-slot test could not tell keep-first from keep-last, because it used identical
  messages in both slots. Replaced the blind spot with cancelling contributions (256, 1, -256):
  keep-first is 0.0, keep-last is 1.0. Verified the function returns 0.0.
- summarize.py raised IndexError on a document with zero measurement rows. It validates nothing by
  design, so it now degrades to "-" rather than losing the whole table.

91 tests pass with torch present (0 skips).

中文：对本分支的对抗性评审发现 docs/methodology.md 有两处与自身或代码矛盾。该文件定义了已发布
指标的**含义**，因此其中的错误句子比错误注释更严重。

- 头条百分位被以互不相容的方式表述了两次：一段称"逐次迭代跨 rank MAX 的 p50"，另一段称"实测
  roundtrip p99 即头条延迟"。在 skew 较大的后端上两者相差 2-5 倍。已发布队列排名依据的是 p99，
  因此 p50 那句是错的；两个百分位都会输出，summarize.py 也都会打印。
- 低延迟拷贝基准的表述过于笼统。MoRI 的 IntraNodeLL 按 rank 去重（其 combine 为非加权 rank 求
  和），因此在该处按 rank 去重的计数是精确值而非下界。此前在汇总表中揭示该差异的提交，遗漏了正文
  中"该不可比属性是统一的"这一断言。
- FP8 `normal` 的 dispatch 现已包含量化，而文档从未说明；FP8 后端列表也漏了 FlashInfer EP。两者
  均已修正，并说明其可比性后果：FP8 `normal` 的 dispatch 涵盖量化加传输，而其 BF16 对照仅涵盖传输。
- `stage_excluded_from_roundtrip` 在两种互不相关的情形下均为 false，单看该字段并不安全。已补充
  判别规则：无 `stage` 组件表示无需 staging；有 `stage` 组件而该标志为 false 表示 dequant 旁路把
  转换放回了链路内。三份独立的实机报告都问到了这一点，足以说明该约定需要写清。

代码修改同样来自该评审：

- FlashInfer wheel 版本判定此前直接抓取数字，导致 `0.6.16rc1` 与 `0.6.16.dev0` 被读作 0.6.16，
  从而对仍逐层取整的 kernel 选用了 FP32 combine 模型。该方向可能超出 COMBINE_REL_TOL 并把正确的
  运行判为失败；反方向仅差几个 ulp。现改用 `packaging` 排序（预发布版本排在正式版本之前），并保留
  保守的正则回退，无法解析时一律返回 False。
- 重复 slot 的测试无法区分保留首个与保留末个，因为两个 slot 使用了相同的消息。现改用可相互抵消的
  贡献值（256、1、-256）：保留首个得 0.0，保留末个得 1.0。已验证函数返回 0.0。
- summarize.py 在测量行为空的文档上会抛出 IndexError。该模块本就不做校验，现改为降级显示 "-"，
  而不是丢失整张表。

在存在 torch 的环境下 91 项测试全部通过（0 项跳过）。
---
 .../CollectiveX/bench/ep_flashinfer.py        | 29 +++++++++++++--
 experimental/CollectiveX/docs/methodology.md  | 35 +++++++++++++++----
 experimental/CollectiveX/summarize.py         |  5 +++
 .../CollectiveX/tests/test_runtime.py         | 16 +++++++++
 4 files changed, 76 insertions(+), 9 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_flashinfer.py b/experimental/CollectiveX/bench/ep_flashinfer.py
index 033d45590..3cfe4c6ac 100644
--- a/experimental/CollectiveX/bench/ep_flashinfer.py
+++ b/experimental/CollectiveX/bench/ep_flashinfer.py
@@ -57,6 +57,32 @@
 # FP32 expectation does not carry, so the oracle needs to know which kernel it is facing.
 _COMBINE_FP32_SINCE = (0, 6, 16)
 
+
+def _wheel_has_fp32_combine(version: str) -> bool:
+    """Does this wheel accumulate combine in FP32, per `_COMBINE_FP32_SINCE`?
+
+    Ordering matters more than it looks. A release candidate sorts BELOW its own release, so
+    `0.6.16rc1` predates the rewrite and must read as False; a naive digit scrape reads it as
+    0.6.16 and models FP32 against a kernel that still rounds per level, which can exceed
+    COMBINE_REL_TOL and RED a correct run. The reverse error is harmless (a few ulps, far
+    inside tolerance), so every unparseable input answers False.
+    """
+    try:
+        from packaging.version import InvalidVersion, Version
+    except ImportError:  # pragma: no cover - packaging ships with torch
+        digits = re.match(r"(?:\d+!)?(\d+(?:\.\d+){0,2})", version.strip())
+        if digits is None:
+            return False
+        release = tuple(int(n) for n in digits.group(1).split("."))
+        # No pre-release ordering available, so treat a marked pre-release as below its release.
+        if re.match(r"[._-]?(a|b|c|rc|alpha|beta|dev|pre)", version[digits.end():]):
+            return release > _COMBINE_FP32_SINCE
+        return release >= _COMBINE_FP32_SINCE
+    try:
+        return Version(version) >= Version(".".join(str(n) for n in _COMBINE_FP32_SINCE))
+    except InvalidVersion:
+        return False
+
 # FP8 block size, matching the DeepSeek-V3 recipe every other FP8 backend here uses.
 _FP8_BLOCK = 128
 
@@ -203,8 +229,7 @@ def create_buffer(self, spec):
             workspace_size_per_rank=workspace_size,
             mnnvl_config=MnnvlConfig(comm_backend=_communicator(_ep_group())),
         )
-        wheel = tuple(int(n) for n in re.findall(r"\d+", flashinfer.__version__)[:3])
-        if wheel >= _COMBINE_FP32_SINCE:
+        if _wheel_has_fp32_combine(flashinfer.__version__):
             self.combine_reduction = "domain-fp32"
         # Every rank must finish mapping its workspace before any peer writes into it;
         # vLLM barriers here for the same reason. Scoped to the EP group, not the world.
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 14b02d86f..fc37ff5c0 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -22,9 +22,16 @@ It does not predict serving throughput without a separate correlation study.
 The implemented workload is `deepseek-v3`: hidden 7168, top-k 8, 256 routed experts, packed
 placement, and one pinned fixed resource profile per backend/topology. Combine is always BF16;
 dispatch precision is a swept dimension — a BF16 control and, on the backends whose FP8 dispatch is
-supported upstream (DeepEP V2, MoRI, UCCL-EP), an FP8 dispatch (`bf16`, `fp8`),
+supported upstream (DeepEP V2, MoRI, UCCL-EP, FlashInfer EP), an FP8 dispatch (`bf16`, `fp8`),
 caller-prequantized in `normal` mode (the `low-latency` kernels quantize FP8 internally from BF16 on
-DeepEP and UCCL-EP, and stay caller-prequantized on MoRI). NCCL EP is BF16-only this release, so its
+DeepEP and UCCL-EP, and stay caller-prequantized on MoRI). Because `normal`-mode FP8 is
+caller-prequantized, that quantize is a cost a production forward pass pays on the critical path, so
+it is charged **inside the measured dispatch** rather than prepared ahead of the timing window; it is
+issued as one fused kernel and guarded bitwise against its eager reference. This means an FP8
+`normal` dispatch number covers quantize-plus-transport while its BF16 control covers transport
+alone, and it is why FP8 `normal` rows are not comparable to runs published before sweep version 2.
+`low-latency` rows are unaffected: those kernels either quantize internally (nothing for the caller
+to charge) or take pre-quantized input by API contract. NCCL EP is BF16-only this release, so its
 cells carry the control alone; the per-backend precision set lives in `sweep_matrix.py`'s
 `BACKEND_PRECISIONS` and a backend never emits a case for a precision it does not support.
 `normal`-mode cases use the
@@ -125,11 +132,22 @@ combine returns activation payload through an unweighted rank-sum path. Expert-o
 outside isolated combine timing AND outside the measured paired roundtrip, so `roundtrip` means
 dispatch then combine — the transport — in every row. It is reported as its own `stage` component
 wherever it does device work. The one exception is the `CX_FP8_CONSUME=dequant` verification hatch,
-which puts the conversion back inside the chain on purpose. Each component declares
+which puts the conversion back inside the chain on purpose.
+
+Read `implementation.stage_excluded_from_roundtrip` as "there was device-work staging and it was
+hoisted out of the chain", not as "this row's roundtrip is stage-free". It is gated on whether the
+backend's `stage()` does device work at all, so it is `false` in two unrelated situations, and the
+`stage` component is what separates them: **absent** means the backend has nothing to stage (the
+staging is a bare pointer assignment, as for NCCL EP and for every BF16 row that hands the receive
+buffer straight to combine), while **present alongside `false`** means the `dequant` hatch put the
+conversion back inside the chain. A reader that treats `false` alone as "roundtrip includes staging"
+will wrongly subtract a cost the row never paid. Each component declares
 availability, origin, and sample count. A paired-only API reports null isolated components.
 `isolated_sum` is derived.
 
-Headline latency is the p50 of the per-iteration cross-rank MAX: a layer is not finished until
+Headline latency is the p99 of the per-iteration cross-rank MAX (`p50` is emitted alongside it, and
+`summarize.py` prints both; the p99 is the figure the published cohorts rank on). MAX is the
+reduction because a layer is not finished until
 its slowest rank is, so MAX is the completion cost, and it charges inter-rank entry stagger to
 whichever component the ranks entered unevenly. How much stagger there is depends on the code
 path AND the precision, not only on the fleet: on identical h200 low-latency decode cells the
@@ -201,9 +219,12 @@ padding, and backend buffer capacity. BF16 moves 2 bytes per value with no scale
 dispatch moves 1 byte per value, plus per-128-block FP32 scales for DeepEP's and UCCL-EP's blockwise
 codec (none for MoRI's plain e4m3 cast), while combine stays BF16 — so the dispatch and combine directions can carry
 different byte counts and the roundtrip is their per-field sum. The rank-deduplicated count is exact
-for the normal-mode layout; the low-latency layout sends one copy per (token, expert) assignment
-rather than per (token, rank), so for a token whose experts share a destination rank this logical
-count is a lower bound on the bytes the low-latency kernels actually move. Latency (the headline) is
+for the normal-mode layout. It is also exact for a low-latency kernel that deduplicates per rank
+(MoRI's `IntraNodeLL`, whose combine is an unweighted rank-sum). The low-latency kernels that apply
+top-k weights inside combine instead send one copy per (token, expert) assignment rather than per
+(token, rank), so for a token whose experts share a destination rank this logical count is a lower
+bound on the bytes those kernels actually move. Each row states which basis it used in
+`logical_copies`, so the two are never silently mixed. Latency (the headline) is
 measured directly and is unaffected. Algorithm bandwidth, bus bandwidth,
 wire utilization, and physical-link utilization are not emitted without a defined primitive model or
 transport counters. Logical bandwidth must never be labeled physical bandwidth. Payload and token
diff --git a/experimental/CollectiveX/summarize.py b/experimental/CollectiveX/summarize.py
index 458b56acb..ca6388375 100644
--- a/experimental/CollectiveX/summarize.py
+++ b/experimental/CollectiveX/summarize.py
@@ -85,6 +85,11 @@ def _headline(document: dict) -> tuple:
     the per-iteration MAX-MIN. Read the pair as a bracket, not the two ends as rival metrics.
     """
     rows = document["measurement"]["rows"]
+    if not rows:
+        # This renderer validates nothing and must degrade rather than crash: a shard that
+        # reported an outcome but no measurement rows is malformed, not a reason to lose the
+        # whole table.
+        return ("-", "-", "-", "-", "-")
     row = next((item for item in rows if item["tokens_per_rank"] == 64), rows[len(rows) // 2])
     latency = row["components"]["roundtrip"]["percentiles_us"]
 
diff --git a/experimental/CollectiveX/tests/test_runtime.py b/experimental/CollectiveX/tests/test_runtime.py
index 750fa0027..a96edb93d 100644
--- a/experimental/CollectiveX/tests/test_runtime.py
+++ b/experimental/CollectiveX/tests/test_runtime.py
@@ -655,6 +655,22 @@ def test_a_rank_claimed_by_an_earlier_slot_contributes_once(self):
         )
         self.assertEqual(combined.item(), 0.5)
 
+    def test_the_first_slot_claiming_a_rank_is_the_one_that_survives(self):
+        torch = _torch
+        # Which duplicate the kernel blanks is invisible when the repeated payload is the only
+        # value in play, so the case above cannot tell keep-first from keep-last. These
+        # contributions cancel instead: blanking the LATER slot reduces (256+1)->256 and
+        # (-256+0)->-256 to 0.0, while blanking the EARLIER one gives (0+1)->1 and
+        # (-256+256)->0, i.e. 1.0. The 257 -> 256 step is the BF16 rounding that makes the two
+        # models disagree at all.
+        destination = torch.tensor([[0, 1, 2, 0]])
+        messages = torch.tensor([[[256.0]], [[1.0]], [[-256.0]]])
+        combined = ep_harness._topk_slot_tree_combine(
+            torch, destination, torch.ones_like(destination, dtype=torch.bool),
+            messages, torch.bfloat16,
+        )
+        self.assertEqual(combined.item(), 0.0)
+
 
 
 @unittest.skipUnless(_torch is not None, "quantize-identity checks require torch")

From cf4b1faa32a8888355739a19c97c23027e4cd178 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 19:24:23 +0800
Subject: [PATCH 11/34] =?UTF-8?q?CollectiveX:=20run=20MoRI=20in=20the=20bu?=
 =?UTF-8?q?ffer=20mode=20its=20pinned=20warp=20count=20belongs=20to=20/=20?=
 =?UTF-8?q?CollectiveX=EF=BC=9A=E8=AE=A9=20MoRI=20=E8=BF=90=E8=A1=8C?=
 =?UTF-8?q?=E5=9C=A8=E4=B8=8E=E5=85=B6=E5=9B=BA=E5=AE=9A=20warp=20?=
 =?UTF-8?q?=E6=95=B0=E7=9B=B8=E5=8C=B9=E9=85=8D=E7=9A=84=E7=BC=93=E5=86=B2?=
 =?UTF-8?q?=E6=A8=A1=E5=BC=8F=E4=B8=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Pinning the engines' `warp_num_per_block` of 16 was only half a configuration. MoRI's own tuning
tables key combine on `zero_copy`, selecting ~16 warps when the caller supplies an external input
buffer and 4-8 when it uses a registered one, and this adapter used a registered buffer. So the
previous commit took production's warp count into a mode production does not run, which matched
neither vLLM/SGLang nor any MoRI-authored artifact.

That mismatch is measured, not theoretical. On-metal A/B of warps 16 vs 8 in registered-buffer mode,
three chips, every arm fully correct:

    T      MI355X          MI300X    MI325X
    <=64   -8..-10% (win)  flat      flat
    128    +17.2/+18.3%    +13.4%    +16.9%
    256    -0.1/-0.3%      +14.5%    +16.8%
    512    +61.2/+62.3%    +74.1%    +77.9%

Roundtrip p50 at T=512 moved +38% on MI300X and +40% on MI325X. MI355X used an exact two-line revert
with two replicates agreeing within ~1pp; the gfx942 arms are cross-commit but decompose per
component, since dispatch warps were already 16 and dispatch/stage stayed matched.

So `use_external_inp_buf` moves to True on every path, which is MoRI's default, what vLLM leaves it
at, and what SGLang sets explicitly. The registered-buffer branches stay, so the mode remains a
one-line A/B.

Two consequences, both intended:

- BF16 rows no longer report a `stage` component. With an external input buffer the kernel does its
  own staging copy, so handing over the dispatch output is a bare assignment with no device work.
  FP8 rows still stage, because the received payload must be dequantized.
- Reading MoRI's source settled a claim this file had wrong. The external-input staging copy loops
  `i < totalRecvTokenNum` over `inpTokenBuf` (intranode.hpp:542-560), so it never reads past the
  receive count -- the comment asserting "the kernel reads the padded plane directly, so all of it
  must be BF16" was false. FP8 now dequantizes only the filled rows on this path too; casting the
  whole cap-sized plane would have been ~99.8% padding at T=1, the same waste the zero-copy branch
  already documented.

methodology.md now states the full pinned tuple including the warp count, that the two settings move
together and why, and the one asymmetry: the low-latency arm has no engine config to match, because
SGLang's low-latency path pins AsyncLL at 8 warps while this suite uses IntraNodeLL (AsyncLL is
split-phase and fails silently under a single-call harness). Inheriting 16 there is a choice, and is
now labelled as one.

The A/B above measured the OLD mode. It has to be re-run in the new one before these numbers are
quoted as the cost of 16 warps generally.

91 tests pass.

中文：固定引擎所用的 `warp_num_per_block`=16 只完成了配置的一半。MoRI 自带的调优表按 `zero_copy`
索引 combine：调用方提供外部输入缓冲时选约 16 warps，使用注册缓冲时选 4-8。而本适配器此前使用注册
缓冲，因此上一个提交把生产环境的 warp 数用在了生产环境并不采用的模式下，既不匹配 vLLM/SGLang，也
不匹配任何 MoRI 官方产物。

该不匹配已实测，并非推测。在注册缓冲模式下对 warps 16 与 8 做实机 A/B，三款芯片，各组结果均完全
正确（表见上）。T=512 时 roundtrip p50 在 MI300X 上上升 38%，在 MI325X 上上升 40%。MI355X 使用
精确的两行回退并有两次重复，彼此相差约 1 个百分点；gfx942 各组为跨提交对比，但可按组件分解，因为
dispatch warps 本就是 16，且 dispatch/stage 在两组间保持一致。

因此 `use_external_inp_buf` 在所有路径上改为 True——这是 MoRI 的默认值、vLLM 未改动的值，也是
SGLang 显式设置的值。注册缓冲分支予以保留，使该模式仍可通过一行改动进行 A/B。

两项预期后果：

- BF16 行不再上报 `stage` 组件。使用外部输入缓冲时，kernel 自行完成 staging 拷贝，因此交出
  dispatch 输出只是一次裸赋值，没有设备端工作。FP8 行仍保留 stage，因为收到的载荷需要反量化。
- 阅读 MoRI 源码纠正了本文件中的一处错误论断。外部输入的 staging 拷贝循环条件为
  `i < totalRecvTokenNum`（intranode.hpp:542-560），绝不会读到接收计数之外，因此"kernel 直接读取
  填充后的整个平面，故必须全部为 BF16"这一注释是错的。现在该路径下 FP8 同样只反量化已填充的行；
  若转换整个按上限分配的平面，在 T=1 时约 99.8% 都是填充，正是零拷贝分支早已记录的浪费。

methodology.md 现已写明完整的固定配置元组（含 warp 数）、两项设置为何必须一同变动，以及唯一的不
对称之处：低延迟分支没有可对标的引擎配置，因为 SGLang 的低延迟路径固定使用 AsyncLL 且 warps=8，
而本套件使用 IntraNodeLL（AsyncLL 为分阶段实现，在单次调用的测量框架下会静默失败）。在该处沿用
16 是一种选择，现已如实标注。

上述 A/B 测量的是**旧**模式。在把这些数字当作 16 warps 的普遍代价引用之前，必须在新模式下重跑。

91 项测试通过。
---
 experimental/CollectiveX/bench/ep_mori.py    | 20 ++++++++++++---
 experimental/CollectiveX/docs/methodology.md | 26 +++++++++++++++-----
 2 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_mori.py b/experimental/CollectiveX/bench/ep_mori.py
index 2bb621e03..cd0b9af0c 100644
--- a/experimental/CollectiveX/bench/ep_mori.py
+++ b/experimental/CollectiveX/bench/ep_mori.py
@@ -124,7 +124,16 @@ def __init__(self, args, rank, world_size, local_rank, device):
         self._inter_node = kernel_name == "InterNodeV1"
         self.num_qps = 1
         self.block_num, self.rdma_block_num, self.dispatch_warps, self.combine_warps = blocks
-        self._external_input = self._inter_node
+        # External input buffer on every path, matching what the engines run: vLLM leaves
+        # `use_external_inp_buf` at MoRI's default of True and SGLang sets it True explicitly, so
+        # registered-buffer (zero-copy) mode is a configuration no production engine selects. It
+        # also has to move together with `combine_warps`: MoRI's own tuned tables key combine on
+        # `zero_copy`, picking ~16 warps for external input against 4-8 for registered, so pinning
+        # the engines' 16 warps while staying registered matched neither. Measured cost of that
+        # mismatch, warps 16 vs 8 in registered mode on three chips: +13-18% combine at T=128 and
+        # +61-78% at T=512, correct in both arms. The zero-copy branches below are kept so the
+        # mode remains a one-line A/B rather than an archaeology exercise.
+        self._external_input = True
         # Registered-input MoRI copies expert output into a device-side symmetric buffer. External
         # input kernels consume the dispatch output directly, so their stage is not applicable.
         # Under FP8, stage also dequantizes the received fp8 payload to BF16 (device work) on
@@ -299,9 +308,14 @@ def stage(self, p, h):
         if not isinstance(rows, int) or rows < 0 or rows > h.dispatch_output.size(0):
             raise RuntimeError("MoRI receive count was not validated before staging")
         if self._external_input:
-            # The kernel reads the padded plane directly here, so all of it must be BF16.
+            # The external-input staging copy is bounded by the receive count, not by the buffer:
+            # `EpCombineIntraNodeKernel_body` loops `i < totalRecvTokenNum` over `inpTokenBuf`
+            # (intranode.hpp:542-560), so it never reads past `rows` and the padding it leaves
+            # behind is untouched. BF16 therefore hands over the dispatch output as-is, and FP8
+            # dequantizes only the filled rows -- casting the whole cap-sized plane here would be
+            # ~99.8% padding at T=1, the same waste the zero-copy branch below documents.
             h.combine_input = (
-                h.dispatch_output.to(torch.bfloat16) if self._fp8 else h.dispatch_output
+                h.dispatch_output[:rows].to(torch.bfloat16) if self._fp8 else h.dispatch_output
             )
             return None
         # Zero-copy path: combine only ever reads the `rows` slots dispatch filled, so cast
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index fc37ff5c0..0946bcbcc 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -66,12 +66,26 @@ represented by two physical RDMA ranks, with eight scale-up ranks per domain. GB
 scale-up and uses LSA. MoRI EP8 uses the direct IntraNode kernel on every CDNA SKU; its EP16 InterNodeV1 path is
 configured but unsupported (transport-layer combine corruption, ROCm/mori#475) and never dispatched.
 MoRI runs under its MANUAL launch mode with a pinned launch config, because that is what the engines
-run: neither vLLM nor SGLang sets `MORI_EP_LAUNCH_CONFIG_MODE`, and both pin block_num 80 with
-rdma_block_num 0. These numbers therefore describe the engine-integrated configuration, not MoRI's
-peak -- its own shipped tuning tables reach roughly 1.3-1.4x faster combine at T=256 on gfx950 with a
-config no engine selects, and AUTO would not reproduce those tables anyway (there is no BF16 gfx950
-dispatch rule and no gfx950 IntraNodeLL combine table, so AUTO falls back to hard-coded defaults and
-couples the result to whichever MoRI revision is pinned). UCCL-EP is a drop-in, API-identical DeepEP replacement that keeps the legacy `Buffer`
+run: neither vLLM nor SGLang sets `MORI_EP_LAUNCH_CONFIG_MODE`, and both pin block_num 80,
+rdma_block_num 0, and `warp_num_per_block` 16 for the intra-node kernel, applied to dispatch and
+combine alike (neither passes a per-call override, so combine inherits the 16). Both also run with an
+external input buffer, which is MoRI's default and which SGLang sets explicitly. Those two settings
+are pinned together deliberately: MoRI's own tuning tables key combine on `zero_copy`, selecting
+roughly 16 warps for external input against 4-8 for a registered buffer, so taking the engines' warp
+count while keeping a registered buffer would match neither. The cost of that mismatch is not
+hypothetical -- measured across MI300X, MI325X and MI355X, 16 warps in registered-buffer mode is
++13-18% combine at T=128 and +61-78% at T=512 against 8, correct in both arms. With an external
+input buffer the kernel does its own staging copy, bounded by the receive count, so BF16 rows hand
+over the dispatch output unchanged and report no `stage` component at all; FP8 rows still stage,
+because the received payload has to be dequantized. These numbers therefore describe the
+engine-integrated configuration, not MoRI's peak: its shipped tuning tables reach a faster combine
+with per-shape block and warp counts no engine selects, and AUTO would not reproduce those tables
+anyway (there is no BF16 gfx950 dispatch rule and no gfx950 IntraNodeLL combine table, so AUTO falls
+back to hard-coded defaults and couples the result to whichever MoRI revision is pinned). One
+asymmetry is worth stating: the low-latency arm has no engine-integrated configuration to match at
+all, because SGLang's low-latency path pins `AsyncLL` at 8 warps while this suite uses `IntraNodeLL`
+(`AsyncLL` is split-phase and fails silently under a single-call harness), so the low-latency launch
+config is inherited from the normal-mode tuple by choice rather than by precedent. UCCL-EP is a drop-in, API-identical DeepEP replacement that keeps the legacy `Buffer`
 `dispatch`/`combine` (unweighted rank-sum) but routes it over CPU-proxy GPUDirect RDMA on plain
 `libibverbs` — no NVSHMEM/IBGDA — with software message ordering, atomics, and flow control; its
 scale-up is single-node `cudaIpc` over NVLink/XGMI (so the scale-up domain is one physical node,

From ff71649e314c00cb2d267e68a20aa4426d1e8223 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 19:28:18 +0800
Subject: [PATCH 12/34] =?UTF-8?q?CollectiveX:=20put=20the=20mode=20in=20th?=
 =?UTF-8?q?e=20artifact=20filename=20/=20CollectiveX=EF=BC=9A=E5=B0=86=20m?=
 =?UTF-8?q?ode=20=E5=86=99=E5=85=A5=E4=BA=A7=E7=89=A9=E6=96=87=E4=BB=B6?=
 =?UTF-8?q?=E5=90=8D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The result path was `{runner}_{backend}_{precision}_{phase}_{ts}-cNNN.json` with no mode segment, so
a low-latency case and a normal case sharing runner, backend, precision, phase and index resolved to
byte-identical paths. Driving several shards from one loop shares the second-resolution timestamp,
and the second case silently overwrote the first — no error, just a missing artifact.

CI hides this: every leg is its own job with its own timestamp. It surfaced on metal, where it cost
two runs before anyone noticed the files were gone rather than never written.

The existing comment already explained why precision is in the filename. Mode belongs there for
exactly the same reason, so this extends that fix rather than inventing a new scheme.

The four pinned filenames in the seam-contract tests move to `..._bf16_normal_decode_...`, and the
low-latency round-trip test now asserts its own `out` — it previously checked mode, phase and scope
but never the filename, which is precisely why a collision between it and its normal-mode sibling
was invisible to the suite.

91 tests pass.

中文：产物路径此前为 `{runner}_{backend}_{precision}_{phase}_{ts}-cNNN.json`，不含 mode 段，因此
runner、backend、precision、phase 与序号相同的 low-latency 用例与 normal 用例会解析出完全相同的
路径。若在一个循环中驱动多个 shard，它们共享秒级时间戳，后一个用例会静默覆盖前一个——没有报错，
只是产物缺失。

CI 掩盖了该问题：每条 leg 都是独立作业，拥有各自的时间戳。它在实机上暴露，并且在有人意识到文件是
被覆盖而非从未写入之前，已经浪费了两次运行。

原有注释已说明 precision 为何要写入文件名。mode 出于完全相同的理由同样应当写入，因此本次修改是对
该处修复的延伸，而非另立方案。

seam 契约测试中四处固定的文件名改为 `..._bf16_normal_decode_...`，并且 low-latency 往返测试现在
会断言自身的 `out`——它此前检查了 mode、phase 与 scope，却从未检查文件名，这正是它与 normal 模式
同胞用例之间的冲突对测试套件不可见的原因。

91 项测试通过。
---
 experimental/CollectiveX/runtime/config.py     | 11 +++++++----
 experimental/CollectiveX/tests/test_runtime.py | 17 +++++++++++++----
 2 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/experimental/CollectiveX/runtime/config.py b/experimental/CollectiveX/runtime/config.py
index d1e6132bd..c1bb3183a 100644
--- a/experimental/CollectiveX/runtime/config.py
+++ b/experimental/CollectiveX/runtime/config.py
@@ -128,11 +128,14 @@ def _emit_argv(case: dict, version: object, runner: str, ts: str, index: int) ->
     iters, trials, warmup = str(case["timing"]).split(":")
     for flag, value in (("--iters", iters), ("--trials", trials), ("--warmup", warmup)):
         argv += [flag, value]
-    # precision is part of the filename so a cell's bf16 and fp8 legs (distinct shards
-    # sharing runner/backend/phase and each numbering cases from index 0) cannot collide
-    # when they land in the shared results/ dir under the same second-resolution ts.
+    # precision and mode are part of the filename so a cell's legs (distinct shards sharing
+    # runner/backend/phase and each numbering cases from index 0) cannot collide when they land
+    # in the shared results/ dir under the same second-resolution ts. Mode matters as much as
+    # precision: CI gives every leg its own job and therefore its own ts, but anyone driving
+    # several shards from one loop shares it, and a low-latency case silently overwrote the
+    # normal case with the same precision, phase and index -- losing artifacts with no error.
     out = (
-        f"results/{runner}_{case['backend']}_{case['precision']}_{case['phase']}"
+        f"results/{runner}_{case['backend']}_{case['precision']}_{case['mode']}_{case['phase']}"
         f"_{ts}-c{index:03d}.json"
     )
     argv += ["--out", out]
diff --git a/experimental/CollectiveX/tests/test_runtime.py b/experimental/CollectiveX/tests/test_runtime.py
index a96edb93d..8bd1407cf 100644
--- a/experimental/CollectiveX/tests/test_runtime.py
+++ b/experimental/CollectiveX/tests/test_runtime.py
@@ -380,7 +380,7 @@ def test_case_args_round_trips_through_the_run_ep_parser(self) -> None:
         self.assertEqual(args.version, 1)
         self.assertEqual(args.seed, self.CASE["seed"])
         self.assertEqual((args.iters, args.trials, args.warmup), (8, 256, 32))
-        self.assertEqual(args.out, "results/h200-dgxc_deepep-v2_bf16_decode_TS-c000.json")
+        self.assertEqual(args.out, "results/h200-dgxc_deepep-v2_bf16_normal_decode_TS-c000.json")
 
     def test_case_args_fails_closed_on_placement_mismatch(self) -> None:
         with self.assertRaises(subprocess.CalledProcessError):
@@ -402,6 +402,15 @@ def test_low_latency_case_round_trips_through_the_run_ep_parser(self) -> None:
         args = self._run_ep_parser().parse_args(argv)
         self.assertEqual((args.mode, args.phase, args.scope), ("low-latency", "decode", "scale-up"))
         self.assertEqual(args.case_id, ll_case["case_id"])
+        # The filename must carry the mode. Without it this case and its normal-mode sibling
+        # produce byte-identical paths (same runner, backend, precision, phase and index), so
+        # driving both under one timestamp silently overwrites one artifact with the other --
+        # which cost two on-metal runs before it was noticed. CI hides it by giving every leg
+        # its own job and therefore its own ts.
+        # ... which is what test_case_args_round_trips pins as `..._bf16_normal_decode_...`.
+        self.assertEqual(
+            args.out, "results/h200-dgxc_deepep-v2_bf16_low-latency_decode_TS-c000.json"
+        )
 
     def test_uccl_ep_case_round_trips_through_the_run_ep_parser(self) -> None:
         # A uccl-ep case flows through the same generic codec; run_ep's --backend choices
@@ -416,7 +425,7 @@ def test_uccl_ep_case_round_trips_through_the_run_ep_parser(self) -> None:
         args = self._run_ep_parser().parse_args(argv)
         self.assertEqual(args.backend, "uccl-ep")
         self.assertEqual(args.case_id, uccl_case["case_id"])
-        self.assertEqual(args.out, "results/h200-dgxc_uccl-ep_bf16_decode_TS-c000.json")
+        self.assertEqual(args.out, "results/h200-dgxc_uccl-ep_bf16_normal_decode_TS-c000.json")
 
     def test_nccl_ep_case_round_trips_through_the_run_ep_parser(self) -> None:
         # A nccl-ep case flows through the same generic codec; run_ep's --backend choices must
@@ -431,7 +440,7 @@ def test_nccl_ep_case_round_trips_through_the_run_ep_parser(self) -> None:
         args = self._run_ep_parser().parse_args(argv)
         self.assertEqual(args.backend, "nccl-ep")
         self.assertEqual(args.case_id, nccl_case["case_id"])
-        self.assertEqual(args.out, "results/h200-dgxc_nccl-ep_bf16_decode_TS-c000.json")
+        self.assertEqual(args.out, "results/h200-dgxc_nccl-ep_bf16_normal_decode_TS-c000.json")
 
     def test_flashinfer_ep_case_round_trips_through_the_run_ep_parser(self) -> None:
         # A flashinfer-ep case flows through the same generic codec; run_ep's --backend
@@ -450,7 +459,7 @@ def test_flashinfer_ep_case_round_trips_through_the_run_ep_parser(self) -> None:
         self.assertEqual(args.backend, "flashinfer-ep")
         self.assertEqual(args.case_id, flashinfer_case["case_id"])
         self.assertEqual(
-            args.out, "results/h200-dgxc_flashinfer-ep_bf16_decode_TS-c000.json"
+            args.out, "results/h200-dgxc_flashinfer-ep_bf16_normal_decode_TS-c000.json"
         )
 
     def test_mirrored_backend_choices_match_run_ep(self) -> None:

From 7ff4533130674c3fcdf592ef5336b4e03754195a Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 19:45:49 +0800
Subject: [PATCH 13/34] =?UTF-8?q?CollectiveX:=20stop=20the=20per-case=20ha?=
 =?UTF-8?q?ng=20guard=20from=20killing=20slow-but-healthy=20legs=20/=20Col?=
 =?UTF-8?q?lectiveX=EF=BC=9A=E9=81=BF=E5=85=8D=E5=8D=95=E7=94=A8=E4=BE=8B?=
 =?UTF-8?q?=E6=8C=82=E8=B5=B7=E4=BF=9D=E6=8A=A4=E6=9D=80=E6=8E=89=E4=BB=85?=
 =?UTF-8?q?=E4=BB=85=E6=98=AF=E8=BE=83=E6=85=A2=E7=9A=84=E6=AD=A3=E5=B8=B8?=
 =?UTF-8?q?=E8=BF=90=E8=A1=8C?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two shards in sweep 30799122697 went red without any benchmark failure. The h100 one is provably not
a measurement fault: its uploaded artifact has both cases at status=success with every rung
passed=true, and the timeline says why it died anyway.

    job start                09:05:45
    decode artifact written  09:17:14   (case 0, fine)
    prefill srun begins      ~09:17:15  -> 900s deadline at 09:32:15
    prefill artifact written  09:32:17  <-- 2 seconds past the deadline
    job fails                 09:32:37  <-- +20s, inside `timeout -k 30`'s grace

The case completed its measurement, wrote a correct artifact, and `timeout` killed the srun two
seconds later. The shard is red with good data attached.

b200's uccl-ep FP8 leg is the same fault with the kill made explicit: `ERROR: case 1 failed`, then
`STEP ... CANCELLED ... DUE to SIGNAL Killed` and `Force Terminated`. Case 1 is prefill in both, and
uccl-ep FP8 is the slowest FP8 path in the suite (it carries a per-iteration scale transpose on top
of the quantize), so it hit the wall on a single node where deepep-v2 needed two.

COLLX_RUN_TIMEOUT is a hang guard, not a work budget, and 900s no longer sits clear of the most
expensive legitimate case. FP8 got slower deliberately: moving the quantize inside the timed dispatch
costs ~233us vs ~126us per dispatch on GB300, so the heaviest FP8 prefill legs now land right on the
old line. That is the change working as intended, colliding with a cap never sized for it.

1800 is the value the AMD launcher had already picked for this exact reason, so this makes it the
shared default and drops the per-launcher override — one number, no drift, and a 2x margin over the
~900s a real FP8 prefill leg needs. AMD behaviour is unchanged.

This is orthogonal to whether the compiled quantize is the right call at all (still being measured):
the cap killed cases that had already finished, so it was too tight regardless of that outcome.

Not fixed here, recorded so nobody re-diagnoses them as this: b300's FP8 EP16 leg failed in
distributed rendezvous with `DistStoreError: Timed out after 601 seconds waiting for clients. 8/16
clients joined` — one node's ranks never arrived, an infrastructure fault unrelated to FP8 or to this
cap. gb300's FP8 EP16 leg failed in source preparation with a GitHub 502 on clone. Both need a
re-run, not a code change.

91 tests pass.

中文：扫描 30799122697 中有两个 shard 在没有任何基准测试失败的情况下变红。其中 h100 那个可以证明不是
测量问题：其上传的产物中两个 case 均为 status=success、所有 rung 均 passed=true，而时间线说明了它为何
仍然失败（见上表）。

该 case 完成了测量、写出了正确的产物，两秒后 `timeout` 杀掉了 srun。该 shard 带着正确数据变红。

b200 的 uccl-ep FP8 leg 是同一问题，且杀死过程更为明确：先 `ERROR: case 1 failed`，随后
`STEP ... CANCELLED ... DUE to SIGNAL Killed` 与 `Force Terminated`。两者的 case 1 都是 prefill，而
uccl-ep FP8 是本套件中最慢的 FP8 路径（除量化外还带有每次迭代的 scale 转置），因此它在单节点上就撞上
了上限，而 deepep-v2 需要两节点才会。

COLLX_RUN_TIMEOUT 是挂起保护，而非工作量预算，900 秒已不再明显高于最昂贵的正常用例。FP8 是有意变慢
的：把量化移入计时期的 dispatch 后，在 GB300 上每次 dispatch 约 233us（原约 126us），因此最重的 FP8
prefill leg 正好压在旧界线上。这是该改动按预期工作，却与一个从未为此设定的上限相撞。

1800 正是 AMD launcher 早已因同样原因选定的值，因此本次将其提升为共享默认值并移除各 launcher 中的重复
覆盖——一个数字、不再漂移，并相对真实 FP8 prefill leg 所需的约 900 秒留有 2 倍余量。AMD 行为不变。

这与"编译版量化本身是否正确"（仍在测量中）是两个独立问题：该上限杀掉的是已经跑完的 case，因此无论那个
结论如何，它都太紧了。

本次未修复、但记录于此以免被误诊为同一问题：b300 的 FP8 EP16 leg 在分布式 rendezvous 阶段失败，报
`DistStoreError: Timed out after 601 seconds waiting for clients. 8/16 clients joined`——某个节点的
rank 从未加入，属于与 FP8 及本上限无关的基础设施故障。gb300 的 FP8 EP16 leg 在源码准备阶段因 GitHub
clone 返回 502 而失败。两者需要重跑，而非代码改动。

91 项测试通过。
---
 experimental/CollectiveX/launchers/launch_mi-amds.sh | 1 -
 experimental/CollectiveX/runtime/common.sh           | 9 ++++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/experimental/CollectiveX/launchers/launch_mi-amds.sh b/experimental/CollectiveX/launchers/launch_mi-amds.sh
index eee35d1fa..1f891295f 100644
--- a/experimental/CollectiveX/launchers/launch_mi-amds.sh
+++ b/experimental/CollectiveX/launchers/launch_mi-amds.sh
@@ -51,7 +51,6 @@ if [ "$NODES" -gt 1 ]; then
 else
   export COLLX_TRANSPORT=xgmi
 fi
-export COLLX_RUN_TIMEOUT="${COLLX_RUN_TIMEOUT:-1800}"
 collx_apply_network_profile "$NODES" "$COLLX_TRANSPORT"
 collx_require_vars COLLX_IMAGE COLLX_IMAGE_PLATFORM COLLX_PARTITION COLLX_SQUASH_DIR COLLX_STAGE_DIR
 PARTITION="$COLLX_PARTITION"; SQUASH_DIR="$COLLX_SQUASH_DIR"
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 24072604a..14fb91a8e 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -906,7 +906,14 @@ collx_run_shard() {
       || { rm -f "$argv_file"; collx_die "case $ci produced no benchmark arguments"; }
     collx_log "EP${NGPUS}[$((ci + 1))/$expected_cases] $COLLX_BENCH"
     runtime_log="$(collx_private_log_path "runtime-c$(printf '%03d' "$ci")")"
-    if ! timeout -k 30 "${COLLX_RUN_TIMEOUT:-900}" \
+    # A hang guard, NOT a work budget: it must sit clear above the most expensive legitimate
+    # case, or it starts killing cases that are merely slow. It did. An h100 FP8 prefill EP16
+    # case wrote a complete, all-rungs-passed artifact 2s past a 900s deadline and was killed
+    # anyway, turning a good measurement into a red shard with correct data attached -- and FP8
+    # got slower on purpose when the quantize moved inside the timed dispatch (~233us vs ~126us
+    # per dispatch on GB300), so the heaviest FP8 prefill legs now sit right on that line.
+    # 1800 is what the AMD launcher already used; one shared number, no per-launcher drift.
+    if ! timeout -k 30 "${COLLX_RUN_TIMEOUT:-1800}" \
       srun --jobid="$JOB_ID" --nodes="$NODES" \
       --ntasks="$NGPUS" --ntasks-per-node="$GPN" --chdir=/tmp \
       --container-name="$container_name" --container-image="$SQUASH_FILE" \

From 71a870038cf5e94b6ffec31b4821a60d21c31ea8 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 19:52:33 +0800
Subject: [PATCH 14/34] =?UTF-8?q?CollectiveX:=20state=20the=20BF16=20stage?=
 =?UTF-8?q?=20shape=20precisely=20/=20CollectiveX=EF=BC=9A=E5=87=86?=
 =?UTF-8?q?=E7=A1=AE=E8=A1=A8=E8=BF=B0=20BF16=20=E7=9A=84=20stage=20?=
 =?UTF-8?q?=E5=BD=A2=E6=80=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On-metal validation of the external-input switch checked the artifact shape and found the claim in
this file too strong. BF16 rows do not "report no `stage` component at all" -- the component is still
declared, it just carries no percentiles, exactly as any backend whose staging is a bare pointer
assignment already reports it (NCCL EP BF16 has always looked like this). What changes for MoRI BF16
is the percentiles and the `stage_excluded_from_roundtrip` flag, not the presence of the key.

The distinction matters to a consumer: one keying on the presence of `stage` sees no change at all,
while one reading percentiles or the flag sees the intended change. Saying "no component" would send
the first kind looking for a key that is still there.

中文：对外部输入缓冲切换的实机验证检查了产物形态，发现本文件中的表述过强。BF16 行并非"完全不上报
`stage` 组件"——该组件仍会声明，只是不携带百分位数据，与任何 staging 为裸指针赋值的后端此前的上报
方式完全一致（NCCL EP 的 BF16 一直如此）。对 MoRI BF16 而言，变化的是百分位数据与
`stage_excluded_from_roundtrip` 标志，而非该键是否存在。

这一区别对消费方很重要：以 `stage` 键是否存在为判据的一方看不到任何变化，而读取百分位或该标志的一方
才会看到预期变化。若表述为"没有该组件"，会让前一类消费方去寻找一个其实仍然存在的键。
---
 experimental/CollectiveX/docs/methodology.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 0946bcbcc..37e089dff 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -76,8 +76,9 @@ count while keeping a registered buffer would match neither. The cost of that mi
 hypothetical -- measured across MI300X, MI325X and MI355X, 16 warps in registered-buffer mode is
 +13-18% combine at T=128 and +61-78% at T=512 against 8, correct in both arms. With an external
 input buffer the kernel does its own staging copy, bounded by the receive count, so BF16 rows hand
-over the dispatch output unchanged and report no `stage` component at all; FP8 rows still stage,
-because the received payload has to be dequantized. These numbers therefore describe the
+over the dispatch output unchanged: their `stage` component is still declared but carries no
+percentiles, the same way any backend whose staging is a bare pointer assignment reports it. FP8 rows
+still stage for real, because the received payload has to be dequantized. These numbers therefore describe the
 engine-integrated configuration, not MoRI's peak: its shipped tuning tables reach a faster combine
 with per-shape block and warp counts no engine selects, and AUTO would not reproduce those tables
 anyway (there is no BF16 gfx950 dispatch rule and no gfx950 IntraNodeLL combine table, so AUTO falls

From d1165a2de636944b0d501f94f43ee71c5fd301a4 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 19:58:10 +0800
Subject: [PATCH 15/34] =?UTF-8?q?CollectiveX:=20blank-line=20spacing=20aft?=
 =?UTF-8?q?er=20the=20wheel-version=20helper=20/=20CollectiveX=EF=BC=9A?=
 =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E7=89=88=E6=9C=AC=E5=88=A4=E5=AE=9A=E8=BE=85?=
 =?UTF-8?q?=E5=8A=A9=E5=87=BD=E6=95=B0=E5=90=8E=E7=9A=84=E7=A9=BA=E8=A1=8C?=
 =?UTF-8?q?=E9=97=B4=E8=B7=9D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two blank lines after a top-level def, matching every other module-level function in this
file. No behaviour change.

中文：顶层函数定义后保留两个空行，与本文件中其他所有模块级函数保持一致。行为无变化。
---
 experimental/CollectiveX/bench/ep_flashinfer.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/experimental/CollectiveX/bench/ep_flashinfer.py b/experimental/CollectiveX/bench/ep_flashinfer.py
index 3cfe4c6ac..fe89a9586 100644
--- a/experimental/CollectiveX/bench/ep_flashinfer.py
+++ b/experimental/CollectiveX/bench/ep_flashinfer.py
@@ -83,6 +83,7 @@ def _wheel_has_fp32_combine(version: str) -> bool:
     except InvalidVersion:
         return False
 
+
 # FP8 block size, matching the DeepSeek-V3 recipe every other FP8 backend here uses.
 _FP8_BLOCK = 128
 

From a8c0411f8513a5b101d0c483a60ea159c4c5003b Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 20:02:25 +0800
Subject: [PATCH 16/34] =?UTF-8?q?CollectiveX:=20say=20that=20the=20FP8=20q?=
 =?UTF-8?q?uantize=20charge=20is=20fixed=20per=20call,=20not=20per=20byte?=
 =?UTF-8?q?=20/=20CollectiveX=EF=BC=9A=E8=AF=B4=E6=98=8E=20FP8=20=E9=87=8F?=
 =?UTF-8?q?=E5=8C=96=E5=BC=80=E9=94=80=E6=98=AF=E6=AF=8F=E6=AC=A1=E8=B0=83?=
 =?UTF-8?q?=E7=94=A8=E7=9A=84=E5=9B=BA=E5=AE=9A=E6=88=90=E6=9C=AC=E8=80=8C?=
 =?UTF-8?q?=E9=9D=9E=E6=8C=89=E5=AD=97=E8=8A=82=E8=AE=A1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Moving the FP8 quantize inside the timed dispatch was the right call and stays. But the resulting
number is easy to misread at the bottom of the ladder, and nothing in the docs said so.

Measured from the sweep's own artifacts, DeepEP V2 decode, FP8 dispatch p50 minus its BF16 control:

    SKU          T=1    T=64   T=256   T=512
    h100        65.4    65.5    21.6   -16.1
    b200        58.8    57.9    45.4    +6.8
    b300        27.3    26.6     1.4    -7.7
    gb300       56.9    57.0    55.1   +50.4

Flat until transport dominates, then decaying, and negative by T=512 on two SKUs where halved
payload bytes more than repay it. FlashInfer EP's is larger (~107us on gb300); its codec is its own
and FP8 adds a fourth dispatch payload.

At T=1 the FP8 path moves FEWER bytes than BF16, so none of this is transport. It is per-call work,
and it exceeds the fused quantize's measured device time (1.5-3.6us depending on SKU) by more than an
order of magnitude — because `time_us` deliberately has no host sync before its start event, so a
near-idle stream at T=1 lets host-side launch cost land inside the window.

Compiling the quantize REDUCES this rather than causing it: an eager quantize measures 33-39us worse
per decode dispatch on h100 through the same window, which is why the fused compile stays. But
production issues one custom quantize op into an already-busy stream, so the small-T end of an FP8
`normal` row is the least production-representative number this suite emits. The doc now says to
compare FP8 against BF16 at the top of the ladder, where the charge is repaid, not at T=1.

中文：把 FP8 量化移入计时期的 dispatch 是正确的决定并将保留。但由此得到的数字在阶梯低端容易被误读，
而文档此前对此毫无说明。

依据扫描自身产物测得（DeepEP V2 decode，FP8 dispatch p50 减去其 BF16 对照，见上表）：在传输开销占据
主导之前基本持平，随后衰减，到 T=512 时在两款 SKU 上已转为负值——此时载荷字节减半带来的收益已超过该
开销。FlashInfer EP 的该开销更大（gb300 上约 107us）：其编解码是自有实现，且 FP8 会增加第四个
dispatch 载荷。

在 T=1 时，FP8 路径搬运的字节数比 BF16 **更少**，因此这部分完全不是传输开销，而是每次调用的固定
开销，并且比融合量化自身的实测设备时间（各 SKU 1.5-3.6us）高出一个数量级以上——原因是 `time_us`
有意在起始事件前不做主机同步，因此 T=1 时近乎空闲的流会把主机侧的启动开销计入窗口内。

编译量化是**降低**而非造成该开销：在同一窗口下，eager 量化在 h100 上每次 decode dispatch 要慢
33-39us，这正是保留融合编译的原因。但生产环境是向一个本已繁忙的流下发一个自定义量化算子，因此 FP8
`normal` 行在小 T 端是本套件给出的最不具生产代表性的数字。文档现已说明：应在阶梯高端（该开销已被
收回处）而非 T=1 处比较 FP8 与 BF16。
---
 experimental/CollectiveX/docs/methodology.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 37e089dff..3017727a4 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -30,6 +30,22 @@ it is charged **inside the measured dispatch** rather than prepared ahead of the
 issued as one fused kernel and guarded bitwise against its eager reference. This means an FP8
 `normal` dispatch number covers quantize-plus-transport while its BF16 control covers transport
 alone, and it is why FP8 `normal` rows are not comparable to runs published before sweep version 2.
+
+Read that charge as a **fixed per-call cost, not a payload-proportional one**, or the FP8-versus-BF16
+comparison will be misread at the bottom of the ladder. Measured on DeepEP V2 decode, FP8 dispatch p50
+minus its BF16 control is roughly flat until the transport starts to dominate — 65us on h100, 59us on
+b200, 27us on b300 and 57us on gb300 at T=1, holding within a microsecond or two through T=64, then
+decaying, and by T=512 it has gone slightly negative on h100 and b300 where halved payload bytes more
+than repay it. FlashInfer EP carries a larger one (~107us on gb300) because its codec is its own and
+FP8 adds a fourth dispatch payload. At T=1 the FP8 path moves *fewer* bytes than BF16, so none of this
+is transport: it is per-call work, and it exceeds the fused quantize's own device time (1.5-3.6us,
+measured per SKU) by more than an order of magnitude, because the timing window has no host sync
+before its start event (see below) and a near-idle stream at T=1 lets host-side launch cost land
+inside it. Compiling the quantize reduces this rather than causing it — an eager quantize measures
+33-39us worse per decode dispatch on h100 through the same window — but a production forward pass
+issues one custom quantize op into a stream that is already busy, so the small-T end of an FP8
+`normal` row is the least production-representative number the suite emits. Compare FP8 and BF16 at
+the top of the ladder, where the charge is repaid, rather than at T=1.
 `low-latency` rows are unaffected: those kernels either quantize internally (nothing for the caller
 to charge) or take pre-quantized input by API contract. NCCL EP is BF16-only this release, so its
 cells carry the control alone; the per-backend precision set lives in `sweep_matrix.py`'s

From c4b034c767ca4285d27a3729c9abdb6a83f967f4 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 20:34:07 +0800
Subject: [PATCH 17/34] =?UTF-8?q?CollectiveX:=20trim=20my=20own=20comments?=
 =?UTF-8?q?=20back=20to=20the=20ambient=20density=20/=20CollectiveX?=
 =?UTF-8?q?=EF=BC=9A=E5=B0=86=E6=96=B0=E5=A2=9E=E6=B3=A8=E9=87=8A=E7=B2=BE?=
 =?UTF-8?q?=E7=AE=80=E8=87=B3=E4=B8=8E=E5=91=A8=E8=BE=B9=E4=BB=A3=E7=A0=81?=
 =?UTF-8?q?=E4=B8=80=E8=87=B4=E7=9A=84=E5=AF=86=E5=BA=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A density check on this branch's additions: 44 comment lines against 45 code lines, a ratio of
0.98, where the files being edited sit at 0.13 (ep_harness.py), 0.15 (ep_flashinfer.py), 0.16
(common.sh, summarize.py) and 0.42 (ep_mori.py). Between 2.3x and 6.5x the surrounding code.

The excess was measured tables and incident history restated inline -- the warps 16-vs-8 numbers
across three chips, the ~233us-vs-~126us dispatch figures, the artifact-written-2s-past-deadline
timeline, the CI-gets-its-own-timestamp narrative. All of it already lives in the commit that
introduced it and, where a reader of published numbers needs it, in methodology.md. A code comment
is the wrong place for a third copy: it is the copy that goes stale silently.

What stays is the part a reader of the code cannot derive from the code: that `use_external_inp_buf`
and `combine_warps` have to move together because MoRI's tables key combine on `zero_copy`; that
the kernel's staging copy is bounded by `totalRecvTokenNum` so only filled rows need converting;
that the timeout is a hang guard rather than a work budget; that the version gate needs real
ordering because the safe failure direction is False.

Now 30 comment lines against the same 45 code lines, ratio 0.67. Still above ambient, which I think
is right for changes whose justification is upstream kernel behaviour a reader cannot see from here.

No behaviour change. 91 tests pass.

中文：对本分支新增内容做了注释密度检查：44 行注释对 45 行代码，比例 0.98；而被修改文件自身的比例为
0.13（ep_harness.py）、0.15（ep_flashinfer.py）、0.16（common.sh、summarize.py）与 0.42
（ep_mori.py）。即为周边代码的 2.3 至 6.5 倍。

超出部分是把实测数据表与问题经过原地复述了一遍——三款芯片上 warps 16 与 8 的对比数字、约 233us 对
约 126us 的 dispatch 数据、产物在截止时刻后 2 秒写出的时间线、CI 每条 leg 各有独立时间戳的来龙去脉。
这些内容已存在于引入它们的提交中，并且在已发布指标的读者需要时也存在于 methodology.md 中。代码注释
不该是第三份副本：它恰恰是会悄然过期的那一份。

保留下来的是读者无法从代码本身推导出的部分：`use_external_inp_buf` 与 `combine_warps` 必须同步变动，
因为 MoRI 的调优表按 `zero_copy` 索引 combine；kernel 的 staging 拷贝以 `totalRecvTokenNum` 为界，
因此只需转换已填充的行；该超时是挂起保护而非工作量预算；版本判定需要真正的版本序，因为安全的失败方向
是 False。

现为 30 行注释对同样的 45 行代码，比例 0.67。仍高于周边水平——对于其依据是读者在此处看不到的上游
kernel 行为的改动，我认为这是合适的。

行为无变化。91 项测试通过。
---
 .../CollectiveX/bench/ep_flashinfer.py        |  9 ++++----
 experimental/CollectiveX/bench/ep_mori.py     | 23 +++++++------------
 experimental/CollectiveX/runtime/common.sh    |  9 ++------
 experimental/CollectiveX/runtime/config.py    | 10 ++++----
 4 files changed, 18 insertions(+), 33 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_flashinfer.py b/experimental/CollectiveX/bench/ep_flashinfer.py
index fe89a9586..cd0fc0ae3 100644
--- a/experimental/CollectiveX/bench/ep_flashinfer.py
+++ b/experimental/CollectiveX/bench/ep_flashinfer.py
@@ -61,11 +61,10 @@
 def _wheel_has_fp32_combine(version: str) -> bool:
     """Does this wheel accumulate combine in FP32, per `_COMBINE_FP32_SINCE`?
 
-    Ordering matters more than it looks. A release candidate sorts BELOW its own release, so
-    `0.6.16rc1` predates the rewrite and must read as False; a naive digit scrape reads it as
-    0.6.16 and models FP32 against a kernel that still rounds per level, which can exceed
-    COMBINE_REL_TOL and RED a correct run. The reverse error is harmless (a few ulps, far
-    inside tolerance), so every unparseable input answers False.
+    Needs real version ordering, not a digit scrape: `0.6.16rc1` predates the rewrite, and
+    reading it as 0.6.16 models FP32 against a kernel that still rounds per level, which can
+    exceed COMBINE_REL_TOL and RED a correct run. The opposite error costs a few ulps, so
+    anything unparseable answers False.
     """
     try:
         from packaging.version import InvalidVersion, Version
diff --git a/experimental/CollectiveX/bench/ep_mori.py b/experimental/CollectiveX/bench/ep_mori.py
index cd0b9af0c..d5d0fb01e 100644
--- a/experimental/CollectiveX/bench/ep_mori.py
+++ b/experimental/CollectiveX/bench/ep_mori.py
@@ -124,15 +124,11 @@ def __init__(self, args, rank, world_size, local_rank, device):
         self._inter_node = kernel_name == "InterNodeV1"
         self.num_qps = 1
         self.block_num, self.rdma_block_num, self.dispatch_warps, self.combine_warps = blocks
-        # External input buffer on every path, matching what the engines run: vLLM leaves
-        # `use_external_inp_buf` at MoRI's default of True and SGLang sets it True explicitly, so
-        # registered-buffer (zero-copy) mode is a configuration no production engine selects. It
-        # also has to move together with `combine_warps`: MoRI's own tuned tables key combine on
-        # `zero_copy`, picking ~16 warps for external input against 4-8 for registered, so pinning
-        # the engines' 16 warps while staying registered matched neither. Measured cost of that
-        # mismatch, warps 16 vs 8 in registered mode on three chips: +13-18% combine at T=128 and
-        # +61-78% at T=512, correct in both arms. The zero-copy branches below are kept so the
-        # mode remains a one-line A/B rather than an archaeology exercise.
+        # External input buffer everywhere, as the engines run it (vLLM leaves MoRI's default,
+        # SGLang sets it explicitly). It must move together with `combine_warps`: MoRI's tuned
+        # tables key combine on `zero_copy`, so the engines' 16 warps belong to this mode and
+        # 4-8 to the registered one. The registered branches below stay, to keep the mode a
+        # one-line A/B; methodology.md carries the measured cost of mismatching them.
         self._external_input = True
         # Registered-input MoRI copies expert output into a device-side symmetric buffer. External
         # input kernels consume the dispatch output directly, so their stage is not applicable.
@@ -308,12 +304,9 @@ def stage(self, p, h):
         if not isinstance(rows, int) or rows < 0 or rows > h.dispatch_output.size(0):
             raise RuntimeError("MoRI receive count was not validated before staging")
         if self._external_input:
-            # The external-input staging copy is bounded by the receive count, not by the buffer:
-            # `EpCombineIntraNodeKernel_body` loops `i < totalRecvTokenNum` over `inpTokenBuf`
-            # (intranode.hpp:542-560), so it never reads past `rows` and the padding it leaves
-            # behind is untouched. BF16 therefore hands over the dispatch output as-is, and FP8
-            # dequantizes only the filled rows -- casting the whole cap-sized plane here would be
-            # ~99.8% padding at T=1, the same waste the zero-copy branch below documents.
+            # The kernel's own staging copy is bounded by the receive count, not the buffer
+            # (`EpCombineIntraNodeKernel_body` loops `i < totalRecvTokenNum`, intranode.hpp:542),
+            # so it never reads past `rows` and only the filled rows need converting.
             h.combine_input = (
                 h.dispatch_output[:rows].to(torch.bfloat16) if self._fp8 else h.dispatch_output
             )
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 14fb91a8e..8f57e861c 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -906,13 +906,8 @@ collx_run_shard() {
       || { rm -f "$argv_file"; collx_die "case $ci produced no benchmark arguments"; }
     collx_log "EP${NGPUS}[$((ci + 1))/$expected_cases] $COLLX_BENCH"
     runtime_log="$(collx_private_log_path "runtime-c$(printf '%03d' "$ci")")"
-    # A hang guard, NOT a work budget: it must sit clear above the most expensive legitimate
-    # case, or it starts killing cases that are merely slow. It did. An h100 FP8 prefill EP16
-    # case wrote a complete, all-rungs-passed artifact 2s past a 900s deadline and was killed
-    # anyway, turning a good measurement into a red shard with correct data attached -- and FP8
-    # got slower on purpose when the quantize moved inside the timed dispatch (~233us vs ~126us
-    # per dispatch on GB300), so the heaviest FP8 prefill legs now sit right on that line.
-    # 1800 is what the AMD launcher already used; one shared number, no per-launcher drift.
+    # A hang guard, NOT a work budget: at 900 it killed FP8 prefill cases that had already
+    # written complete, all-rungs-passed artifacts. 1800 is what the AMD launcher already used.
     if ! timeout -k 30 "${COLLX_RUN_TIMEOUT:-1800}" \
       srun --jobid="$JOB_ID" --nodes="$NODES" \
       --ntasks="$NGPUS" --ntasks-per-node="$GPN" --chdir=/tmp \
diff --git a/experimental/CollectiveX/runtime/config.py b/experimental/CollectiveX/runtime/config.py
index c1bb3183a..083f790ce 100644
--- a/experimental/CollectiveX/runtime/config.py
+++ b/experimental/CollectiveX/runtime/config.py
@@ -128,12 +128,10 @@ def _emit_argv(case: dict, version: object, runner: str, ts: str, index: int) ->
     iters, trials, warmup = str(case["timing"]).split(":")
     for flag, value in (("--iters", iters), ("--trials", trials), ("--warmup", warmup)):
         argv += [flag, value]
-    # precision and mode are part of the filename so a cell's legs (distinct shards sharing
-    # runner/backend/phase and each numbering cases from index 0) cannot collide when they land
-    # in the shared results/ dir under the same second-resolution ts. Mode matters as much as
-    # precision: CI gives every leg its own job and therefore its own ts, but anyone driving
-    # several shards from one loop shares it, and a low-latency case silently overwrote the
-    # normal case with the same precision, phase and index -- losing artifacts with no error.
+    # precision and mode are in the filename so a cell's legs -- distinct shards sharing
+    # runner/backend/phase, each numbering cases from index 0 -- cannot overwrite each other in
+    # the shared results/ dir when they share a second-resolution ts, as they do whenever
+    # several shards are driven from one loop rather than one CI job each.
     out = (
         f"results/{runner}_{case['backend']}_{case['precision']}_{case['mode']}_{case['phase']}"
         f"_{ts}-c{index:03d}.json"

From b4f4e39576071b87abeb1eaca8d667653825dbdf Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 20:43:48 +0800
Subject: [PATCH 18/34] =?UTF-8?q?CollectiveX:=20record=20what=20the=20warp?=
 =?UTF-8?q?s-and-buffer=20pairing=20actually=20measures=20/=20CollectiveX?=
 =?UTF-8?q?=EF=BC=9A=E8=AE=B0=E5=BD=95=20warps=20=E4=B8=8E=E7=BC=93?=
 =?UTF-8?q?=E5=86=B2=E6=A8=A1=E5=BC=8F=E9=85=8D=E5=AF=B9=E7=9A=84=E5=AE=9E?=
 =?UTF-8?q?=E6=B5=8B=E7=BB=93=E6=9E=9C?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The doc carried only half the evidence: the cost of 16 warps in the WRONG buffer mode. On-metal A/Bs
across all three CDNA chips now cover the mode the engines actually run, and the sign flips.

In external-input mode, 16 warps beats 8 by 14-19% combine at T=128, 26-27% at T=256, and 9-14% at
every prefill rung including T=8192 — the true top of the ladder, which a decode-only A/B would have
missed entirely. MI355X reads a tie at T=512 where both gfx942 chips keep a 5-7% edge, so the T=512
tie is a local quirk rather than the win petering out. Below T=32, 16 gives up 0.2-2.5us; that is the
only range where 8 is ahead. Every arm was correct at every rung on every chip, so this is a
throughput question and not a correctness one.

Two smaller corrections from the same runs:

- BF16's `stage` component is not merely "declared with no percentiles" but an explicit unavailable
  marker: availability unavailable, null percentile block, zero sample count. Stated that way now,
  because a consumer distinguishing "absent" from "present but empty" needs to know which it is.
- How far the engine config sits off MoRI's achievable peak is arch-dependent and only partly
  measured. MI355X, with the registered mode's excluded BF16 stage added back so the cross-mode
  comparison is not flattered, puts a registered buffer at 8 warps 15% ahead at T=512 decode and 8%
  ahead at T=8192 prefill. That did not reproduce as a clear win on gfx942, so the doc gives 0-15% as
  a range rather than a single number, and says plainly that the faster configuration is one no engine
  runs.

No code change. 91 tests pass.

中文：文档此前只记录了一半证据：16 warps 在**错误**缓冲模式下的代价。现在三款 CDNA 芯片的实机 A/B
已覆盖引擎真正使用的模式，结论符号发生反转。

在外部输入缓冲模式下，16 warps 的 combine 在 T=128 领先 8 约 14-19%，T=256 领先 26-27%，并在包括
T=8192（阶梯真正的顶端，仅测 decode 的 A/B 会完全遗漏该点）在内的每个 prefill 档位领先 9-14%。
MI355X 在 T=512 呈现持平，而两款 gfx942 芯片仍保持 5-7% 优势，因此 T=512 的持平属于局部特例，而非
优势逐渐消失。在 T<32 时 16 会损失 0.2-2.5us，这是 8 唯一领先的区间。所有芯片、所有档位、所有分组
的正确性均通过，因此这是吞吐问题而非正确性问题。

同批运行带来的两处小修正：

- BF16 的 `stage` 组件并非仅"已声明但无百分位"，而是显式的不可用标记：availability 为 unavailable、
  百分位块为 null、样本数为 0。现按此表述，因为需要区分"缺失"与"存在但为空"的消费方必须知道是哪一种。
- 引擎配置距 MoRI 可达峰值有多远，取决于架构且仅部分测得。在 MI355X 上，将注册缓冲模式被排除的 BF16
  stage 计回以避免跨模式比较失真后，注册缓冲配 8 warps 在 T=512 decode 领先 15%、在 T=8192 prefill
  领先 8%。该结果在 gfx942 上未复现为明确优势，因此文档给出 0-15% 的区间而非单一数值，并明确说明更快
  的那个配置是没有任何引擎会采用的配置。

无代码改动。91 项测试通过。
---
 experimental/CollectiveX/docs/methodology.md | 32 +++++++++++++-------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 3017727a4..ba8df12c0 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -88,17 +88,27 @@ combine alike (neither passes a per-call override, so combine inherits the 16).
 external input buffer, which is MoRI's default and which SGLang sets explicitly. Those two settings
 are pinned together deliberately: MoRI's own tuning tables key combine on `zero_copy`, selecting
 roughly 16 warps for external input against 4-8 for a registered buffer, so taking the engines' warp
-count while keeping a registered buffer would match neither. The cost of that mismatch is not
-hypothetical -- measured across MI300X, MI325X and MI355X, 16 warps in registered-buffer mode is
-+13-18% combine at T=128 and +61-78% at T=512 against 8, correct in both arms. With an external
-input buffer the kernel does its own staging copy, bounded by the receive count, so BF16 rows hand
-over the dispatch output unchanged: their `stage` component is still declared but carries no
-percentiles, the same way any backend whose staging is a bare pointer assignment reports it. FP8 rows
-still stage for real, because the received payload has to be dequantized. These numbers therefore describe the
-engine-integrated configuration, not MoRI's peak: its shipped tuning tables reach a faster combine
-with per-shape block and warp counts no engine selects, and AUTO would not reproduce those tables
-anyway (there is no BF16 gfx950 dispatch rule and no gfx950 IntraNodeLL combine table, so AUTO falls
-back to hard-coded defaults and couples the result to whichever MoRI revision is pinned). One
+count while keeping a registered buffer would match neither, and the mismatch is not hypothetical in
+either direction. Measured on MI300X, MI325X and MI355X: **in registered-buffer mode** 16 warps costs
++13-18% combine at T=128 and +61-78% at T=512 against 8, while **in the external-input mode the
+engines actually run** the same 16 warps *wins* — 14-19% at T=128, 26-27% at T=256, and 9-14% at every
+prefill rung including T=8192, the true top of the ladder. Below T=32 it gives up 0.2-2.5us, which is
+the only rung range where 8 is ahead. All arms were correct at every rung, so the pairing is a
+throughput question, not a correctness one. With an external input buffer the kernel does its own
+staging copy, bounded by the receive count, so BF16 rows hand over the dispatch output unchanged:
+their `stage` component is still declared, as an explicit unavailable marker with a null percentile
+block and a zero sample count, the same way any backend whose staging is a bare pointer assignment
+reports it. FP8 rows still stage for real, because the received payload has to be dequantized. These
+numbers therefore describe the engine-integrated configuration, not MoRI's peak: its shipped tuning
+tables reach a faster combine with per-shape block and warp counts no engine selects, and AUTO would
+not reproduce those tables anyway (there is no BF16 gfx950 dispatch rule and no gfx950 IntraNodeLL
+combine table, so AUTO falls back to hard-coded defaults and couples the result to whichever MoRI
+revision is pinned). How far off peak is arch-dependent and only partly measured: comparing across
+buffer modes on MI355X, with the registered mode's excluded BF16 stage added back so the comparison is
+not flattered, a registered buffer at 8 warps is still 15% faster at T=512 decode and 8% at T=8192
+prefill than the shipped pairing. That margin did not reproduce as a clear win on gfx942, so treat
+0-15% as the honest range rather than a single figure -- and note that the faster configuration is one
+no engine runs, which is why this suite does not chase it. One
 asymmetry is worth stating: the low-latency arm has no engine-integrated configuration to match at
 all, because SGLang's low-latency path pins `AsyncLL` at 8 warps while this suite uses `IntraNodeLL`
 (`AsyncLL` is split-phase and fails silently under a single-call harness), so the low-latency launch

From 7314f9fcbd3c6629de4e90143ca926b6f0265f81 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 22:27:08 +0800
Subject: [PATCH 19/34] =?UTF-8?q?CollectiveX:=20exclude=20the=20B200=20nod?=
 =?UTF-8?q?e=20with=20a=20thermally-clamped=20GPU=20/=20CollectiveX?=
 =?UTF-8?q?=EF=BC=9A=E6=8E=92=E9=99=A4=E5=AD=98=E5=9C=A8=20GPU=20=E6=B8=A9?=
 =?UTF-8?q?=E5=BA=A6=E9=99=8D=E9=A2=91=E6=95=85=E9=9A=9C=E7=9A=84=20B200?=
 =?UTF-8?q?=20=E8=8A=82=E7=82=B9?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`b200-dgxc-deepep-v2-ll-fp8-n1` was killed by the per-case hang guard twice, at 900s and again at
1800s, and looked like a genuine wall-clock pathology in that backend and precision. It is one bad
GPU. Sampled from inside an allocation on gpu-2-6:

    index, clocks.sm, temperature.gpu, power.draw, clocks_event_reasons.active
    0, 1965 MHz, 34, 269.15 W, 0x0
    ...
    7,  120 MHz, 93, 302.26 W, 0x20        <- SW Thermal Slowdown

GPU 7 sits at 120 MHz and 93 C while its seven siblings run 1965 MHz at 34-39 C, across 23 samples
over four minutes. 93 C at ~294 W on a B200 is a cooling or sensor fault, not a workload effect.

Every EP collective is a barrier across all eight ranks, so the leg runs at rank 7's pace: 3.4-7.8s
per trial against 0.445s on a healthy node, projecting ~34 min — which is the 32 min that got killed.
Both CI kills landed on gpu-2-6; the run that passed on 2026-08-02 was on gpu-2-9.

True case time on a healthy node is fp8 2m39s and bf16 1m04s, a ratio of 2.5x. The "10x" that made
this look pathological came from dividing a kill by a pass, which is not a ratio — a killed case has
no duration, only a lower bound.

Excluding the node follows the h100-dgxc precedent in this file, which already drops nine unhealthy
pods. This is a workaround, not a repair: gpu-2-6 needs a Slurm drain and a hardware/cooling check on
its GPU 7, and the exclusion should come back out once that is done.

Note the exclusion does NOT rescue an in-flight sweep, which uses the config frozen at dispatch.

Falsified along the way, so nobody re-runs them: a recompile loop (dynamo counters total=5 ok=5), an
expensive fp8 oracle (oracle_pre 3.29s at T=1, <=0.01s at every other rung), a regression in the
`_ll_dequant_static` compile (stage p50 230-247us throughout), one dominant rung (Pass 2 flat, all 36
rung-by-component cells 2.75-3.85s), and memory (flat at 10.18 GiB/GPU, no OOM).

中文：`b200-dgxc-deepep-v2-ll-fp8-n1` 曾两次被单用例挂起保护杀掉（900 秒与 1800 秒），看起来像该后端
与精度组合下真实存在的墙钟时间病态问题。实际原因是一块故障 GPU。在 gpu-2-6 的分配内采样（见上表）：
GPU 7 停在 120 MHz、93 C，而其余七块运行在 1965 MHz、34-39 C，四分钟内 23 次采样均如此。B200 在约
294 W 下达到 93 C，属于散热或传感器故障，而非负载所致。

每次 EP 集合通信都是跨全部八个 rank 的屏障，因此整条 leg 以 rank 7 的速度运行：每个 trial 为
3.4-7.8 秒，而健康节点为 0.445 秒，据此推算约 34 分钟——正是被杀掉的那 32 分钟。两次 CI 被杀都发生在
gpu-2-6；2026-08-02 通过的那次运行在 gpu-2-9。

健康节点上的真实用例时间为 fp8 2 分 39 秒、bf16 1 分 04 秒，比值 2.5 倍。让人误以为存在病态问题的
"10 倍"来自用一次被杀的时长除以一次通过的时长，而这并不构成比值——被杀的用例没有时长，只有下界。

排除该节点沿用本文件中 h100-dgxc 的既有做法（其已排除九个不健康 pod）。这是权宜之计而非修复：
gpu-2-6 需要 Slurm drain 并对其 GPU 7 做硬件/散热检查，修好后应移除该排除项。

注意：该排除**不会**挽救正在运行中的扫描，后者使用派发时冻结的配置。
---
 experimental/CollectiveX/configs/platform_config.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/experimental/CollectiveX/configs/platform_config.json b/experimental/CollectiveX/configs/platform_config.json
index 52de60ba9..14b323fb2 100644
--- a/experimental/CollectiveX/configs/platform_config.json
+++ b/experimental/CollectiveX/configs/platform_config.json
@@ -61,7 +61,8 @@
         "partition": "gpu-2",
         "account": "benchmark",
         "qos": "gpu-2_qos",
-        "squash_dir": "/home/sa-shared/containers"
+        "squash_dir": "/home/sa-shared/containers",
+        "exclude_nodes": "gpu-2-6"
       },
       "network": {
         "rdma_devices": "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7",

From 161c3196169ad2493f02326703f27d881994047d Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 22:44:56 +0800
Subject: [PATCH 20/34] =?UTF-8?q?CollectiveX:=20call=20FP8=20staging=20wha?=
 =?UTF-8?q?t=20it=20is,=20and=20stop=20rehearsing=20it=20in=20warm-up=20/?=
 =?UTF-8?q?=20CollectiveX=EF=BC=9A=E6=98=8E=E7=A1=AE=20FP8=20staging=20?=
 =?UTF-8?q?=E7=9A=84=E6=80=A7=E8=B4=A8=EF=BC=8C=E5=B9=B6=E5=81=9C=E6=AD=A2?=
 =?UTF-8?q?=E5=9C=A8=E9=A2=84=E7=83=AD=E4=B8=AD=E9=87=8D=E5=A4=8D=E6=89=A7?=
 =?UTF-8?q?=E8=A1=8C?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two changes, both from the same realisation: under FP8, `stage` is not a phase a serving stack has.
Its work is converting the received FP8 payload into the BF16 that combine sends, and production does
not do that as a separate step -- the FP8 goes into the expert GEMM, which reads FP8 operands natively
and emits BF16, and that output is what combine receives. This suite does not run the expert GEMM
because it measures the collective and not the layer, so `stage` stands in for it.

1. The doc now says that plainly, and adds the consequence that matters to a consumer: `stage` must
   not be summed into a total or compared between backends, because each adapter converts a different
   amount -- DeepEP V2 a padded low-latency plane, MoRI only the received rows, FlashInfer only the
   filled slots. One component name, three different quantities. It also names the one production path
   that DOES pay a separate materialised dequant (vLLM's quant-format mismatch fallback), which is
   what `CX_FP8_CONSUME=dequant` models and why that is not the default.

2. `warm()` no longer stages on every iteration. Where staging is excluded from the chain, the timed
   roundtrip stages nothing at all, so warming it 32 times per component per trial rehearsed a path
   the measurement never takes -- and at ~247us against a 61us roundtrip that was the largest single
   cost in an FP8 leg, roughly 34,816 dequant calls per rung. It now stages once and reuses the
   payload, which is exactly what `benchmark_roundtrip` already does in the timed path on the same
   documented reasoning ("routing is fixed for a ladder point, so the same staged tensor is valid for
   every iteration"). So warm-up now resembles the region it warms instead of diverging from it.

   `benchmark_stage` opts back in via `stage_every=True`, because there staging IS the timed operation
   and starving its warm-up would change the number it reports. The `dequant` hatch keeps staging every
   iteration too, since that configuration genuinely has the conversion in the chain.

   Expected saving is about 73% of the dequant calls per rung; the remainder is `benchmark_stage`'s own
   warm-up plus the untimed `pre` that has to feed each timed combine. No published number changes --
   `stage` was already excluded from `roundtrip` -- so this is CI wall-clock only.

Three tests pin the hoist, and they discriminate: reverting it fails them with `5 != 1` rather than
passing quietly, which is the failure mode a staging change of this shape usually has. They stub torch
so they run without a GPU. 94 tests pass.

中文：两项改动源于同一认识：在 FP8 下，`stage` 并非服务栈中真实存在的阶段。它的工作是把收到的 FP8
载荷转换为 combine 所发送的 BF16，而生产环境并不把这作为独立步骤——FP8 直接进入 expert GEMM，后者
原生读取 FP8 操作数并输出 BF16，该输出才是 combine 收到的内容。本套件不运行 expert GEMM（它测量的是
集合通信而非整层），因此 `stage` 只是其替身。

1. 文档现已明确说明这一点，并补充了对消费方最重要的推论：`stage` 不可计入总和，也不可在后端之间比较，
   因为各适配器转换的数据量并不相同——DeepEP V2 为补齐后的低延迟平面，MoRI 仅为已接收的行，
   FlashInfer 仅为已填充的槽位。同一个组件名对应三种不同的量。文档同时指出确实会付出独立物化反量化
   代价的那条生产路径（vLLM 的量化格式不匹配回退），这正是 `CX_FP8_CONSUME=dequant` 所建模的情形，
   也是它并非默认值的原因。

2. `warm()` 不再在每次迭代都执行 staging。当 staging 被排除在链路之外时，计时的 roundtrip 完全不做
   staging，因此按每组件每 trial 预热 32 次去重复该路径，等于预热了测量根本不会走的路径——而在约
   247us 对 61us roundtrip 的量级下，这是 FP8leg 中最大的单项开销，每档位约 34,816 次反量化调用。
   现改为 staging 一次并复用其载荷，这与 `benchmark_roundtrip` 在计时路径中早已采用的做法完全一致
   （"给定阶梯点的路由是固定的，因此同一 staged 张量对每次迭代都有效"）。预热由此变得与其所预热的
   区域相似，而非与之背离。

   `benchmark_stage` 通过 `stage_every=True` 重新启用逐次 staging，因为在该处 staging 就是被计时的
   操作，削减其预热会改变它上报的数值。`dequant` 旁路同样保持逐次 staging，因为该配置下转换确实位于
   链路内。

   预计可减少每档位约 73% 的反量化调用；其余部分为 `benchmark_stage` 自身的预热，以及必须为每次计时
   combine 供数的未计时 `pre`。不改变任何已发布指标——`stage` 本就已被排除在 `roundtrip` 之外——因此
   这仅影响 CI 的墙钟时间。

三项测试锁定该 hoist，且具备区分能力：回退该改动会以 `5 != 1` 失败而非静默通过，而后者正是此类
staging 改动通常的失效模式。测试对 torch 做了打桩，因此无需 GPU 即可运行。94 项测试通过。
---
 experimental/CollectiveX/bench/ep_backend.py  | 21 +++++++--
 experimental/CollectiveX/docs/methodology.md  | 13 ++++++
 .../tests/test_roundtrip_staging.py           | 44 +++++++++++++++++++
 3 files changed, 75 insertions(+), 3 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_backend.py b/experimental/CollectiveX/bench/ep_backend.py
index 7a71e683f..e5d82dca9 100644
--- a/experimental/CollectiveX/bench/ep_backend.py
+++ b/experimental/CollectiveX/bench/ep_backend.py
@@ -396,19 +396,33 @@ def timed_components(self):
                 components.append("stage")
         return components
 
-    def warm(self, problem, count):
+    def warm(self, problem, count, stage_every=False):
         """Untimed synchronized full round trips (fabric/clock warm-up; cold-jump-safe).
 
         Caches the dynamic receive cardinality once so adapters never read a device
         scalar during a timed trial (the count is stable for a fixed routing trace).
+
+        `stage_every` re-materialises the combine input on every iteration. The default hoists it
+        after the first, mirroring `benchmark_roundtrip` on the same reasoning: where staging is
+        excluded from the chain the timed region stages nothing at all, so staging once per warm
+        iteration warms work the measurement never performs. For an FP8 dequant that is not free --
+        it is ~247us against a 61us roundtrip, and at 32 warm iterations per component per trial it
+        was the largest single cost in the leg. `benchmark_stage` opts in, because there staging IS
+        the timed operation and its warm-up has to match.
         """
         import torch
 
+        staged = None
         for _ in range(count):
             handle = self.dispatch(problem)
             if not hasattr(problem, "recv_tokens"):
                 problem.recv_tokens = self.recv_tokens(handle)
-            self.stage(problem, handle)
+            if staged is None:
+                self.stage(problem, handle)
+                if not stage_every and self.stage_excluded_from_roundtrip:
+                    staged = getattr(handle, self.combine_input_attr)
+            else:
+                setattr(handle, self.combine_input_attr, staged)
             self.combine(problem, handle)
             torch.cuda.synchronize()
 
@@ -483,7 +497,8 @@ def finish_dispatch(hh, p=problem):
     def benchmark_stage(self, problem, warmup, iters):
         import torch
 
-        self.warm(problem, warmup)
+        # Staging is the timed operation here, so it must be warmed on every iteration.
+        self.warm(problem, warmup, stage_every=True)
 
         def prep_stage(p=problem):
             return self.dispatch(p)
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index ba8df12c0..51525ed2a 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -175,6 +175,19 @@ dispatch then combine — the transport — in every row. It is reported as its
 wherever it does device work. The one exception is the `CX_FP8_CONSUME=dequant` verification hatch,
 which puts the conversion back inside the chain on purpose.
 
+Under FP8, treat `stage` as **harness scaffolding rather than a phase a serving stack has**. Its work
+is converting the received FP8 payload to the BF16 that combine sends, and in production nothing does
+that as a separate step: the FP8 lands in the expert GEMM, which reads FP8 operands natively and emits
+BF16, and that GEMM output is what combine receives. This suite deliberately does not run the expert
+GEMM — it measures the collective, not the layer — so `stage` stands in for it. That is why `stage` is
+excluded from `roundtrip`, and it is also why **`stage` must not be summed into a total or compared
+between backends**: each adapter converts a different amount (DeepEP V2 a padded low-latency plane,
+MoRI only the received rows, FlashInfer only the filled slots), so the same component name covers
+three different quantities. The one production path that *does* pay a separate materialised dequant is
+a quant-format mismatch fallback (vLLM dequantises when `block_k` disagrees with DeepEP's block size);
+`CX_FP8_CONSUME=dequant` exists to model exactly that case, and it is not the default because it is
+not the fast path.
+
 Read `implementation.stage_excluded_from_roundtrip` as "there was device-work staging and it was
 hoisted out of the chain", not as "this row's roundtrip is stage-free". It is gated on whether the
 backend's `stage()` does device work at all, so it is `false` in two unrelated situations, and the
diff --git a/experimental/CollectiveX/tests/test_roundtrip_staging.py b/experimental/CollectiveX/tests/test_roundtrip_staging.py
index 3906cbbd7..1c5d04f6a 100644
--- a/experimental/CollectiveX/tests/test_roundtrip_staging.py
+++ b/experimental/CollectiveX/tests/test_roundtrip_staging.py
@@ -151,3 +151,47 @@ def test_the_hatch_does_not_apply_to_bf16(self):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class WarmStaging(unittest.TestCase):
+    """Warm-up must not rehearse work the timed region skips.
+
+    Where staging is excluded from the chain, the timed roundtrip stages nothing, so staging on
+    every warm iteration warms a path the measurement never takes. For an FP8 dequant that was
+    the largest single cost in the leg (~247us x 32 iterations x every component x every trial).
+    `benchmark_stage` is the exception: staging is its timed operation.
+    """
+
+    @staticmethod
+    def _warm(backend, count, **kwargs):
+        # `warm` imports torch for one synchronize; a stub keeps this runnable without a GPU.
+        fake = types.ModuleType("torch")
+        fake.cuda = types.SimpleNamespace(synchronize=lambda: None)
+        saved = sys.modules.get("torch")
+        sys.modules["torch"] = fake
+        try:
+            backend.warm(types.SimpleNamespace(), count, **kwargs)
+        finally:
+            if saved is None:
+                del sys.modules["torch"]
+            else:
+                sys.modules["torch"] = saved
+
+    def test_stages_once_when_the_chain_excludes_staging(self):
+        b = _StubBackend(stage_device_work=True, fp8_consume="native")
+        self._warm(b, 5)
+        self.assertEqual(b.calls.count("dispatch"), 5)
+        self.assertEqual(b.calls.count("stage"), 1)
+        # Every later iteration still hands combine the staged payload, not a stale None.
+        self.assertEqual(b.calls.count("combine(staged-by-stage)"), 5)
+
+    def test_stage_every_rehearses_it_on_every_iteration(self):
+        b = _StubBackend(stage_device_work=True, fp8_consume="native")
+        self._warm(b, 5, stage_every=True)
+        self.assertEqual(b.calls.count("stage"), 5)
+
+    def test_a_chain_that_includes_staging_keeps_warming_it(self):
+        # The `dequant` hatch puts the conversion back in the timed chain, so warm-up must match.
+        b = _StubBackend(stage_device_work=True, fp8_consume="dequant")
+        self._warm(b, 5)
+        self.assertEqual(b.calls.count("stage"), 5)

From 151b0077c2233a55af537056f118ef91394a9012 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 3 Aug 2026 22:53:21 +0800
Subject: [PATCH 21/34] =?UTF-8?q?CollectiveX:=20reject=20an=20allocation?=
 =?UTF-8?q?=20holding=20a=20throttled=20GPU=20/=20CollectiveX=EF=BC=9A?=
 =?UTF-8?q?=E6=8B=92=E7=BB=9D=E5=8C=85=E5=90=AB=E9=99=8D=E9=A2=91=20GPU=20?=
 =?UTF-8?q?=E7=9A=84=E8=B5=84=E6=BA=90=E5=88=86=E9=85=8D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A B200 node had GPU 7 pinned at 120 MHz and 93 C while its seven siblings ran 1965 MHz at 34-39 C,
with both SW and HW thermal slowdown Active and a cumulative slowdown counter of 55.5 hours. Its
T.Limit readings were NEGATIVE, i.e. above both the slowdown and the shutdown threshold.

Every collective is a barrier across all ranks, so that one device paced the entire leg: 7.1-9.0s per
trial against 0.445s on a healthy node. The plateau ratio of 17.4x tracks the clock ratio of 16.4x
almost exactly. The case was killed by the wall-clock guard twice — at 900s and again after the guard
was raised to 1800s — having printed no rungs and produced no artifact, which is indistinguishable
from a hang in that backend and precision. Its real cost is 2m39s.

The launcher already validates each allocation (network profile everywhere, CUDA context on b300) and
already excludes a rejected node and retries elsewhere. This adds a GPU health check to that chain,
so the same fault costs seconds and one retry instead of thirty minutes and a misdiagnosis. It is
retryable on every SKU: a throttled device is never this leg's to tolerate.

Design notes, both learned from the incident:

- **The throttle FLAG is the signal, not the clock.** This runs while the allocation is idle, and an
  idle B200 also reads 120 MHz — the same number the clamped one reports under load. A clock
  threshold cannot separate health from idleness; the flag can.
- **Temperature is a second, independent signal**, because the flag can clear between samples while
  the fault persists.
- **It fails OPEN** on a missing `nvidia-smi`, a non-zero exit, or unparseable output. A check that
  blocks legs when it cannot read the hardware is worse than the fault it looks for.

Parsing is split from the I/O (`gpu_health_faults`) so it is testable without hardware. Six tests
cover healthy, both flags, either flag alone, heat with no flag, and four unreadable shapes. One of
them pins the specific trap that "Active" as a substring also matches "Not Active" and would pass
every fault straight through.

Wired into `launch_single-slurm.sh`, which covers h100/h200/b200/b300. `launch_gb-nv.sh` is a
one-line follow-up if wanted; the AMD launchers would need the rocm-smi equivalent.

100 tests pass.

中文：某 B200 节点的 GPU 7 被固定在 120 MHz、93 C，而其余七块运行在 1965 MHz、34-39 C，SW 与 HW 温度
降频均为 Active，累计降频计数达 55.5 小时。其 T.Limit 读数为**负值**，即已超过降频与关机两个阈值。

每次集合通信都是跨全部 rank 的屏障，因此这一块设备决定了整条 leg 的节奏：每 trial 7.1-9.0 秒，而健康
节点为 0.445 秒。17.4 倍的平台期比值与 16.4 倍的时钟比值几乎完全吻合。该用例被墙钟保护杀掉两次——先是
900 秒，随后在保护放宽至 1800 秒后再次被杀——且未打印任何档位、未产出任何产物，这与该后端与精度下的
挂起完全无法区分。其真实耗时为 2 分 39 秒。

launcher 本就会校验每次分配（各处校验网络画像，b300 额外校验 CUDA 上下文），并且本就会排除被拒节点并
在别处重试。本次在该链路中加入 GPU 健康检查，使同类故障的代价从三十分钟加一次误诊，变为数秒加一次
重试。该项在所有 SKU 上均可重试：降频设备从不属于本条 leg 应当容忍的范围。

两点设计取舍均来自本次事件：

- **信号是降频标志而非时钟。** 该检查在分配处于空闲时运行，而空闲的 B200 同样读到 120 MHz——与被钳制
  的那块在负载下报告的数值相同。时钟阈值无法区分健康与空闲，标志可以。
- **温度作为第二个独立信号**，因为标志可能在两次采样之间清除而故障仍然存在。
- **失败时放行**：`nvidia-smi` 缺失、非零退出或输出无法解析时均视为健康。一个在读不到硬件时就阻塞
  leg 的检查，比它所要寻找的故障更糟。

解析逻辑已与 I/O 分离（`gpu_health_faults`），因此无需硬件即可测试。六项测试覆盖健康、双标志、单标志、
仅高温无标志，以及四种无法解析的形态。其中一项专门锁定一个陷阱：把 "Active" 当作子串匹配会同时命中
"Not Active"，从而让所有故障直接通过。

已接入 `launch_single-slurm.sh`（覆盖 h100/h200/b200/b300）。`launch_gb-nv.sh` 如需接入只是一行改动；
AMD 的 launcher 需要 rocm-smi 的对应实现。

100 项测试通过。
---
 .../launchers/launch_single-slurm.sh          |  9 +++
 experimental/CollectiveX/runtime/common.sh    | 15 ++++
 experimental/CollectiveX/runtime/probe.py     | 68 +++++++++++++++++++
 .../CollectiveX/tests/test_runtime.py         | 49 +++++++++++++
 4 files changed, 141 insertions(+)

diff --git a/experimental/CollectiveX/launchers/launch_single-slurm.sh b/experimental/CollectiveX/launchers/launch_single-slurm.sh
index f30eef42e..6e3215eab 100644
--- a/experimental/CollectiveX/launchers/launch_single-slurm.sh
+++ b/experimental/CollectiveX/launchers/launch_single-slurm.sh
@@ -114,17 +114,26 @@ for allocation_attempt in 1 2 3; do
   elif [ "$RUNNER" = b300 ] \
       && ! collx_validate_cuda_context_on_job "$JOB_ID" "$NODES" "$GPN"; then
     validation_failure=cuda-context
+  elif ! collx_validate_gpu_health_on_job "$JOB_ID" "$NODES"; then
+    validation_failure=gpu-health
   else
     break
   fi
   retryable=0
   [ "$RUNNER:$validation_failure" != h100-dgxc:network ] || retryable=1
   [ "$RUNNER:$validation_failure" != b300:cuda-context ] || retryable=1
+  # A throttled GPU is always someone else's node to fix, never this leg's to tolerate: one
+  # clamped device paces every rank, so retrying elsewhere is right on every SKU.
+  [ "$validation_failure" != gpu-health ] || retryable=1
   if [ "$retryable" = 0 ] || [ "$allocation_attempt" = 3 ]; then
     if [ "$validation_failure" = network ]; then
       collx_log_tail "${COLLX_NETWORK_PROFILE_LOG:-}"
       collx_die "allocated nodes failed the network profile"
     fi
+    if [ "$validation_failure" = gpu-health ]; then
+      collx_log_tail "${COLLX_GPU_HEALTH_LOG:-}"
+      collx_die "allocated nodes hold a thermally throttled GPU"
+    fi
     collx_log_tail "$COLLX_CUDA_CONTEXT_LOG"
     collx_die "allocated nodes failed accelerator context validation"
   fi
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 8f57e861c..5452c118c 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -763,6 +763,21 @@ BASH
 # A clean nvidia-smi inventory does not prove that a prior cancelled workload
 # released every CUDA context. Retaining each primary context catches poisoned
 # allocations before a full shard spends time failing every case.
+collx_validate_gpu_health_on_job() {
+  local job_id="$1" nodes="$2" log_label=gpu-health log
+  case "${COLLX_SALLOC_ATTEMPT:-1}" in
+    1) ;;
+    2|3) log_label+="-a${COLLX_SALLOC_ATTEMPT}" ;;
+    *) return 1 ;;
+  esac
+  log="$(collx_private_log_path "$log_label")"
+  export COLLX_GPU_HEALTH_LOG="$log"
+  srun --jobid="$job_id" --nodes="$nodes" --ntasks="$nodes" --ntasks-per-node=1 \
+    --chdir=/tmp --input=all \
+    --export="$(collx_host_exports)" python3 /dev/stdin gpu-health \
+    < "$COLLX_RUNTIME_DIR/probe.py" >"$log" 2>&1
+}
+
 collx_validate_cuda_context_on_job() {
   local job_id="$1" nodes="$2" gpus_per_node="$3" log_label=cuda-context log
   case "${COLLX_SALLOC_ATTEMPT:-1}" in
diff --git a/experimental/CollectiveX/runtime/probe.py b/experimental/CollectiveX/runtime/probe.py
index 922a99682..9f1ccaa21 100644
--- a/experimental/CollectiveX/runtime/probe.py
+++ b/experimental/CollectiveX/runtime/probe.py
@@ -31,6 +31,72 @@ def validate_cuda_context(expected: int) -> None:
         raise SystemExit(1)
 
 
+_GPU_HEALTH_FIELDS = ("index", "clocks_event_reasons.sw_thermal_slowdown",
+                      "clocks_event_reasons.hw_thermal_slowdown", "temperature.gpu")
+
+
+def gpu_health_faults(output: str, max_temperature_c: int = 90) -> list[str]:
+    """Throttled or overheating GPUs in an `nvidia-smi --format=csv,noheader` block.
+
+    Split out from the I/O so the parsing is testable without hardware; see
+    tests/test_runtime.py::GpuHealthProbe. Returns [] for anything it cannot read, because the
+    caller treats an unreadable probe as healthy rather than blocking a leg on it.
+    """
+    faults = []
+    for line in output.splitlines():
+        cells = [cell.strip() for cell in line.split(",")]
+        if len(cells) != len(_GPU_HEALTH_FIELDS):
+            continue
+        index, software, hardware, temperature = cells
+        # "Not Active" is the healthy reading, so compare exactly rather than searching for
+        # "Active" -- a substring test passes the fault straight through.
+        throttled = "Active" in (software, hardware)
+        try:
+            too_hot = int(temperature.split()[0]) > max_temperature_c
+        except (IndexError, ValueError):
+            too_hot = False
+        if throttled or too_hot:
+            faults.append(
+                f"gpu {index}: sw_thermal={software} hw_thermal={hardware} temp={temperature}"
+            )
+    return faults
+
+
+def validate_gpu_health(max_temperature_c: int = 90) -> None:
+    """Reject an allocation holding a thermally throttled GPU.
+
+    Every collective is a barrier across all ranks, so one clamped device paces the whole leg. A
+    B200 with GPU 7 held at 120 MHz against 1965 MHz on its siblings ran a case 17x slower and was
+    killed by the wall-clock guard twice, at 900s and again at 1800s, looking exactly like a code
+    pathology. Rejecting the allocation costs seconds; diagnosing it cost two 30-minute burns.
+
+    The signal is the throttle FLAG, not the clock: the allocation is idle at this point and an idle
+    B200 also reads 120 MHz, so a clock threshold cannot tell health from idleness. Temperature is a
+    second, independent signal because the flag can clear between samples while the fault persists.
+
+    Fails OPEN on anything unexpected -- no `nvidia-smi`, non-zero exit, unparseable output. A check
+    that blocks legs when it cannot read the hardware is worse than the fault it looks for.
+    """
+    import shutil
+    import subprocess
+
+    if shutil.which("nvidia-smi") is None:
+        return
+    try:
+        output = subprocess.run(
+            ["nvidia-smi", f"--query-gpu={','.join(_GPU_HEALTH_FIELDS)}",
+             "--format=csv,noheader"],
+            capture_output=True, text=True, timeout=60, check=True,
+        ).stdout
+    except (OSError, subprocess.SubprocessError):
+        return
+    faults = gpu_health_faults(output, max_temperature_c)
+    for fault in faults:
+        _emit(f"gpu-health-fault {fault}")
+    if faults:
+        raise SystemExit(1)
+
+
 def _emit(marker: str) -> None:
     # collx_validate_network_profile_on_job (runtime/common.sh) greps these exact strings
     # out of the per-node probe log to derive COLLX_SOCKET_IFNAME / COLLX_RDMA_LINK_LAYER and to
@@ -113,11 +179,13 @@ def main() -> None:
     commands.add_parser("default-route-interface")
     command = commands.add_parser("prepare-cache"); command.add_argument("parent")
     command = commands.add_parser("cuda-context"); command.add_argument("expected", type=int)
+    commands.add_parser("gpu-health")
     command = commands.add_parser("network-profile"); command.add_argument("socket_names"); command.add_argument("rdma_devices"); command.add_argument("gid_index")
     args = parser.parse_args()
     if args.command == "default-route-interface": print(default_route_interface(), end="")
     elif args.command == "prepare-cache": print(prepare_cache(args.parent), end="")
     elif args.command == "cuda-context": validate_cuda_context(args.expected)
+    elif args.command == "gpu-health": validate_gpu_health()
     else: validate_network_profile(args.socket_names, args.rdma_devices, args.gid_index)
 
 
diff --git a/experimental/CollectiveX/tests/test_runtime.py b/experimental/CollectiveX/tests/test_runtime.py
index 8bd1407cf..8fba43307 100644
--- a/experimental/CollectiveX/tests/test_runtime.py
+++ b/experimental/CollectiveX/tests/test_runtime.py
@@ -757,3 +757,52 @@ def shape_dependent(t):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class GpuHealthProbe(unittest.TestCase):
+    """Reject an allocation holding a throttled GPU before it burns the wall-clock guard.
+
+    Collectives are barriers, so one clamped device paces every rank. A B200 with GPU 7 held at
+    120 MHz against 1965 MHz on its siblings ran a case 17x slower and was killed twice, at 900s
+    and again at 1800s, looking exactly like a code pathology. The flag is the signal, not the
+    clock: the allocation is idle when this runs and an idle B200 also reads 120 MHz.
+    """
+
+    HEALTHY = "\n".join(f"{i}, Not Active, Not Active, 3{i} " for i in range(8))
+
+    def _swap(self, line_in: str, line_out: str) -> str:
+        self.assertIn(line_in, self.HEALTHY)  # guard the fixture against silent drift
+        return self.HEALTHY.replace(line_in, line_out)
+
+    def test_healthy_allocation_passes(self):
+        self.assertEqual(probe.gpu_health_faults(self.HEALTHY), [])
+
+    def test_a_thermally_throttled_gpu_is_rejected(self):
+        output = self._swap("7, Not Active, Not Active, 37 ", "7, Active, Active, 93 ")
+        faults = probe.gpu_health_faults(output)
+        self.assertEqual(len(faults), 1)
+        self.assertIn("gpu 7", faults[0])
+
+    def test_either_throttle_flag_alone_is_enough(self):
+        for cells in ("7, Active, Not Active, 88 ", "7, Not Active, Active, 88 "):
+            with self.subTest(cells=cells):
+                output = self._swap("7, Not Active, Not Active, 37 ", cells)
+                self.assertEqual(len(probe.gpu_health_faults(output)), 1)
+
+    def test_not_active_is_not_read_as_active(self):
+        # A substring search for "Active" matches "Not Active" and passes every fault through,
+        # which is the whole failure mode this probe exists to avoid.
+        self.assertEqual(probe.gpu_health_faults(self.HEALTHY), [])
+
+    def test_temperature_is_an_independent_signal(self):
+        # The flag can clear between samples while the fault persists, so heat alone rejects.
+        output = self._swap("3, Not Active, Not Active, 33 ", "3, Not Active, Not Active, 95 ")
+        faults = probe.gpu_health_faults(output)
+        self.assertEqual(len(faults), 1)
+        self.assertIn("gpu 3", faults[0])
+
+    def test_unreadable_output_fails_open(self):
+        # Blocking legs when the hardware cannot be read is worse than the fault being looked for.
+        for output in ("", "nonsense\n", "1, Not Active\n", self.HEALTHY.replace("32 ", "[N/A] ")):
+            with self.subTest(output=output[:20]):
+                self.assertEqual(probe.gpu_health_faults(output), [])

From bb11716b49162e27d132e9da65b277dd2dfd5280 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 4 Aug 2026 01:23:27 +0800
Subject: [PATCH 22/34] =?UTF-8?q?CollectiveX:=20record=20the=20library=20v?=
 =?UTF-8?q?ersion=20the=20combine=20model=20was=20chosen=20from=20/=20Coll?=
 =?UTF-8?q?ectiveX=EF=BC=9A=E8=AE=B0=E5=BD=95=E9=80=89=E5=AE=9A=20combine?=
 =?UTF-8?q?=20=E6=A8=A1=E5=9E=8B=E6=89=80=E4=BE=9D=E6=8D=AE=E7=9A=84?=
 =?UTF-8?q?=E5=BA=93=E7=89=88=E6=9C=AC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`implementation.combine_reduction` exists, by its own comment, so that "a wheel bump [does not]
silently change the arithmetic behind `passed` with no trace". It records the outcome but not the
input: a reader can see the oracle used the slot-tree model, but not which wheel selected it, and so
cannot distinguish a correct selection from a mis-parse of the version string.

That is not hypothetical for this backend. FlashInfer EP picks its reduction from
`flashinfer.__version__`, and the parse behind that choice was rewritten in 866833876 precisely
because the old one read `0.6.16rc1` as 0.6.16 and would have modelled FP32 against a kernel that
rounds per level. On-metal validation of the rewrite had to read the wheel string out of the image's
dist-info to check it, because the artifact does not carry it.

`library_version` is additive and opt-in via `getattr`, so it is None for the four backends that do
not report one; FlashInfer EP now sets it. No other backend's measurement SEMANTICS depend on its
library version today — for the rest a version change alters performance, not what the oracle models
— so this deliberately does not become a required field.

Verified on metal at a8c0411f8 before this commit: the shipped wheel is flashinfer_python-0.6.8.post1
and both EP16 BF16 artifacts recorded `combine_reduction=topk-slot-tree`, i.e. the rewritten gate
selects correctly on hardware. That check needed a dist-info read; after this commit the artifact
answers it alone.

100 tests pass.

中文：`implementation.combine_reduction` 之所以存在，按其自身注释所述，是为了让"wheel 版本变动不会在
无任何痕迹的情况下悄然改变 `passed` 背后的算术"。但它只记录了结果而未记录输入：读者能看到 oracle 使用
了 slot-tree 模型，却看不出是哪个 wheel 选中了它，因而无法区分正确选择与版本字符串的误解析。

对该后端而言这并非假设。FlashInfer EP 依据 `flashinfer.__version__` 选择其 reduction，而该选择背后的
解析逻辑正是在 866833876 中被重写的——原因恰恰是旧逻辑会把 `0.6.16rc1` 读成 0.6.16，从而对一个逐层
取整的 kernel 套用 FP32 模型。对该重写的实机验证不得不从镜像的 dist-info 中读取 wheel 字符串来核对，
因为产物本身并不携带它。

`library_version` 通过 `getattr` 实现为附加且可选字段，因此对未上报该值的四个后端为 None；FlashInfer
EP 现已设置该值。目前没有其他后端的测量**语义**依赖其库版本——对其余后端而言版本变化影响性能，而非
oracle 所建模的内容——因此本次有意不将其设为必填字段。

本提交前已在 a8c0411f8 于实机验证：所用 wheel 为 flashinfer_python-0.6.8.post1，且两个 EP16 BF16
产物均记录 `combine_reduction=topk-slot-tree`，即重写后的判定在硬件上选择正确。该核对此前需要读取
dist-info；本提交之后，产物本身即可回答该问题。

100 项测试通过。
---
 experimental/CollectiveX/bench/ep_flashinfer.py | 1 +
 experimental/CollectiveX/bench/ep_harness.py    | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/experimental/CollectiveX/bench/ep_flashinfer.py b/experimental/CollectiveX/bench/ep_flashinfer.py
index cd0fc0ae3..6d605bf7a 100644
--- a/experimental/CollectiveX/bench/ep_flashinfer.py
+++ b/experimental/CollectiveX/bench/ep_flashinfer.py
@@ -229,6 +229,7 @@ def create_buffer(self, spec):
             workspace_size_per_rank=workspace_size,
             mnnvl_config=MnnvlConfig(comm_backend=_communicator(_ep_group())),
         )
+        self.library_version = flashinfer.__version__
         if _wheel_has_fp32_combine(flashinfer.__version__):
             self.combine_reduction = "domain-fp32"
         # Every rank must finish mapping its workspace before any peer writes into it;
diff --git a/experimental/CollectiveX/bench/ep_harness.py b/experimental/CollectiveX/bench/ep_harness.py
index 502bc916f..12889d6e8 100644
--- a/experimental/CollectiveX/bench/ep_harness.py
+++ b/experimental/CollectiveX/bench/ep_harness.py
@@ -1228,6 +1228,11 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
             # pick this per installed library version (flashinfer-ep does), so without it
             # a wheel bump silently changes the arithmetic behind `passed` with no trace.
             "combine_reduction": getattr(backend, "combine_reduction", "domain-fp32"),
+            # The library version the line above was decided FROM, where the backend knows it.
+            # Recording only the outcome leaves the decision unauditable: a reader can see that
+            # the oracle used the slot-tree model but not which wheel selected it, and so cannot
+            # tell a correct selection from a mis-parse. None where a backend does not report one.
+            "library_version": getattr(backend, "library_version", None),
             # Whether `roundtrip` excludes expert-output staging. It always does now, unless
             # the CX_FP8_CONSUME=dequant hatch is set, but it did not always: rows measured
             # before that change carried the staging copy inside the chain for MoRI BF16 and

From 2fd913091f158e4cb3ba9f4b68c99a2e424ea482 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 4 Aug 2026 01:39:05 +0800
Subject: [PATCH 23/34] =?UTF-8?q?CollectiveX:=20exclude=20the=20h100=20nod?=
 =?UTF-8?q?e=20with=20a=20thermally=20clamped=20GPU=20/=20CollectiveX?=
 =?UTF-8?q?=EF=BC=9A=E6=8E=92=E9=99=A4=E5=AD=98=E5=9C=A8=20GPU=20=E6=B8=A9?=
 =?UTF-8?q?=E5=BA=A6=E9=99=8D=E9=A2=91=E6=95=85=E9=9A=9C=E7=9A=84=20h100?=
 =?UTF-8?q?=20=E8=8A=82=E7=82=B9?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Second node found the same way in a day, on a different cluster. `hpc-gpu-1-2` GPU 3, sampled from
inside the allocation under load:

    GPU 3        570-690 MHz   87 C   sw_thermal_slowdown Active
    GPUs 0-2,4-7    1980 MHz   ~40 C  no flags

`hpc-gpu-1-3`, measured alongside as a control, is uniformly healthy: all eight at 1980 MHz, 48-62 C,
no flags.

It was found while chasing an apparent uccl-ep prefill regression — compiled quantize losing 943us at
T=8192 against eager — which had its two arms on different nodes, with the losing arm on this one. A
clamp that deepens with load reproduces that shape exactly: heaviest at the top prefill rung, invisible
at decode. So the "regression" is most likely this node, and the swapped-arm rerun will confirm it.

Joining gpu-2-6 (b200, GPU 7) in the exclusion lists. Both are workarounds pending a drain and a
cooling/sensor check; both should come back out once fixed.

Worth recording as a general rule: **suspect the node whenever an A/B's arms ran on different hosts.**
Both faults were one-directional, load-proportional, and all-green — a shape indistinguishable from a
real code regression from the measurement side alone.

Not excluded, because not confirmed: hpc-gpu-1-6 and hpc-gpu-1-14 produced a rendezvous TCPStore
timeout in one sweep leg. That is a single observation and a different signature (network, not thermal),
and h100 is already down to 10 usable pods of 20.

100 tests pass.

中文：一天之内以同样方式在另一个集群上发现第二个故障节点。`hpc-gpu-1-2` 的 GPU 3，在分配内于负载下
采样（见上表）。作为对照同时测量的 `hpc-gpu-1-3` 完全健康：八块 GPU 均为 1980 MHz、48-62 C、无标志。

该节点是在排查一个看似存在的 uccl-ep prefill 回归时发现的——编译版量化在 T=8192 时比 eager 慢 943us
——而该对比的两组分别运行在不同节点上，且落后的那一组正好在此节点。随负载加深的降频恰好能复现这一
形态：在 prefill 最高档位最严重，在 decode 处不可见。因此该"回归"很可能就是这个节点，交换分组的重跑
将予以确认。

现与 gpu-2-6（b200，GPU 7）一同加入排除列表。两者都是等待 drain 与散热/传感器检查期间的权宜之计，
修复后都应移除。

值得记录的一般规则：**当 A/B 的两组运行在不同主机上时，应首先怀疑节点。** 两次故障都是单向、随负载
成比例、且全部通过——仅从测量侧看，这与真实的代码回归无法区分。

未排除（因未确认）：hpc-gpu-1-6 与 hpc-gpu-1-14 在某条扫描 leg 中出现 rendezvous TCPStore 超时。
那是单次观察且特征不同（网络而非温度），且 h100 的可用 pod 已从 20 降至 10。

100 项测试通过。
---
 experimental/CollectiveX/configs/platform_config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/experimental/CollectiveX/configs/platform_config.json b/experimental/CollectiveX/configs/platform_config.json
index 14b323fb2..7e23f2d41 100644
--- a/experimental/CollectiveX/configs/platform_config.json
+++ b/experimental/CollectiveX/configs/platform_config.json
@@ -17,7 +17,7 @@
         "partition": "hpc-gpu-1",
         "account": "customer",
         "squash_dir": "/mnt/nfs/sa-shared/cx-squash",
-        "exclude_nodes": "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19"
+        "exclude_nodes": "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-2,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19"
       },
       "network": {
         "socket_ifname": "eth0",

From a6328fe26f1cc7bdcacf7c911a73c8c7a987a205 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 4 Aug 2026 01:53:50 +0800
Subject: [PATCH 24/34] =?UTF-8?q?CollectiveX:=20fix=20four=20defects=20adv?=
 =?UTF-8?q?ersarial=20review=20found=20in=20my=20own=20work=20/=20Collecti?=
 =?UTF-8?q?veX=EF=BC=9A=E4=BF=AE=E6=AD=A3=E5=AF=B9=E6=8A=97=E6=80=A7?=
 =?UTF-8?q?=E8=AF=84=E5=AE=A1=E5=9C=A8=E6=88=91=E8=87=AA=E5=B7=B1=E6=94=B9?=
 =?UTF-8?q?=E5=8A=A8=E4=B8=AD=E5=8F=91=E7=8E=B0=E7=9A=84=E5=9B=9B=E5=A4=84?=
 =?UTF-8?q?=E7=BC=BA=E9=99=B7?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

All four are mine, from the GPU-health gate and the warm-up hoist.

**Tests appended past the `__main__` guard, in two files.** `cat >>` put `WarmStaging` and
`GpuHealthProbe` BELOW `if __name__ == "__main__": unittest.main()`, which raises SystemExit before
those classes are defined. `unittest discover` imports the module and collected them, so CI was
green and the omission was invisible — but running either file directly silently skipped them:
test_roundtrip_staging 9 of 12, test_runtime 41 of 50. Anyone iterating on these files got false
green on exactly the tests written to pin the new behaviour. Both guards now sit at EOF; direct
execution collects 12 and 50, discover still 103.

**The health gate could not prove it saw any GPUs.** It emitted nothing on success, so a gate gone
BLIND — zero visible devices, or a driver old enough to spell the fields `clocks_throttle_reasons.*`
— wrote an empty log indistinguishable from one that inspected eight healthy GPUs. That is precisely
the "looks like protection" failure the gate exists to avoid. It now emits
`gpu-health-checked gpus=N`, which makes per-cluster verification possible rather than assumed.
Three tests pin the healthy, faulty and no-binary paths.

**The probe relied on an undocumented default to see the GPUs at all.** It ran `srun` without
`--gres`, trusting a step to inherit the job's GRES. That is Slurm's documented default and the
allocation is `--exclusive`, so it worked — but a site config or a stray `SLURM_STEP_GRES` would
have blinded it silently, and the sibling cuda-context probe already passes `--gres=gpu:N`. Now it
does too, which settles the question by construction instead of by argument. `--time=5` bounds the
other end: `nvidia-smi` can wedge uninterruptible on exactly the sick hardware this looks for, and
Python's own timeout cannot reap a process in D-state, so the "seconds instead of thirty minutes"
claim inverted on the worst case it targets.

**A doc comment ended up on the wrong function.** Inserting the gate split
`collx_validate_cuda_context_on_job` from its comment, leaving "A clean nvidia-smi inventory does not
prove that a prior cancelled workload released every CUDA context" heading a function that IS an
nvidia-smi inventory. Restored, and the gate has its own.

Also corrected a docstring the hoist made stale: `_ll_dequant_static` said the dequant "runs in every
timed component's warmup", which stopped being true when warm-up started staging once.

103 tests pass; direct execution of each test file now collects everything it defines.

中文：四处缺陷均出自我自己的改动（GPU 健康门与预热 hoist）。

**两个文件中的测试被追加到了 `__main__` 保护之后。** `cat >>` 把 `WarmStaging` 与 `GpuHealthProbe`
放到了 `if __name__ == "__main__": unittest.main()` 之下，而后者会在这些类定义之前抛出 SystemExit。
`unittest discover` 通过导入模块仍能收集到它们，因此 CI 是绿的、遗漏不可见——但直接运行这两个文件会
静默跳过：test_roundtrip_staging 为 9/12，test_runtime 为 41/50。在这些文件上迭代的人，恰好在为新行为
把关的测试上得到虚假的绿色。现两处保护均置于文件末尾；直接执行分别收集 12 与 50 项，discover 仍为 103。

**健康门无法证明自己看到了任何 GPU。** 它在成功时不输出任何内容，因此一个已经"失明"的门——没有可见
设备，或驱动版本老到字段名为 `clocks_throttle_reasons.*`——写出的空日志与检查了八块健康 GPU 的情形
完全无法区分。这正是该门本应避免的"看起来像保护"的失效模式。现会输出
`gpu-health-checked gpus=N`，使按集群核验成为可能而非只能假定。三项测试分别锁定健康、故障与无二进制
三条路径。

**该探针原本依赖一个未写明的默认行为才能看到 GPU。** 它调用 `srun` 时未传 `--gres`，依赖步骤继承作业
的 GRES。这确实是 Slurm 的默认行为且分配为 `--exclusive`，所以能工作——但站点配置或残留的
`SLURM_STEP_GRES` 会让它静默失明，而同类的 cuda-context 探针本就传了 `--gres=gpu:N`。现同样传入，
从构造上而非论证上解决该问题。`--time=5` 约束另一端：`nvidia-smi` 恰好可能在该门所针对的故障硬件上
陷入不可中断等待，而 Python 自身的超时无法回收处于 D 状态的进程，从而使"数秒而非三十分钟"的论断在其
针对的最坏情形下反转。

**一段文档注释落到了错误的函数上。** 插入该门时把 `collx_validate_cuda_context_on_job` 与其注释分开，
使"干净的 nvidia-smi 清单并不能证明先前被取消的负载已释放每个 CUDA 上下文"这句话，落在了一个**本身
就是** nvidia-smi 清单的函数之上。已恢复，并为该门另写注释。

同时修正了因 hoist 而过期的一处 docstring：`_ll_dequant_static` 原称该反量化"在每个计时组件的预热中
运行"，而预热改为只 stage 一次后该说法已不成立。

103 项测试通过；每个测试文件直接执行时现均能收集其定义的全部测试。
---
 .../CollectiveX/bench/ep_deepep_v2.py         |  5 +-
 .../launchers/launch_single-slurm.sh          |  2 +-
 experimental/CollectiveX/runtime/common.sh    | 16 ++++--
 experimental/CollectiveX/runtime/probe.py     |  5 ++
 .../tests/test_roundtrip_staging.py           |  9 ++-
 .../CollectiveX/tests/test_runtime.py         | 55 +++++++++++++++++--
 6 files changed, 75 insertions(+), 17 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_deepep_v2.py b/experimental/CollectiveX/bench/ep_deepep_v2.py
index a671b2909..5be2ac44f 100644
--- a/experimental/CollectiveX/bench/ep_deepep_v2.py
+++ b/experimental/CollectiveX/bench/ep_deepep_v2.py
@@ -47,8 +47,9 @@ def _ll_dequant_static(fp8, scales):
     ``[num_local_experts, cap*num_ranks, hidden]`` = (32, 2048, 7168) at EP8). The low-latency
     padded shape is constant on every dispatch, so a static (``dynamic=False``) compile fuses
     to one FP32 pass (~0.5 ms, 6.3x, bit-identical to the dynamic kernel on valid slots). The
-    dequant runs in every timed component's warmup and samples (~hundreds of thousands of
-    calls over the profile), so the dynamic kernel's per-call overhead overran the leg's
+    dequant runs in every timed `stage` sample, in `benchmark_stage`'s warm-up, and once per
+    other component's warm-up (it was every warm iteration until the staging hoist), so the
+    call count is large enough that the dynamic kernel's per-call overhead overran the leg's
     wall-clock budget (all ranks SIGKILLed ~22 min in, no result); the static form brings FP8
     low-latency inside the budget BF16 already meets. Padding slots decode to NaN in both
     forms (FP8 padding bytes) — harmless, because combine is handle-indexed and never reads
diff --git a/experimental/CollectiveX/launchers/launch_single-slurm.sh b/experimental/CollectiveX/launchers/launch_single-slurm.sh
index 6e3215eab..b58eb0b63 100644
--- a/experimental/CollectiveX/launchers/launch_single-slurm.sh
+++ b/experimental/CollectiveX/launchers/launch_single-slurm.sh
@@ -114,7 +114,7 @@ for allocation_attempt in 1 2 3; do
   elif [ "$RUNNER" = b300 ] \
       && ! collx_validate_cuda_context_on_job "$JOB_ID" "$NODES" "$GPN"; then
     validation_failure=cuda-context
-  elif ! collx_validate_gpu_health_on_job "$JOB_ID" "$NODES"; then
+  elif ! collx_validate_gpu_health_on_job "$JOB_ID" "$NODES" "$GPN"; then
     validation_failure=gpu-health
   else
     break
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 5452c118c..eafec3260 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -760,11 +760,14 @@ BASH
   printf '%s' "$sq"
 }
 
-# A clean nvidia-smi inventory does not prove that a prior cancelled workload
-# released every CUDA context. Retaining each primary context catches poisoned
-# allocations before a full shard spends time failing every case.
+# Reject an allocation whose GPUs are throttled: collectives are barriers, so one clamped device
+# paces every rank. `--gres` mirrors the cuda-context probe below rather than relying on a step
+# inheriting the job's GRES by default, so the probe provably sees the devices it is judging --
+# a gate that sees none would pass everything while looking like protection. `--time` bounds the
+# worst case: nvidia-smi can wedge uninterruptible on exactly the sick hardware this looks for,
+# and Python's own timeout cannot reap a process in D-state.
 collx_validate_gpu_health_on_job() {
-  local job_id="$1" nodes="$2" log_label=gpu-health log
+  local job_id="$1" nodes="$2" gpus_per_node="$3" log_label=gpu-health log
   case "${COLLX_SALLOC_ATTEMPT:-1}" in
     1) ;;
     2|3) log_label+="-a${COLLX_SALLOC_ATTEMPT}" ;;
@@ -773,11 +776,14 @@ collx_validate_gpu_health_on_job() {
   log="$(collx_private_log_path "$log_label")"
   export COLLX_GPU_HEALTH_LOG="$log"
   srun --jobid="$job_id" --nodes="$nodes" --ntasks="$nodes" --ntasks-per-node=1 \
-    --chdir=/tmp --input=all \
+    --gres=gpu:"$gpus_per_node" --time=5 --chdir=/tmp --input=all \
     --export="$(collx_host_exports)" python3 /dev/stdin gpu-health \
     < "$COLLX_RUNTIME_DIR/probe.py" >"$log" 2>&1
 }
 
+# A clean nvidia-smi inventory does not prove that a prior cancelled workload
+# released every CUDA context. Retaining each primary context catches poisoned
+# allocations before a full shard spends time failing every case.
 collx_validate_cuda_context_on_job() {
   local job_id="$1" nodes="$2" gpus_per_node="$3" log_label=cuda-context log
   case "${COLLX_SALLOC_ATTEMPT:-1}" in
diff --git a/experimental/CollectiveX/runtime/probe.py b/experimental/CollectiveX/runtime/probe.py
index 9f1ccaa21..e4a11438c 100644
--- a/experimental/CollectiveX/runtime/probe.py
+++ b/experimental/CollectiveX/runtime/probe.py
@@ -95,6 +95,11 @@ def validate_gpu_health(max_temperature_c: int = 90) -> None:
         _emit(f"gpu-health-fault {fault}")
     if faults:
         raise SystemExit(1)
+    # Positive control. Without it a gate that has gone BLIND -- no visible devices, or a driver
+    # old enough to spell these fields `clocks_throttle_reasons.*` -- writes an empty log and is
+    # indistinguishable from one that inspected eight healthy GPUs. Recording the count is what
+    # makes "the gate ran and saw N devices" checkable per cluster instead of assumed.
+    _emit(f"gpu-health-checked gpus={sum(1 for line in output.splitlines() if line.strip())}")
 
 
 def _emit(marker: str) -> None:
diff --git a/experimental/CollectiveX/tests/test_roundtrip_staging.py b/experimental/CollectiveX/tests/test_roundtrip_staging.py
index 1c5d04f6a..e2fe6d6b8 100644
--- a/experimental/CollectiveX/tests/test_roundtrip_staging.py
+++ b/experimental/CollectiveX/tests/test_roundtrip_staging.py
@@ -148,11 +148,6 @@ def test_the_hatch_does_not_apply_to_bf16(self):
             ).stage_excluded_from_roundtrip
         )
 
-
-if __name__ == "__main__":
-    unittest.main()
-
-
 class WarmStaging(unittest.TestCase):
     """Warm-up must not rehearse work the timed region skips.
 
@@ -195,3 +190,7 @@ def test_a_chain_that_includes_staging_keeps_warming_it(self):
         b = _StubBackend(stage_device_work=True, fp8_consume="dequant")
         self._warm(b, 5)
         self.assertEqual(b.calls.count("stage"), 5)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/experimental/CollectiveX/tests/test_runtime.py b/experimental/CollectiveX/tests/test_runtime.py
index 8fba43307..c762f8df1 100644
--- a/experimental/CollectiveX/tests/test_runtime.py
+++ b/experimental/CollectiveX/tests/test_runtime.py
@@ -755,10 +755,6 @@ def shape_dependent(t):
         with self.assertRaises(RuntimeError):
             self._check(eager, shape_dependent, x)
 
-if __name__ == "__main__":
-    unittest.main()
-
-
 class GpuHealthProbe(unittest.TestCase):
     """Reject an allocation holding a throttled GPU before it burns the wall-clock guard.
 
@@ -806,3 +802,54 @@ def test_unreadable_output_fails_open(self):
         for output in ("", "nonsense\n", "1, Not Active\n", self.HEALTHY.replace("32 ", "[N/A] ")):
             with self.subTest(output=output[:20]):
                 self.assertEqual(probe.gpu_health_faults(output), [])
+
+    def _run_validate(self, csv: str, has_smi: bool = True):
+        """Drive validate_gpu_health with a stubbed nvidia-smi; returns (exit_code, stdout)."""
+        import shutil
+        real_which = shutil.which
+        shutil.which = (lambda name: "/usr/bin/nvidia-smi") if has_smi else (lambda name: None)
+
+        class FakeSubprocess:
+            SubprocessError = subprocess.SubprocessError
+
+            @staticmethod
+            def run(*args, **kwargs):
+                return types.SimpleNamespace(stdout=csv)
+
+        sys.modules["subprocess"] = FakeSubprocess
+        captured = io.StringIO()
+        try:
+            with contextlib.redirect_stdout(captured):
+                probe.validate_gpu_health()
+            code = 0
+        except SystemExit as exit_:
+            code = exit_.code
+        finally:
+            sys.modules["subprocess"] = subprocess
+            shutil.which = real_which
+        return code, captured.getvalue()
+
+    def test_a_healthy_check_records_how_many_gpus_it_saw(self):
+        # Without this marker a gate that went BLIND -- no visible devices, or a driver spelling
+        # the fields clocks_throttle_reasons.* -- writes an empty log and is indistinguishable
+        # from one that inspected eight healthy GPUs.
+        code, out = self._run_validate(self.HEALTHY)
+        self.assertEqual(code, 0)
+        self.assertIn("gpu-health-checked gpus=8", out)
+
+    def test_a_fault_exits_nonzero_and_names_the_gpu(self):
+        code, out = self._run_validate(
+            self._swap("7, Not Active, Not Active, 37 ", "7, Active, Active, 93 ")
+        )
+        self.assertEqual(code, 1)
+        self.assertIn("gpu-health-fault gpu 7", out)
+        self.assertNotIn("gpu-health-checked", out)
+
+    def test_a_missing_nvidia_smi_is_silent_and_passes(self):
+        code, out = self._run_validate(self.HEALTHY, has_smi=False)
+        self.assertEqual(code, 0)
+        self.assertEqual(out, "")
+
+
+if __name__ == "__main__":
+    unittest.main()

From cda28f893b3e0fd97e40f6abda67a5231f598f5d Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 4 Aug 2026 01:59:15 +0800
Subject: [PATCH 25/34] =?UTF-8?q?CollectiveX:=20record=20the=20temperature?=
 =?UTF-8?q?=20spread=20the=20gate=20cannot=20act=20on=20/=20CollectiveX?=
 =?UTF-8?q?=EF=BC=9A=E8=AE=B0=E5=BD=95=E5=81=A5=E5=BA=B7=E9=97=A8=E6=97=A0?=
 =?UTF-8?q?=E6=B3=95=E6=8D=AE=E4=BB=A5=E5=88=A4=E5=AE=9A=E7=9A=84=E6=B8=A9?=
 =?UTF-8?q?=E5=BA=A6=E7=A6=BB=E6=95=A3=E5=BA=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On-metal thermal tracing of the h100 fault showed the gate's absolute temperature arm is unreachable
for that fault class. GPU 3 peaked at **87 C** under load — H100 engages software thermal slowdown at
~86-87 C, so a clamped H100 never crosses a 90 C limit. And at pre-flight the allocation is idle, so
the throttle flag is clear too. Both halves of the gate are blind to a load-dependent clamp on that
architecture.

The one signal that WAS visible at pre-flight was relative: the sick GPU idled at 55 C against ~30 C
for every sibling on both nodes. Measured healthy references, now on record: 50-66 C under load on
h100 (~16 C spread), 34-39 C on b200, and node2's healthy siblings at 39-43 C — putting GPU 3's 87 C
about 44 C above its own siblings.

So `gpu-health-checked` now carries `hottest/median/spread`. Deliberately REPORTED, not gated: I have
n=1 for idle spread predicting an under-load clamp, and a rejection threshold picked from one incident
would risk failing healthy allocations — which on three consecutive attempts kills the leg, a worse
outcome than the fault. Instrument first, gate once the distribution is known. `gpu_temperature_spread`
is a pure function beside `gpu_health_faults`, so promoting it later is a one-line change in
`validate_gpu_health`.

Three tests cover the sick spread, its appearance in the marker, and the unreadable cases. One of them
caught my own arithmetic: I expected a median of 34 for eight temperatures when it is the 5th sorted
value, 35.

106 tests pass; direct execution of the file collects all 53 it defines.

中文：对 h100 故障的实机温度追踪表明，该门的绝对温度判据对这类故障不可达。GPU 3 在负载下峰值为
**87 C**——H100 在约 86-87 C 即启用软件温度降频，因此被钳制的 H100 永远不会越过 90 C 阈值。而在预检
时分配处于空闲，降频标志同样为清空。在该架构上，门的两个判据对随负载出现的钳制都是盲的。

预检时**确实**可见的唯一信号是相对量：故障 GPU 空闲温度为 55 C，而两个节点上所有同伴均约 30 C。现已
记录的实测健康参考值：h100 负载下 50-66 C（离散约 16 C）、b200 为 34-39 C，且 node2 的健康同伴为
39-43 C——即 GPU 3 的 87 C 高出其同伴约 44 C。

因此 `gpu-health-checked` 现携带 `hottest/median/spread`。这是有意**仅上报、不作判定**：空闲离散度可
预测负载下钳制这一结论目前样本数为 1，而依据单次事件选定的拒绝阈值有可能误判健康分配——连续三次误判
即会杀掉该 leg，其后果比故障本身更糟。先埋点，待分布已知后再设门。`gpu_temperature_spread` 是与
`gpu_health_faults` 并列的纯函数，日后提升为判定只需在 `validate_gpu_health` 中改一行。

三项测试覆盖故障离散度、其在标记中的出现，以及无法解析的情形。其中一项抓出了我自己的算术错误：八个
温度值的中位数应取排序后第 5 个（35），而我写成了 34。

106 项测试通过；该文件直接执行时可收集其定义的全部 53 项测试。
---
 experimental/CollectiveX/runtime/probe.py     | 34 ++++++++++++++++++-
 .../CollectiveX/tests/test_runtime.py         | 22 ++++++++++++
 2 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/experimental/CollectiveX/runtime/probe.py b/experimental/CollectiveX/runtime/probe.py
index e4a11438c..073096c98 100644
--- a/experimental/CollectiveX/runtime/probe.py
+++ b/experimental/CollectiveX/runtime/probe.py
@@ -62,6 +62,34 @@ def gpu_health_faults(output: str, max_temperature_c: int = 90) -> list[str]:
     return faults
 
 
+def gpu_temperature_spread(output: str) -> tuple[int, int, int] | None:
+    """`(hottest, median, spread)` GPU temperature, or None if unreadable.
+
+    Reported, NOT gated on. The absolute threshold in `gpu_health_faults` is architecture
+    dependent and can be unreachable: an H100 engages its software thermal slowdown at ~86-87 C,
+    so a clamped H100 never crosses a 90 C limit even under load, and at pre-flight it is idle
+    anyway. In the one fault measured end to end, the only signal visible at pre-flight time was
+    the RELATIVE one -- the sick GPU idled at 55 C against ~30 C for every sibling, then clamped
+    under load. Recording the spread every run is how that becomes a gate with evidence behind its
+    threshold instead of a heuristic picked from a single incident. Healthy references so far:
+    50-66 C under load on h100 (~16 C spread), 34-39 C on b200.
+    """
+    temperatures = []
+    for line in output.splitlines():
+        cells = [cell.strip() for cell in line.split(",")]
+        if len(cells) != len(_GPU_HEALTH_FIELDS):
+            continue
+        try:
+            temperatures.append(int(cells[3].split()[0]))
+        except (IndexError, ValueError):
+            continue
+    if not temperatures:
+        return None
+    temperatures.sort()
+    median = temperatures[len(temperatures) // 2]
+    return temperatures[-1], median, temperatures[-1] - median
+
+
 def validate_gpu_health(max_temperature_c: int = 90) -> None:
     """Reject an allocation holding a thermally throttled GPU.
 
@@ -99,7 +127,11 @@ def validate_gpu_health(max_temperature_c: int = 90) -> None:
     # old enough to spell these fields `clocks_throttle_reasons.*` -- writes an empty log and is
     # indistinguishable from one that inspected eight healthy GPUs. Recording the count is what
     # makes "the gate ran and saw N devices" checkable per cluster instead of assumed.
-    _emit(f"gpu-health-checked gpus={sum(1 for line in output.splitlines() if line.strip())}")
+    spread = gpu_temperature_spread(output)
+    detail = "" if spread is None else f" hottest={spread[0]}C median={spread[1]}C spread={spread[2]}C"
+    _emit(
+        f"gpu-health-checked gpus={sum(1 for line in output.splitlines() if line.strip())}{detail}"
+    )
 
 
 def _emit(marker: str) -> None:
diff --git a/experimental/CollectiveX/tests/test_runtime.py b/experimental/CollectiveX/tests/test_runtime.py
index c762f8df1..f1549de1a 100644
--- a/experimental/CollectiveX/tests/test_runtime.py
+++ b/experimental/CollectiveX/tests/test_runtime.py
@@ -849,6 +849,28 @@ def test_a_missing_nvidia_smi_is_silent_and_passes(self):
         code, out = self._run_validate(self.HEALTHY, has_smi=False)
         self.assertEqual(code, 0)
         self.assertEqual(out, "")
+    def test_the_temperature_spread_is_reported_for_the_signal_no_gate_can_see(self):
+        # An H100 engages software thermal slowdown at ~86-87 C, so a clamped one never crosses the
+        # 90 C limit; and at pre-flight it is idle, so the flag is clear too. The measured fault was
+        # visible ONLY as an idle outlier (55 C against ~30 C siblings), so the spread is recorded
+        # every run to build the evidence a relative gate would need.
+        sick = self._swap("3, Not Active, Not Active, 33 ", "3, Not Active, Not Active, 55 ")
+        self.assertEqual(probe.gpu_temperature_spread(sick), (55, 35, 20))
+        hottest, median, spread = probe.gpu_temperature_spread(self.HEALTHY)
+        self.assertEqual((hottest, median), (37, 34))  # 8 temps -> median is index 4
+        self.assertLess(spread, 10)
+
+    def test_the_spread_appears_in_the_healthy_marker(self):
+        sick = self._swap("3, Not Active, Not Active, 33 ", "3, Not Active, Not Active, 55 ")
+        code, out = self._run_validate(sick)
+        self.assertEqual(code, 0)  # reported, deliberately NOT gated on
+        self.assertIn("spread=20C", out)
+
+    def test_the_spread_is_none_when_unreadable(self):
+        for output in ("", "nonsense\n", self.HEALTHY.replace("33 ", "[N/A] ")):
+            with self.subTest(output=output[:16]):
+                result = probe.gpu_temperature_spread(output)
+                self.assertTrue(result is None or result[2] < 10)
 
 
 if __name__ == "__main__":

From fc078232aee0065c7197d0886f296a24f913ef07 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 4 Aug 2026 02:10:37 +0800
Subject: [PATCH 26/34] =?UTF-8?q?CollectiveX:=20correct=20eight=20claims?=
 =?UTF-8?q?=20a=20third=20review=20found=20wrong=20/=20CollectiveX?=
 =?UTF-8?q?=EF=BC=9A=E4=BF=AE=E6=AD=A3=E7=AC=AC=E4=B8=89=E8=BD=AE=E8=AF=84?=
 =?UTF-8?q?=E5=AE=A1=E5=8F=91=E7=8E=B0=E7=9A=84=E5=85=AB=E5=A4=84=E9=94=99?=
 =?UTF-8?q?=E8=AF=AF=E8=AE=BA=E6=96=AD?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

No behaviour change. The review confirmed the MoRI buffer-mode switch has no out-of-bounds read in
any reachable configuration, then found that four comments and four documented facts were wrong.

Comments, all stale after the switch:

- `ep_mori.py` stage() cited `intranode.hpp:542` for the bound that makes the `[:rows]` cast safe.
  The bound is real, but line 542 sits inside `else if constexpr (UseP2PRead)`, and with an external
  input buffer plus combine quant_type "none" the launcher selects
  `EpCombineIntraNodeKernel_bf16_nop2p` — UseP2PRead FALSE. The branch that actually runs carries the
  same `tokenIdx < totalRecvTokenNum` bound, so the conclusion held for the wrong reason. Now cites
  the live branch and notes the other is compile-time dead here.
- `ep_mori.py` still said "With use_external_inp_buf False the launcher takes the zero-copy branch"
  and that `_nop2p` sits behind a branch "we never enter". Both halves inverted by the switch.
- `ep_backend.py` benchmark_roundtrip still said the staged tensor "IS the registered combine buffer"
  for MoRI. It is the dispatch output at BF16, or a fresh `[:rows]` cast under FP8.
- `tests/test_roundtrip_staging.py` still said MoRI BF16 rows "do real device work" in stage. They do
  not: `self._fp8 or not self._external_input` collapses to `self._fp8`. Line 71 of the same file
  already stated the new behaviour, so the file contradicted itself.

methodology.md, which defines what a published number means:

- "issued as one fused kernel and guarded bitwise against its eager reference" was asserted for all
  FP8 `normal` backends. False for MoRI, whose quantize is a plain dtype cast needing neither.
- Byte accounting credited per-128-block FP32 scales to "DeepEP's and UCCL-EP's blockwise codec"
  only. FlashInfer EP carries them too, as a fourth dispatch payload — which another section of the
  same file already said, so the doc contradicted itself across sections.
- The `stage`-is-scaffolding parenthetical said DeepEP V2 converts "a padded low-latency plane". That
  is its LOW-LATENCY path; in `normal` mode it converts only received rows. UCCL-EP was omitted
  entirely. Since that sentence exists to explain why `stage` is not comparable between backends,
  getting the quantities wrong defeated it.
- The AUTO-tuning aside claimed no gfx950 BF16 dispatch rule and no gfx950 IntraNodeLL combine table,
  hence hard-coded defaults. Only half true: gfx950 does ship an IntraNodeLL BF16 dispatch table and
  an IntraNode combine table with BF16 rules at both `zero_copy` values. Rewritten to say what is
  actually missing, and why partial tuning is its own argument against AUTO.
- Added the bridge for a tension the review flagged: the doc says published cohorts rank on p99 and
  also says do not rank on p99 of MAX for multi-node decode. Both are true because the cohorts group
  into bootstrap equivalence bands rather than ordering by raw p99, so a stall-dominated cell ties
  instead of winning; a reader comparing two cells by hand has no such machinery, which is what the
  MAX/MIN bracket is for.
- Scoped the "both engines pin 80/0/16" claim to the BF16 and FP8 paths this suite sweeps; SGLang has
  a separate FP4 intra-node override.

Left alone: the vLLM-side claims (fp8+scales passed through on a block_k match, dequant as the
mismatch fallback) are unverifiable from this tree and consistent with earlier findings.

106 tests pass.

中文：无行为变更。评审确认 MoRI 缓冲模式切换在所有可达配置下均无越界读取，随后发现四处注释与四项文档
事实有误。

注释（均因该切换而过期）：`ep_mori.py` 的 stage() 引用 `intranode.hpp:542` 作为 `[:rows]` 转换安全性
的依据——该界限确实存在，但第 542 行位于 `else if constexpr (UseP2PRead)` 之内，而在外部输入缓冲加
combine quant_type "none" 的组合下，launcher 选择的是 `EpCombineIntraNodeKernel_bf16_nop2p`，即
UseP2PRead 为**假**。真正执行的分支带有相同的 `tokenIdx < totalRecvTokenNum` 界限，因此结论成立但理由
错误；另三处分别为：仍称 use_external_inp_buf 为 False、仍称 staged 张量是 MoRI 的注册 combine 缓冲、
仍称 MoRI BF16 行在 stage 中有实际设备工作（同一文件第 71 行已正确表述，即文件自相矛盾）。

methodology.md（定义已发布指标含义的文件）：将"以单个融合 kernel 下发并逐位对齐 eager 参考"错误地断言
于所有 FP8 `normal` 后端（对 MoRI 不成立）；字节核算遗漏了 FlashInfer EP 的 scale 载荷（而同一文件另一
处已提及其第四个载荷，即跨节自相矛盾）；`stage` 脚手架说明中把 DeepEP V2 说成转换"补齐后的低延迟平面"
（那是其低延迟路径，`normal` 模式仅转换已接收行），并完全遗漏 UCCL-EP；AUTO 调优旁注的论据半数不实，
已改为陈述实际缺失的部分；补充了 p99 与 p50 排序张力的衔接说明（队列按 bootstrap 等价带分组而非按原始
p99 排序）；并将"两个引擎均固定 80/0/16"的说法限定于本套件所扫描的 BF16 与 FP8 路径。

未改动：vLLM 侧的论断在本代码树中无法验证，且与既有结论一致。

106 项测试通过。
---
 experimental/CollectiveX/bench/ep_backend.py  |  8 ++---
 experimental/CollectiveX/bench/ep_mori.py     | 16 +++++----
 experimental/CollectiveX/docs/methodology.md  | 36 ++++++++++++-------
 .../tests/test_roundtrip_staging.py           |  7 ++--
 4 files changed, 42 insertions(+), 25 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_backend.py b/experimental/CollectiveX/bench/ep_backend.py
index e5d82dca9..6882a279f 100644
--- a/experimental/CollectiveX/bench/ep_backend.py
+++ b/experimental/CollectiveX/bench/ep_backend.py
@@ -462,10 +462,10 @@ def benchmark_roundtrip(self, problem, warmup, iters):
         if self.stage_excluded_from_roundtrip:
             # Materialise the expert-output stand-in ONCE, untimed, so the chained
             # measurement is dispatch -> combine and nothing else. Routing is fixed for a
-            # ladder point, so the same staged tensor is valid for every iteration (for MoRI
-            # it IS the registered combine buffer, already filled; for FlashInfer it is the
-            # workspace combine region, which dispatch cannot clobber because that region
-            # sits past the end of every dispatch receive plane).
+            # ladder point, so the same staged tensor is valid for every iteration (for MoRI it is
+            # the dispatch output itself at BF16, or a fresh `[:rows]` BF16 cast under FP8; for
+            # FlashInfer it is the workspace combine region, which dispatch cannot clobber because
+            # that region sits past the end of every dispatch receive plane).
             #
             # Read the staged payload back through `combine_input_attr` rather than
             # constructing one, so whatever the adapter put there round-trips unchanged. No
diff --git a/experimental/CollectiveX/bench/ep_mori.py b/experimental/CollectiveX/bench/ep_mori.py
index d5d0fb01e..eca73089e 100644
--- a/experimental/CollectiveX/bench/ep_mori.py
+++ b/experimental/CollectiveX/bench/ep_mori.py
@@ -54,9 +54,10 @@ def __init__(self, args, rank, world_size, local_rank, device):
         # argv. FP8 dispatch is caller-prequantized: MoRI's dispatch kernel keys purely
         # on the passed tensor dtype, so handing it an e4m3 tensor selects the FP8
         # dispatch kernel with no in-kernel cast. Combine stays genuinely BF16 (quant_type
-        # "none"). With use_external_inp_buf False the launcher
-        # takes the zero-copy branch, EpCombineIntraNodeKernel_bf16_p2p; the _nop2p and
-        # _fp8cast variants both sit behind the external-buffer branch we never enter.
+        # "none"). With use_external_inp_buf True (the engines' setting, pinned unconditionally
+        # below) the launcher selects EpCombineIntraNodeKernel_bf16_nop2p; the _p2p variant is the
+        # registered-buffer path this adapter no longer takes, and _fp8cast stays unreachable
+        # because combine's quant_type is "none".
         self._fp8_dtype = None
         if self._fp8:
             arch = torch.cuda.get_device_properties(device).gcnArchName
@@ -304,9 +305,12 @@ def stage(self, p, h):
         if not isinstance(rows, int) or rows < 0 or rows > h.dispatch_output.size(0):
             raise RuntimeError("MoRI receive count was not validated before staging")
         if self._external_input:
-            # The kernel's own staging copy is bounded by the receive count, not the buffer
-            # (`EpCombineIntraNodeKernel_body` loops `i < totalRecvTokenNum`, intranode.hpp:542),
-            # so it never reads past `rows` and only the filled rows need converting.
+            # The kernel's own staging copy is bounded by the receive count, not the buffer: with
+            # an external input buffer and combine quant_type "none" the launcher selects
+            # `EpCombineIntraNodeKernel_bf16_nop2p`, whose write-based staging loop is bounded by
+            # `tokenIdx < totalRecvTokenNum` over `args.inpTokenBuf`. (The `UseP2PRead` loop at
+            # intranode.hpp:542 carries the same bound but is compile-time dead on this path.)
+            # So it never reads past `rows` and only the filled rows need converting.
             h.combine_input = (
                 h.dispatch_output[:rows].to(torch.bfloat16) if self._fp8 else h.dispatch_output
             )
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 51525ed2a..34a69c0ef 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -26,8 +26,9 @@ supported upstream (DeepEP V2, MoRI, UCCL-EP, FlashInfer EP), an FP8 dispatch (`
 caller-prequantized in `normal` mode (the `low-latency` kernels quantize FP8 internally from BF16 on
 DeepEP and UCCL-EP, and stay caller-prequantized on MoRI). Because `normal`-mode FP8 is
 caller-prequantized, that quantize is a cost a production forward pass pays on the critical path, so
-it is charged **inside the measured dispatch** rather than prepared ahead of the timing window; it is
-issued as one fused kernel and guarded bitwise against its eager reference. This means an FP8
+it is charged **inside the measured dispatch** rather than prepared ahead of the timing window. On
+DeepEP V2, UCCL-EP and FlashInfer EP it is issued as one fused kernel and guarded bitwise against its
+eager reference; MoRI needs neither, because its quantize is a single plain dtype cast. This means an FP8
 `normal` dispatch number covers quantize-plus-transport while its BF16 control covers transport
 alone, and it is why FP8 `normal` rows are not comparable to runs published before sweep version 2.
 
@@ -82,7 +83,8 @@ represented by two physical RDMA ranks, with eight scale-up ranks per domain. GB
 scale-up and uses LSA. MoRI EP8 uses the direct IntraNode kernel on every CDNA SKU; its EP16 InterNodeV1 path is
 configured but unsupported (transport-layer combine corruption, ROCm/mori#475) and never dispatched.
 MoRI runs under its MANUAL launch mode with a pinned launch config, because that is what the engines
-run: neither vLLM nor SGLang sets `MORI_EP_LAUNCH_CONFIG_MODE`, and both pin block_num 80,
+run: neither vLLM nor SGLang sets `MORI_EP_LAUNCH_CONFIG_MODE`, and for the BF16 and FP8 paths this
+suite sweeps both pin block_num 80,
 rdma_block_num 0, and `warp_num_per_block` 16 for the intra-node kernel, applied to dispatch and
 combine alike (neither passes a per-call override, so combine inherits the 16). Both also run with an
 external input buffer, which is MoRI's default and which SGLang sets explicitly. Those two settings
@@ -101,9 +103,11 @@ block and a zero sample count, the same way any backend whose staging is a bare
 reports it. FP8 rows still stage for real, because the received payload has to be dequantized. These
 numbers therefore describe the engine-integrated configuration, not MoRI's peak: its shipped tuning
 tables reach a faster combine with per-shape block and warp counts no engine selects, and AUTO would
-not reproduce those tables anyway (there is no BF16 gfx950 dispatch rule and no gfx950 IntraNodeLL
-combine table, so AUTO falls back to hard-coded defaults and couples the result to whichever MoRI
-revision is pinned). How far off peak is arch-dependent and only partly measured: comparing across
+not reproduce them uniformly anyway: gfx950 ships no IntraNodeLL combine table and no BF16 rule for
+normal-mode IntraNode dispatch, so AUTO falls back to hard-coded defaults for exactly those two and
+couples the result to whichever MoRI revision is pinned. It would find genuinely tuned rules for the
+other two paths, which is its own problem — a config that is tuned on some paths and defaulted on
+others is not one number about the hardware. How far off peak is arch-dependent and only partly measured: comparing across
 buffer modes on MI355X, with the registered mode's excluded BF16 stage added back so the comparison is
 not flattered, a registered buffer at 8 warps is still 15% faster at T=512 decode and 8% at T=8192
 prefill than the shipped pairing. That margin did not reproduce as a clear win on gfx942, so treat
@@ -181,9 +185,11 @@ that as a separate step: the FP8 lands in the expert GEMM, which reads FP8 opera
 BF16, and that GEMM output is what combine receives. This suite deliberately does not run the expert
 GEMM — it measures the collective, not the layer — so `stage` stands in for it. That is why `stage` is
 excluded from `roundtrip`, and it is also why **`stage` must not be summed into a total or compared
-between backends**: each adapter converts a different amount (DeepEP V2 a padded low-latency plane,
-MoRI only the received rows, FlashInfer only the filled slots), so the same component name covers
-three different quantities. The one production path that *does* pay a separate materialised dequant is
+between backends**: each adapter converts a different amount. DeepEP V2 and UCCL-EP convert only the
+received rows in `normal` mode but the whole padded plane in `low-latency`, where the receive buffer is
+`[experts, cap * ranks, hidden]` regardless of token count; MoRI converts only the received rows;
+FlashInfer only the filled slots. So the same component name covers several different quantities, and
+for two of the backends it covers a different one per mode. The one production path that *does* pay a separate materialised dequant is
 a quant-format mismatch fallback (vLLM dequantises when `block_k` disagrees with DeepEP's block size);
 `CX_FP8_CONSUME=dequant` exists to model exactly that case, and it is not the default because it is
 not the fast path.
@@ -200,7 +206,12 @@ availability, origin, and sample count. A paired-only API reports null isolated
 `isolated_sum` is derived.
 
 Headline latency is the p99 of the per-iteration cross-rank MAX (`p50` is emitted alongside it, and
-`summarize.py` prints both; the p99 is the figure the published cohorts rank on). MAX is the
+`summarize.py` prints both; the p99 is the figure the published cohorts rank on). That is not in
+tension with the guidance below to rank by hand on p50: the published cohorts do not order cells by
+raw p99, they group them into bootstrap equivalence bands, so a cell whose p99 is dominated by
+worst-rank stalls rather than transport lands in a tie band instead of being declared a winner or a
+loser. Reading a single pair of cells yourself has no such machinery, which is why the bracket below
+is the manual procedure. MAX is the
 reduction because a layer is not finished until
 its slowest rank is, so MAX is the completion cost, and it charges inter-rank entry stagger to
 whichever component the ranks entered unevenly. How much stagger there is depends on the code
@@ -270,8 +281,9 @@ Logical payload bandwidth is:
 
 Payload bytes use rank-deduplicated token-rank activations and exclude expert metadata,
 padding, and backend buffer capacity. BF16 moves 2 bytes per value with no scale payload; an FP8
-dispatch moves 1 byte per value, plus per-128-block FP32 scales for DeepEP's and UCCL-EP's blockwise
-codec (none for MoRI's plain e4m3 cast), while combine stays BF16 — so the dispatch and combine directions can carry
+dispatch moves 1 byte per value, plus per-128-block FP32 scales for every blockwise codec here —
+DeepEP V2, UCCL-EP and FlashInfer EP, which carries them as a fourth dispatch payload — and none for
+MoRI's plain e4m3 cast, while combine stays BF16 — so the dispatch and combine directions can carry
 different byte counts and the roundtrip is their per-field sum. The rank-deduplicated count is exact
 for the normal-mode layout. It is also exact for a low-latency kernel that deduplicates per rank
 (MoRI's `IntraNodeLL`, whose combine is an unweighted rank-sum). The low-latency kernels that apply
diff --git a/experimental/CollectiveX/tests/test_roundtrip_staging.py b/experimental/CollectiveX/tests/test_roundtrip_staging.py
index e2fe6d6b8..a87fbd5b7 100644
--- a/experimental/CollectiveX/tests/test_roundtrip_staging.py
+++ b/experimental/CollectiveX/tests/test_roundtrip_staging.py
@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 """Contract for what the chained roundtrip measures.
 
-`stage` is not an FP8-only cost: deepep-v2 and uccl-ep set `stage_device_work = self._fp8`, but
-MoRI sets `self._fp8 or not self._external_input` and FlashInfer sets it unconditionally, so BF16
-rows on those two do real device work there. Charging it to the chained roundtrip therefore made
+`stage` is not an FP8-only cost: deepep-v2, uccl-ep and MoRI all set
+`stage_device_work = self._fp8` (MoRI's `self._fp8 or not self._external_input` collapses to that,
+now that it always uses an external input buffer), but FlashInfer sets it unconditionally, so its
+BF16 rows do real device work there. Charging it to the chained roundtrip therefore made
 `roundtrip` mean different things in different rows. Real stacks decide
 this on quant-format match: SGLang's DeepEP dispatcher contains no dequant at all, and vLLM
 returns the dispatched fp8 + scales untouched when `block_k == DEEPEP_QUANT_BLOCK_SIZE`,

From c59956b4e276a66d4ba9f66bd2e55d70f8d158f0 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 4 Aug 2026 13:31:59 +0800
Subject: [PATCH 27/34] =?UTF-8?q?CollectiveX:=20correct=20the=20README's?=
 =?UTF-8?q?=20FP8=20coverage=20and=20state=20the=20roundtrip=20contract=20?=
 =?UTF-8?q?/=20CollectiveX=EF=BC=9A=E4=BF=AE=E6=AD=A3=20README=20=E7=9A=84?=
 =?UTF-8?q?=20FP8=20=E8=A6=86=E7=9B=96=E8=8C=83=E5=9B=B4=E5=B9=B6=E5=86=99?=
 =?UTF-8?q?=E6=98=8E=20roundtrip=20=E7=BA=A6=E5=AE=9A?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The README carried the same stale-coverage error a review had already found in methodology.md, in the
entry-point document rather than the detailed one:

- It listed FP8 support as "DeepEP V2, MoRI, UCCL-EP", omitting FlashInfer EP, and its FlashInfer row
  still said "BF16 only". FlashInfer FP8 landed in f78330adb and is green on-metal at EP8 and EP16.
  The row now describes the shape that matters to a reader: caller-prequantized blockwise e4m3fn as a
  fourth dispatch payload with per-128-block FP32 scales, and the combine plane forced to BF16 —
  which is mandatory, not cosmetic, because the C++ `toNvDataType` accepts only fp16/bf16/fp32 for
  combine, so an FP8 combine buffer raises instead of corrupting.
- It never mentioned `stage` or the fact that `roundtrip` excludes it, which is THE contract change of
  this branch. A reader starting here would have had no way to know that rows published before sweep
  version 2 carried the staging copy inside the chain for MoRI BF16 and FlashInfer BF16, i.e. that
  `roundtrip` meant different things in different rows. Now stated, with the FP8-stage-is-scaffolding
  point and the do-not-sum-or-compare warning, pointing at methodology.md for the full contract.

Also noted that the caller-side FP8 quantize is charged inside the measured dispatch, since the
README describes the precision sweep and that is a load-bearing property of those numbers.

Checked and left alone: "NCCL EP is BF16-only" is still true, and the "32 synchronized full roundtrip
warmups" description still holds — the warm-up hoist changed what staging those warmups repeat, not
that there are 32 full roundtrips.

106 tests pass.

中文：README 中存在与评审此前在 methodology.md 中发现的同一处覆盖范围过期错误，而且出现在入口文档而
非细节文档中：

- 它把 FP8 支持列为"DeepEP V2、MoRI、UCCL-EP"，遗漏了 FlashInfer EP，且其 FlashInfer 条目仍写作
  "BF16 only"。FlashInfer 的 FP8 于 f78330adb 落地，并已在实机 EP8 与 EP16 上通过。该条目现描述了对
  读者重要的形态：调用方预量化的分块 e4m3fn 作为第四个 dispatch 载荷，附带每 128 块的 FP32 scale，
  且 combine 平面强制为 BF16——这是必需而非修饰，因为 C++ 的 `toNvDataType` 对 combine 仅接受
  fp16/bf16/fp32，因此 FP8 的 combine 缓冲会直接抛错而不会产生静默错误。
- 它完全没有提到 `stage`，也没有说明 `roundtrip` 将其排除，而这正是本分支的核心约定变更。从此处开始
  阅读的人无从得知：在 sweep 版本 2 之前发布的行，对 MoRI BF16 与 FlashInfer BF16 是把 staging 拷贝
  计入链路内的，即 `roundtrip` 在不同行中含义不同。现已写明，并附上"FP8 的 stage 属于测量脚手架"这一
  要点与"不可求和、不可跨后端比较"的警示，并指向 methodology.md 获取完整约定。

同时说明调用方侧的 FP8 量化计入被测 dispatch 之内，因为 README 描述了精度扫描，而这是这些数字的关键
性质。

已核查并保留不变："NCCL EP 为 BF16 only"仍然成立；"32 次同步完整 roundtrip 预热"的描述亦仍成立——
预热 hoist 改变的是这些预热重复的 staging，而非仍有 32 次完整 roundtrip 这一事实。

106 项测试通过。
---
 experimental/CollectiveX/README.md | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 7715624d5..905d98f60 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -14,8 +14,10 @@ responsibility. The full measurement methodology is in [docs/methodology.md](doc
 The workload uses packed placement and one pinned `fixed-profile` resource configuration per
 backend/topology; there is no tuning sweep. Combine is always BF16; dispatch precision is a swept
 dimension — a BF16 control plus, on every backend whose FP8 dispatch is supported upstream
-(DeepEP V2, MoRI, UCCL-EP), an FP8 dispatch, caller-prequantized in `normal` mode (in `low-latency`
-the DeepEP and UCCL-EP kernels quantize internally from BF16; MoRI stays caller-prequantized). NCCL
+(DeepEP V2, MoRI, UCCL-EP, FlashInfer EP), an FP8 dispatch, caller-prequantized in `normal` mode (in
+`low-latency` the DeepEP and UCCL-EP kernels quantize internally from BF16; MoRI stays
+caller-prequantized; FlashInfer has no `low-latency` path). That caller-side quantize is charged
+inside the measured dispatch, because a production forward pass pays it on the critical path. NCCL
 EP is BF16-only this release, so it emits the control alone. Coverage is uniform routing only. Cases
 run in one of two modes:
 
@@ -50,6 +52,15 @@ every position in the sequence; each iteration takes the cross-rank maximum befo
 p50/p90/p95/p99, and roundtrip p99 is the headline latency. A keyed BLAKE2b counter produces
 byte-identical routing and gate weights on every runtime.
 
+`roundtrip` means dispatch then combine — the transport — in every row. Expert-output staging sits
+outside it and is reported separately as `stage`; under FP8 that component is harness scaffolding
+standing in for the expert GEMM, which in production consumes FP8 operands natively rather than
+materialising a BF16 copy, so `stage` must not be summed into a total or compared between backends.
+That was not always true: rows published before sweep version 2 carried the staging copy inside the
+chain for MoRI BF16 and FlashInfer BF16, so `roundtrip` meant different things in different rows.
+`implementation.stage_excluded_from_roundtrip` distinguishes the generations from the artifact alone.
+See [docs/methodology.md](docs/methodology.md) for the full contract.
+
 Correctness is checked against an implementation-independent oracle that reproduces the backend's
 two-level reduction — intra-scale-up-domain FP32, then a BF16 cast of each domain's partial for the
 scale-out send. The combine gate is a tight max elementwise relative error below `8 * 2^-8`
@@ -80,7 +91,7 @@ scale-up domain.
 | MoRI | `production` — vLLM `--all2all-backend mori_*`, SGLang `--moe-a2a-backend mori` | `normal` mode uses the direct `IntraNode` kernel for scale-up EP8 on every CDNA SKU. EP16 is an unsupported coverage row on all three: the adapter pins `InterNodeV1` over 2x8 XGMI + RDMA, but its combine corrupts at the transport layer (ROCm/mori#475), so the registry ships `mori: [8]` and no EP16 case is dispatched. `low-latency` mode selects the `IntraNodeLL` decode kernel (single-call, pure-intranode, same compact layout and unweighted combine as `IntraNode`), decode/EP8 only. FP8 dispatch is caller-prequantized (per-SKU e4m3fnuz on gfx942, e4m3fn on gfx950); combine stays BF16 (`quant_type=none`) alongside BF16 dispatch |
 | UCCL-EP | `candidate` — no engine exposes a UCCL-EP selector | [UCCL](https://github.com/uccl-project/uccl) EP: a drop-in, API-identical DeepEP replacement whose CPU proxies issue GPUDirect RDMA over plain `libibverbs` (no NVSHMEM/IBGDA), with software message ordering, atomics, and flow control; scale-up is single-node `cudaIpc` over NVLink/XGMI (never MNNVL). `normal` mode is the legacy `Buffer` `dispatch`/`combine` (unweighted rank-sum); `low-latency` reuses the legacy `low_latency_dispatch`/`low_latency_combine` decode kernels (weighted combine), decode/EP8 only. FP8 dispatch is caller-prequantized in `normal` mode (blockwise e4m3fn, per-SKU e4m3fnuz on gfx942); in `low-latency` mode the caller sends BF16 and the decode kernel quantizes to e4m3 internally (`use_fp8`). Combine is BF16. Runs on NVIDIA and AMD (H100/H200/B200 + MI300X/MI325X/MI355X), EP8 scale-up. Cross-node EP16 is functional (the internode RDMA path connects and the light case passes correctness) but its CPU-proxy throughput overruns the standardized per-case wall-clock budget on heavy token counts, so EP16 is an unsupported coverage row for now |
 | NCCL EP | `candidate` — NVIDIA's own library, but no engine exposes an NCCL-EP selector | [NCCL EP](https://github.com/NVIDIA/nccl/tree/master/contrib/nccl_ep): NVIDIA's native MoE dispatch/combine on the NCCL Device API — LSA (NVLink load/store) intra-node, GIN (GPU-Initiated Networking) inter-node — driven through the `nccl4py` bindings. `normal` mode selects the `HIGH_THROUGHPUT` algorithm (FLAT `[N, hidden]` receive, unweighted rank-sum combine); the `LOW_LATENCY` algorithm carries an EP8 `ll_backends` row on all six NVIDIA SKUs, restored once the single-handle fix removed the NVIDIA/nccl#2303 signal aliasing. BF16 only: `contrib/nccl_ep/RELEASE.md` says "No FP8 support", so no FP8 case is emitted. That note is worth re-testing rather than trusting — the C library at our pinned commit does read `inputs->scales` and switch on e4m3/e5m2, the two documented FP8 exclusions are expert-major layouts we do not use, and `NVIDIA/nccl` has not moved since 2026-06-11 while `NVIDIA/nccl-extensions` has replaced that row outright. NVIDIA-only and CUDA 13 only. EP8 scale-up on H100/H200/B200/B300 plus EP8 and EP16 on GB200/GB300, where EP16 stays inside the MNNVL scale-up domain. x86 EP16 scale-out is an unsupported coverage row: the cross-node GIN path faults inside `nccl_ep.cc` identically on RoCE and IB across four SKUs, a GDAKI limit rather than a fabric-selection one |
-| FlashInfer EP | `production` — vLLM `--all2all-backend flashinfer_nvlink_one_sided` | [FlashInfer](https://github.com/flashinfer-ai/flashinfer) `MoeAlltoAll`: TensorRT-LLM's one-sided MNNVL all-to-all, where each rank writes tokens straight into its peers' workspace windows and combine reads them back — no send/recv pairing and no NVSHMEM. `normal` mode only (there is one kernel family; no separate decode path), BF16 only, and GB200/GB300 only, since the transport is MNNVL. EP8 and EP16, both inside the scale-up domain. Unlike every other backend here, its combine accumulates in the PAYLOAD dtype rather than FP32: wheels before 0.6.16 reduce the top-k contributions with a pairwise BF16 tree that rounds at every level, so the oracle models that reduction directly (`combine_reduction = "topk-slot-tree"`) instead of widening the tolerance. 0.6.16 moved the accumulator to FP32, and the adapter switches models on the installed version |
+| FlashInfer EP | `production` — vLLM `--all2all-backend flashinfer_nvlink_one_sided` | [FlashInfer](https://github.com/flashinfer-ai/flashinfer) `MoeAlltoAll`: TensorRT-LLM's one-sided MNNVL all-to-all, where each rank writes tokens straight into its peers' workspace windows and combine reads them back — no send/recv pairing and no NVSHMEM. `normal` mode only (there is one kernel family; no separate decode path), and GB200/GB300 only, since the transport is MNNVL. FP8 dispatch is caller-prequantized blockwise e4m3fn, carried as a fourth dispatch payload alongside its per-128-block FP32 scales, with the combine plane forced to BF16 — the C++ `toNvDataType` accepts only fp16/bf16/fp32 for combine, so an FP8 combine buffer would raise rather than corrupt. EP8 and EP16, both inside the scale-up domain. Unlike every other backend here, its combine accumulates in the PAYLOAD dtype rather than FP32: wheels before 0.6.16 reduce the top-k contributions with a pairwise BF16 tree that rounds at every level, so the oracle models that reduction directly (`combine_reduction = "topk-slot-tree"`) instead of widening the tolerance. 0.6.16 moved the accumulator to FP32, and the adapter switches models on the installed version |
 
 DeepEP V2 means the `ElasticBuffer` implementation introduced by
 [DeepEP PR #605](https://github.com/deepseek-ai/DeepEP/pull/605), not a newer legacy `Buffer` build.

From a7fc2ef119498d5a05a78b901fc8beb00b8a745a Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 4 Aug 2026 22:37:30 +0800
Subject: [PATCH 28/34] CollectiveX: keep the sweep at version 1
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The timing-window change altered what `roundtrip` means, which is normally a reason to bump the
sweep version so two generations can never be mixed in one comparison. Keeping it at 1 instead, so
this lands without requiring the frontend reader to accept a new generation first — the reader SKIPS
runs whose version it does not list, so a version bump merged ahead of the frontend would make new
runs silently invisible on the dashboard rather than fail loudly.

The consequence is stated rather than hidden: the version tag no longer separates the generations, so
`implementation.stage_excluded_from_roundtrip` and whether a `stage` component is present are the only
discriminators. Both the README and methodology.md now say that explicitly, in place of the sentences
that pointed at "sweep version 2".

Worth being clear about what this trades away: rows measured before this branch and rows measured
after it will both carry `version: 1` while `roundtrip` means something different in each — for MoRI
BF16 and FlashInfer BF16 the older rows include the staging copy, and FP8 `normal` rows now charge the
caller-side quantize inside dispatch. Anything reading the durable store across that boundary needs to
key on the two fields above, not on the version.

106 tests pass.
---
 experimental/CollectiveX/README.md           | 7 ++++---
 experimental/CollectiveX/configs/sweep.json  | 2 +-
 experimental/CollectiveX/docs/methodology.md | 5 ++++-
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 905d98f60..7f5738458 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -56,9 +56,10 @@ byte-identical routing and gate weights on every runtime.
 outside it and is reported separately as `stage`; under FP8 that component is harness scaffolding
 standing in for the expert GEMM, which in production consumes FP8 operands natively rather than
 materialising a BF16 copy, so `stage` must not be summed into a total or compared between backends.
-That was not always true: rows published before sweep version 2 carried the staging copy inside the
-chain for MoRI BF16 and FlashInfer BF16, so `roundtrip` meant different things in different rows.
-`implementation.stage_excluded_from_roundtrip` distinguishes the generations from the artifact alone.
+That was not always true: rows measured before this change carried the staging copy inside the chain
+for MoRI BF16 and FlashInfer BF16, so `roundtrip` meant different things in different rows. The sweep
+`version` stays 1 across the change, so `implementation.stage_excluded_from_roundtrip` and whether a
+`stage` component is present are the only way to tell the two generations apart.
 See [docs/methodology.md](docs/methodology.md) for the full contract.
 
 Correctness is checked against an implementation-independent oracle that reproduces the backend's
diff --git a/experimental/CollectiveX/configs/sweep.json b/experimental/CollectiveX/configs/sweep.json
index 89313eab8..22c873e9f 100644
--- a/experimental/CollectiveX/configs/sweep.json
+++ b/experimental/CollectiveX/configs/sweep.json
@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 1,
   "suite": "ep-core",
   "modes": {
     "normal": ["decode", "prefill"],
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 34a69c0ef..d101563ba 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -30,7 +30,10 @@ it is charged **inside the measured dispatch** rather than prepared ahead of the
 DeepEP V2, UCCL-EP and FlashInfer EP it is issued as one fused kernel and guarded bitwise against its
 eager reference; MoRI needs neither, because its quantize is a single plain dtype cast. This means an FP8
 `normal` dispatch number covers quantize-plus-transport while its BF16 control covers transport
-alone, and it is why FP8 `normal` rows are not comparable to runs published before sweep version 2.
+alone, and it is why an FP8 `normal` row is not comparable to one measured before that change. The
+sweep `version` deliberately stays 1 across it, so the version tag does NOT separate the two
+generations — `implementation.stage_excluded_from_roundtrip` and the presence of a `stage` component
+are the only discriminators, and a consumer comparing rows across that boundary has to key on them.
 
 Read that charge as a **fixed per-call cost, not a payload-proportional one**, or the FP8-versus-BF16
 comparison will be misread at the bottom of the ladder. Measured on DeepEP V2 decode, FP8 dispatch p50

From 7e8c2293ebcb9ce405922577d5806b62381492b5 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 5 Aug 2026 01:39:28 +0800
Subject: [PATCH 29/34] CollectiveX: let low-latency use MNNVL, cover it on GB,
 and measure steady-state period
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Three changes, the first a plain bug.

**The low-latency path was disabling the fabric it should have used.** `_create_ll_buffer` passed
`allow_nvlink_for_low_latency_mode=True` but never passed `allow_mnnvl`, so the legacy Buffer took its
default of False — and a False there self-sets NVSHMEM_DISABLE_MNNVL. On GB200/GB300, whose scale-up
transport IS MNNVL, that forced the decode kernels onto IBGDA and measured the rack's slow path. Now
keyed on `scale_up_transport == "mnnvl"`, a topology fact the adapter already receives, and passed only
if the pinned wheel accepts the keyword — raising rather than silently running over IBGDA if it does
not.

**deepep-v2 low-latency was never dispatched on GB at all.** gb200/gb300 carried
`ll_backends = {"nccl-ep": [8]}`, so this is new coverage rather than a change to any published
number: 8 shards, EP8 and EP16, both precisions. EP16 is included because it stays inside the NVL72
domain with no scale-out, exactly as flashinfer-ep and nccl-ep already do EP16 there.

**A new `period` component, because `roundtrip` answers a different question than decode serving
asks.** `roundtrip` drains the GPU around every pair, so it measures the latency of an idle pipeline.
A decode loop runs dispatch->combine->dispatch->combine without stopping, and its per-layer cost is
the pipeline's period — smaller than the sum of separately-drained stages, and indifferent to how
inter-rank entry stagger gets attributed, which is the effect behind the falling small-T curves on
b200 uccl-ep low-latency. Added alongside the existing components, never replacing them, so nothing
already published changes meaning.

It is opt-in per backend (`pipeline_pairs`, default 0) for a specific reason. Issuing pairs
back-to-back lets ranks drift, and dispatch is a peer WRITE into another rank's buffer: stream order
on the receiver does not order the sender's remote writes, so overlap is only sound where the receive
buffer can absorb the drift. A collective bounds that drift to about one iteration — a dispatch cannot
complete until every rank enters it — so DeepEP's low-latency receive, double-buffered with a parity
that flips per dispatch, covers it; a single shared buffer would not. That is the same two-micro-batch
overlap SGLang and vLLM run. Only that path opts in; enabling it elsewhere would give a fast number
over corrupted data, which is why the default is off and a row without the component simply did not
measure it.

Three tests pin the opt-in: off by default, present when declared, and absent at `pipeline_pairs = 1`
since a single pair is not a pipeline and must not advertise a second name for `roundtrip`.

109 tests pass.
---
 experimental/CollectiveX/bench/ep_backend.py  | 45 +++++++++++++++++++
 .../CollectiveX/bench/ep_deepep_v2.py         | 25 +++++++++++
 experimental/CollectiveX/bench/ep_harness.py  |  6 ++-
 .../CollectiveX/configs/platform_config.json  |  4 +-
 experimental/CollectiveX/docs/methodology.md  | 17 +++++++
 .../tests/test_roundtrip_staging.py           | 30 +++++++++++++
 6 files changed, 124 insertions(+), 3 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_backend.py b/experimental/CollectiveX/bench/ep_backend.py
index 6882a279f..43b830f3f 100644
--- a/experimental/CollectiveX/bench/ep_backend.py
+++ b/experimental/CollectiveX/bench/ep_backend.py
@@ -386,6 +386,18 @@ def _topk_idx_dtype(self):
 
     # ---- Timing template methods -----------------------------------------------------
 
+    # Pairs issued back-to-back per `period` sample; 0 disables the component. A serving decode
+    # loop runs dispatch->combine->dispatch->combine without stopping, so its cost per layer is
+    # the pipeline's PERIOD, not the sum of separately-drained stages -- `roundtrip` measures the
+    # latter and overstates the former. Opt-in rather than default because the overlap is only
+    # sound where the backend tolerates rank drift: dispatch is a peer WRITE into another rank's
+    # buffer, and stream order on the receiver does not order the sender's remote writes. The
+    # collective bounds that drift to roughly one iteration (a dispatch cannot complete until
+    # every rank enters it), so a backend whose receive buffer is double-buffered per dispatch is
+    # safe and one with a single shared buffer is not. Enabling it for a backend that is not
+    # means a fast number over corrupted data, which is why the default is off.
+    pipeline_pairs = 0
+
     def timed_components(self):
         """Components measured for this backend: roundtrip always; the rest unless
         the backend exposes only a stateful paired round trip."""
@@ -394,6 +406,8 @@ def timed_components(self):
             components.extend(["dispatch", "combine"])
             if self.stage_device_work:
                 components.append("stage")
+        if self.pipeline_pairs > 1:
+            components.append("period")
         return components
 
     def warm(self, problem, count, stage_every=False):
@@ -442,10 +456,41 @@ def run_roundtrip(self, problem, staged=None):
             setattr(handle, self.combine_input_attr, staged)
         return self.combine(problem, handle)
 
+    def benchmark_period(self, problem, warmup, iters):
+        """Steady-state cost per dispatch->combine pair, pairs issued back-to-back.
+
+        `roundtrip` drains the GPU around every pair, so it reports the latency of an idle
+        pipeline and charges inter-rank entry stagger to whichever component the ranks entered
+        unevenly. A decode loop never stops between layers, so what it pays per layer is this
+        period. The two are different quantities, not competing estimates of one: quote
+        `roundtrip` for how long a single collective takes and `period` for what a continuous
+        stream costs, and never sum or compare them across backends -- only backends that opt in
+        via `pipeline_pairs` report it at all.
+        """
+        import torch
+
+        self.warm(problem, warmup)
+        pairs = max(2, int(self.pipeline_pairs))
+        samples = []
+        for _ in range(iters):
+            start = torch.cuda.Event(enable_timing=True)
+            end = torch.cuda.Event(enable_timing=True)
+            start.record()
+            for _ in range(pairs):
+                handle = self.dispatch(problem)
+                self.stage(problem, handle)
+                self.combine(problem, handle)
+            end.record()
+            torch.cuda.synchronize()
+            samples.append(start.elapsed_time(end) * 1000.0 / pairs)
+        return samples
+
     def benchmark_component(self, component, problem, warmup, iters):
         """Measure one named component; every component gets the same warm-up first."""
         if component == "roundtrip":
             return self.benchmark_roundtrip(problem, warmup, iters)
+        if component == "period":
+            return self.benchmark_period(problem, warmup, iters)
         if component == "dispatch":
             return self.benchmark_dispatch(problem, warmup, iters)
         if component == "stage":
diff --git a/experimental/CollectiveX/bench/ep_deepep_v2.py b/experimental/CollectiveX/bench/ep_deepep_v2.py
index 5be2ac44f..ce019f7e2 100644
--- a/experimental/CollectiveX/bench/ep_deepep_v2.py
+++ b/experimental/CollectiveX/bench/ep_deepep_v2.py
@@ -186,6 +186,13 @@ def create_buffer(self, spec):
         self.max_tokens = spec.max_tokens_per_rank
         if self.mode == "low-latency":
             self._create_ll_buffer(spec)
+            # The legacy LL receive is double-buffered with a parity that flips per dispatch,
+            # and a collective bounds rank drift to about one iteration (a dispatch cannot
+            # complete until every rank enters it), so two parities cover the worst overlap and
+            # pairs may be issued back-to-back. This is the pattern SGLang and vLLM use for
+            # two-micro-batch overlap. Only this path opts in; the ElasticBuffer normal-mode
+            # receive is not double-buffered that way.
+            self.pipeline_pairs = 8
             return
         _require_runtime()
         jit_root = Path(os.environ["EP_JIT_CACHE_DIR"])
@@ -255,6 +262,23 @@ def _create_ll_buffer(self, spec):
         num_rdma_bytes = deep_ep.Buffer.get_low_latency_rdma_size_hint(
             self.max_tokens, args.hidden, world_size, args.experts
         )
+        kwargs = {}
+        # On an MNNVL rack the scale-up fabric IS NVLink across trays, but the legacy Buffer
+        # defaults `allow_mnnvl=False` and a False there self-sets NVSHMEM_DISABLE_MNNVL --
+        # so leaving it unset forced the low-latency kernels onto IBGDA on exactly the systems
+        # whose fast path is MNNVL, and measured the rack's slow path. Keyed on the topology
+        # the platform already reports rather than on the SKU name. Passed only when the pinned
+        # wheel accepts it, so an older deep_ep keeps working instead of raising on an unknown
+        # keyword.
+        if str(getattr(args, "scale_up_transport", "")) == "mnnvl":
+            import inspect
+            if "allow_mnnvl" in inspect.signature(deep_ep.Buffer.__init__).parameters:
+                kwargs["allow_mnnvl"] = True
+            else:
+                raise RuntimeError(
+                    "MNNVL scale-up needs deep_ep.Buffer(allow_mnnvl=...); this wheel lacks it, "
+                    "so the low-latency path would silently run over IBGDA"
+                )
         self.buffer = deep_ep.Buffer(
             self.group,
             num_rdma_bytes=num_rdma_bytes,
@@ -262,6 +286,7 @@ def _create_ll_buffer(self, spec):
             num_qps_per_rank=num_qps_per_rank,
             allow_nvlink_for_low_latency_mode=True,
             explicitly_destroy=True,
+            **kwargs,
         )
 
     def _ll_recv_bf16(self, recv_x):
diff --git a/experimental/CollectiveX/bench/ep_harness.py b/experimental/CollectiveX/bench/ep_harness.py
index 12889d6e8..695ba1740 100644
--- a/experimental/CollectiveX/bench/ep_harness.py
+++ b/experimental/CollectiveX/bench/ep_harness.py
@@ -962,6 +962,7 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
     stage_pool = {T: [] for T in ladder}    # measured only when stage launches device work
     comb_pool = {T: [] for T in ladder}     # ... combine
     rt_pool = {T: [] for T in ladder}       # independently measured round trip
+    period_pool = {T: [] for T in ladder}   # steady-state per-pair cost, opt-in backends only
     spread_pool = {T: [] for T in ladder}   # cross-rank (max-min) of the round trip, per iter
     # Cross-rank MIN per component. The LAST rank to enter a collective is the one that waited
     # least -- it started when its peers were already there -- so its duration is the closest
@@ -977,7 +978,7 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
             # timed_components() encodes the roundtrip-only vs full-component contract
             # (and whether stage launches device work) once, in the base class.
             component_order = trial_order(backend.timed_components(), trial_index)
-            measured = {name: [] for name in ("dispatch", "stage", "combine", "roundtrip")}
+            measured = {name: [] for name in ("dispatch", "stage", "combine", "roundtrip", "period")}
             for component_name in component_order:
                 # The base template gives every component the same synchronized
                 # full-roundtrip warm-up before its timed trial and encodes the two
@@ -993,6 +994,8 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
                 stage_pool[T] += _reduce_vec(torch, dist, device, measured["stage"], MAX)
             rt_max = _reduce_vec(torch, dist, device, measured["roundtrip"], MAX)
             rt_pool[T] += rt_max
+            if measured["period"]:
+                period_pool[T] += _reduce_vec(torch, dist, device, measured["period"], MAX)
             # Cross-rank SPREAD (max-min) of the same iterations. A collective cannot finish
             # before its slowest participant, so when ranks enter together every rank measures
             # nearly the same duration and the spread is small; a large spread means the ranks
@@ -1091,6 +1094,7 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
                 "dispatch": _component(dp, len(d)),
                 "isolated_sum": _component(isum, 0, derived=True),
                 "roundtrip": _component(rtp, len(rt)),
+                "period": _component(_pcts(period_pool[T]), len(period_pool[T])),
                 "stage": _component(sp, len(s)),
             },
             # Skew-excluded companion to `components`: same iterations reduced with cross-rank
diff --git a/experimental/CollectiveX/configs/platform_config.json b/experimental/CollectiveX/configs/platform_config.json
index 7e23f2d41..c6f5d1fb5 100644
--- a/experimental/CollectiveX/configs/platform_config.json
+++ b/experimental/CollectiveX/configs/platform_config.json
@@ -104,7 +104,7 @@
       "scale_up_transport": "mnnvl",
       "launcher": "gb-nv",
       "backends": {"deepep-v2": [8, 16], "nccl-ep": [8, 16], "flashinfer-ep": [8, 16]},
-      "ll_backends": {"nccl-ep": [8]},
+      "ll_backends": {"deepep-v2": [8, 16], "nccl-ep": [8]},
       "fabric": {"nic": "MNNVL (scale-out not used)", "switch": "NVLink NVL72"},
       "operator": {
         "partition": "batch",
@@ -124,7 +124,7 @@
       "scale_up_transport": "mnnvl",
       "launcher": "gb-nv",
       "backends": {"deepep-v2": [8, 16], "nccl-ep": [8, 16], "flashinfer-ep": [8, 16]},
-      "ll_backends": {"nccl-ep": [8]},
+      "ll_backends": {"deepep-v2": [8, 16], "nccl-ep": [8]},
       "fabric": {"nic": "MNNVL (scale-out not used)", "switch": "NVLink NVL72"},
       "operator": {
         "partition": "batch_1",
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index d101563ba..7a81a9f9c 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -224,6 +224,23 @@ per-iteration spread is ~9.3 us for deepep-v2 and uccl-ep at BF16 (they share th
 FP8, where the kernel quantises in-kernel and the heavier dispatch self-aligns the ranks. So the
 term is not subtractable in any principled way, and MAX alone taxes some rows more than others.
 
+Some rows also carry a `period` component, and it answers a different question from `roundtrip`.
+`roundtrip` drains the GPU around each pair, so it reports the latency of an idle pipeline. A decode
+loop never stops between layers — the next dispatch is already in flight while the previous combine's
+stragglers land — so what a serving stack pays per layer is the pipeline's PERIOD, which is smaller
+than the sum of separately-drained stages and is also indifferent to how inter-rank entry stagger gets
+attributed. Both are real; quote `roundtrip` for how long one collective takes and `period` for what a
+continuous stream costs, and never sum them or treat one as a correction to the other.
+
+`period` is opt-in per backend (`pipeline_pairs`) rather than universal, because issuing pairs
+back-to-back lets ranks drift, and dispatch is a peer WRITE into another rank's buffer — stream order
+on the receiver does not order the sender's remote writes. A collective bounds that drift to roughly
+one iteration, since a dispatch cannot complete until every rank enters it, so a receive buffer that
+is double-buffered per dispatch is safe and one shared buffer is not. Today only DeepEP V2's
+low-latency path opts in, which is the same two-micro-batch overlap SGLang and vLLM run. Enabling it
+where the buffer cannot absorb the drift would produce a fast number over corrupted data, so it is off
+by default and a row without the component simply did not measure it.
+
 Every row therefore also carries `cross_rank_min_us` (the same iterations reduced with MIN — the
 skew-excluded floor) and `cross_rank_spread_us` (per-iteration MAX minus MIN). Read MAX and MIN
 as a bracket. Two cells whose MAX gap is smaller than the larger contender's spread are not
diff --git a/experimental/CollectiveX/tests/test_roundtrip_staging.py b/experimental/CollectiveX/tests/test_roundtrip_staging.py
index a87fbd5b7..5a8071d40 100644
--- a/experimental/CollectiveX/tests/test_roundtrip_staging.py
+++ b/experimental/CollectiveX/tests/test_roundtrip_staging.py
@@ -192,6 +192,36 @@ def test_a_chain_that_includes_staging_keeps_warming_it(self):
         self._warm(b, 5)
         self.assertEqual(b.calls.count("stage"), 5)
 
+class SteadyStatePeriod(unittest.TestCase):
+    """`period` is opt-in, because the overlap it measures is only sound for some backends.
+
+    A decode loop never stops between layers, so its per-layer cost is the pipeline's period,
+    not the sum of separately-drained stages. Measuring that means issuing pairs back-to-back,
+    which lets ranks drift — and dispatch is a peer WRITE into another rank's buffer, so stream
+    order on the receiver does not order the sender's remote writes. A double-buffered receive
+    covers the ~one iteration of drift a collective permits; a single shared buffer does not.
+    Defaulting this on would produce a fast number over corrupted data.
+    """
+
+    def test_off_by_default_so_no_backend_pipelines_accidentally(self):
+        b = _StubBackend(stage_device_work=False, fp8_consume="native")
+        self.assertEqual(b.pipeline_pairs, 0)
+        self.assertNotIn("period", b.timed_components())
+
+    def test_declaring_pairs_adds_the_component(self):
+        b = _StubBackend(stage_device_work=False, fp8_consume="native")
+        b.pipeline_pairs = 8
+        self.assertIn("period", b.timed_components())
+        # and it never displaces the drained latency measurement
+        self.assertIn("roundtrip", b.timed_components())
+
+    def test_a_single_pair_is_not_a_pipeline(self):
+        # pipeline_pairs = 1 measures exactly what roundtrip already does, so it must not
+        # advertise a second name for the same quantity.
+        b = _StubBackend(stage_device_work=False, fp8_consume="native")
+        b.pipeline_pairs = 1
+        self.assertNotIn("period", b.timed_components())
+
 
 if __name__ == "__main__":
     unittest.main()

From b368eae1d1e83bf8578c8e623d77357827477998 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 5 Aug 2026 10:33:08 +0800
Subject: [PATCH 30/34] CollectiveX: stop the deepep-v2 LL ladder below its
 receive cap

DeepEP's low-latency combine corrupts the 256 tokens/rank rung on every Blackwell SKU we
run -- B200, GB200 and GB300, EP8 and EP16, both precisions, MNNVL and RDMA alike -- while
Hopper stays clean. It is stochastic at roughly 1.5-3.3% per invocation and surfaces as one
wrong token row whose norm still matches, so the correctness gate catches it as a 0.07-6.6
relative error against a 0.03125 tolerance. Tracked upstream as DeepEP issue #700.

Clamp the measured low-latency ladder to 128 and leave the receive sized at 256. The two are
now separate constants on purpose: the receive footprint drives the transport's memory traffic
and the FP8 dequant volume, so sizing it from max(ladder) -- as it was -- would have halved it
the moment the ladder moved and shifted every retained rung out of comparability with the
published series. Holding it at 256 also leaves the top measured rung at half occupancy, which
is the ladder/capacity decoupling the original capacity probe had to hand-roll. The clamp is
not silent: the harness already reports every dropped ladder point into the artifact.

The likely upstream fix is DeepEP PR #642, which adds a CTA-scope fence so the combine
consumer's shared-memory reads retire before the stage is recycled and the producer's next TMA
load refills it -- that mechanism predicts the observed signature, and it closed #621, the same
race reached from NVL72. Our pin is the head of PR #605 and was branched before #642 merged, so
the fence is simply absent from our build. Raising the ladder back to 256 is therefore gated on
a pin bump, deliberately not bundled here: it spans months of upstream change, re-baselines
every deepep-v2 row including normal mode, and needs rewrite_deepep_v2 made tolerant first,
since main already carries the 'libnccl' fix that rewrite asserts it must apply.
---
 .../CollectiveX/bench/ep_deepep_v2.py         | 59 +++++++++++++++--
 experimental/CollectiveX/docs/methodology.md  | 14 +++++
 .../CollectiveX/tests/test_runtime.py         | 63 +++++++++++++++++++
 3 files changed, 131 insertions(+), 5 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_deepep_v2.py b/experimental/CollectiveX/bench/ep_deepep_v2.py
index ce019f7e2..b65796825 100644
--- a/experimental/CollectiveX/bench/ep_deepep_v2.py
+++ b/experimental/CollectiveX/bench/ep_deepep_v2.py
@@ -26,6 +26,18 @@
 # verifies the wheel's commit tag against the pin — it checks only that the loaded
 # deep_ep exposes ElasticBuffer (the from-source PR #605 capability).
 
+# Low-latency receive sizing. These are deliberately two numbers, not one: _LL_BUFFER_CAP
+# sizes the pre-allocated receive (and so fixes the transport footprint and the fp8 dequant
+# volume), while _LL_LADDER_CAP bounds which token counts are measured. See `buffer_cap` for
+# why the measured ladder stops below the buffer -- an upstream Blackwell combine defect at
+# the 256 rung -- and `create_buffer` for why the buffer must not follow the ladder down.
+_LL_BUFFER_CAP = 256
+_LL_LADDER_CAP = 128
+assert _LL_LADDER_CAP <= _LL_BUFFER_CAP <= 511, (
+    "the LL receive cap must fit NVSHMEM_QP_DEPTH=1024 ((cap + 1) * 2 <= 1024 => cap <= 511) "
+    "and the measured ladder must fit inside the buffer"
+)
+
 
 def _fp8_cast_helpers():
     """The pinned per-token FP8 cast pair (blockwise e4m3fn, per-128-block FP32 scale).
@@ -171,11 +183,34 @@ def __init__(self, args, rank, world_size, local_rank, device):
     def buffer_cap(self, args):
         if self.mode == "low-latency":
             # LL pre-allocates a fixed [num_local_experts, cap * num_ranks, hidden] receive
-            # buffer, so cap is a hard per-rank dispatch-slot bound (the harness clamps the
-            # decode ladder to it and reports the dropped point). 256 sits well under the
-            # default NVSHMEM_QP_DEPTH ceiling ((cap + 1) * 2 <= 1024 => cap <= 511 with
-            # NVSHMEM_QP_DEPTH=1024) and is adjustable if the decode ladder needs more.
-            return 256
+            # buffer, so the buffer cap is a hard per-rank dispatch-slot bound. The MEASURED
+            # ladder is clamped tighter than that buffer (the harness reports every dropped
+            # point, so the omission lands in the artifact rather than being silent) because
+            # DeepEP's low-latency combine corrupts at T=256 on every Blackwell SKU -- b200,
+            # gb200 and gb300, EP8 and EP16, both precisions, MNNVL and RDMA alike -- while
+            # Hopper stays clean. It is stochastic at roughly 1.5-3.3% per oracle invocation,
+            # so a passing leg proves nothing; the gate catches it as a 0.07-6.6 relative
+            # error against a 0.03125 tolerance. Tracked upstream as DeepEP issue #700.
+            #
+            # The likely fix already exists upstream and our pin simply predates it:
+            # PR #642 adds a CTA-scope `fence.proxy.async.shared::cta` before
+            # `mbarrier_arrive(empty_barriers[stage_idx])` in LOW_LATENCY_COMBINE_RECV, so the
+            # consumer's shared-memory reads retire before the stage is declared empty and the
+            # producer's next TMA load refills it. Signalling empty too early lets a row be
+            # assembled from two tiles, which is exactly the observed signature (one token row,
+            # norm preserved to 4 s.f., 16-40% of elements deviating). It closed #621, the same
+            # race reached from NVL72. COLLX_DEEPEP_V2_COMMIT is the head of PR #605, branched
+            # before #642 merged, so the fence is absent from our build; upstream main has both.
+            # An earlier on-metal test that appeared to rule fencing out used a device-scope
+            # __threadfence_system after the grid sync at internode_ll.cu:976 -- a different
+            # fence at a different site -- so it does not bear on #642.
+            #
+            # Raising this back to _LL_BUFFER_CAP is therefore gated on a pin bump, not on a
+            # new upstream release. That bump is deliberately NOT bundled here: it spans months
+            # of upstream change, re-baselines every deepep-v2 row including normal mode, and
+            # needs `rewrite_deepep_v2` made tolerant first (main already carries that
+            # 'libnccl' fix, so the rewrite's count(old) == 1 assertion would abort the stage).
+            return _LL_LADDER_CAP
         return None
 
     def create_buffer(self, spec):
@@ -185,6 +220,20 @@ def create_buffer(self, spec):
         args, world_size = self.args, self.world_size
         self.max_tokens = spec.max_tokens_per_rank
         if self.mode == "low-latency":
+            # Size the LL buffer from the fixed cap, NOT from the clamped ladder. Deriving it
+            # from max(ladder) would halve the receive tensor the moment the ladder was
+            # clamped, and the receive footprint sets both the transport's memory traffic and
+            # the fp8 dequant volume (`_ll_recv_bf16` converts the whole padded receive) -- so
+            # every retained rung's numbers would shift and stop being comparable with the
+            # published series. Holding the buffer at 256 keeps them bit-comparable and leaves
+            # the top measured rung at half occupancy, which is the ladder/capacity decoupling
+            # the earlier capacity probe had to hand-roll.
+            if spec.max_tokens_per_rank > _LL_BUFFER_CAP:
+                raise RuntimeError(
+                    f"low-latency ladder maximum {spec.max_tokens_per_rank} exceeds the LL "
+                    f"buffer cap {_LL_BUFFER_CAP}"
+                )
+            self.max_tokens = _LL_BUFFER_CAP
             self._create_ll_buffer(spec)
             # The legacy LL receive is double-buffered with a parity that flips per dispatch,
             # and a collective bounds rank drift to about one iteration (a dispatch cannot
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 7a81a9f9c..0a4e4456b 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -62,6 +62,20 @@ cells carry the control alone; the per-backend precision set lives in `sweep_mat
   T=1..512 powers of two and prefill T=1024..8192 powers of two. Ladders are model-specific and
   live with the workload in `configs/sweep.json`.
 
+A backend may clamp the ladder below that, and every clamped point is reported in the artifact
+rather than dropped silently. Exactly one backend clamps today: DeepEP V2 in `low-latency` mode
+pre-allocates a fixed receive, so its ladder cannot exceed that buffer, and it is currently held at
+**T=128**, one rung below the 256-slot receive, because DeepEP's low-latency combine corrupts the
+256 rung on every Blackwell SKU we run — B200, GB200 and GB300, EP8 and EP16, both precisions,
+MNNVL and RDMA alike, while Hopper stays clean. It is stochastic at roughly 1.5-3.3% per
+invocation and shows up as one wrong token row whose norm still matches, so it is a correctness
+gate failure rather than a crash (upstream DeepEP issue #700). The receive stays sized at 256
+even though the ladder stops at 128: its footprint drives both the transport's memory traffic and
+the FP8 dequant volume, so shrinking it with the ladder would move every retained rung and break
+comparability with the published series. The likely upstream fix (DeepEP PR #642, a CTA-scope
+fence so the combine consumer's shared-memory reads retire before the stage is recycled) landed
+after the commit we pin, so raising this back to 256 is gated on a pin bump.
+
 `sweep_matrix.py` materializes the requested SKUs, backends, EP sizes, and token ladders into a
 matrix document, then extracts strict per-shard controls. `--only-sku`, `--exclude-skus`,
 `--ep-sizes`, and `--precisions` select a subset; a subset produces a smaller matrix, not a
diff --git a/experimental/CollectiveX/tests/test_runtime.py b/experimental/CollectiveX/tests/test_runtime.py
index f1549de1a..23c859e18 100644
--- a/experimental/CollectiveX/tests/test_runtime.py
+++ b/experimental/CollectiveX/tests/test_runtime.py
@@ -873,5 +873,68 @@ def test_the_spread_is_none_when_unreadable(self):
                 self.assertTrue(result is None or result[2] < 10)
 
 
+class LowLatencyCapDecoupling(unittest.TestCase):
+    """The LL receive size and the measured ladder are two numbers and must stay two.
+
+    The measured ladder stops below the receive cap to skip a token count DeepEP's low-latency
+    combine corrupts on Blackwell (upstream #700; the fix is #642, which our pin predates). The
+    receive must NOT follow the ladder down: its footprint sets the transport's memory traffic
+    and the fp8 dequant volume, so sizing it from `max(ladder)` would shift every retained
+    rung and break comparability with the published series. Asserted over the source because
+    importing the adapter needs a built deep_ep.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        cls.tree = ast.parse((BENCH / "ep_deepep_v2.py").read_text())
+        cls.consts = {
+            t.id: node.value.value
+            for node in cls.tree.body
+            if isinstance(node, ast.Assign)
+            for t in node.targets
+            if isinstance(t, ast.Name) and isinstance(node.value, ast.Constant)
+        }
+
+    def _func(self, name):
+        for node in ast.walk(self.tree):
+            if isinstance(node, ast.FunctionDef) and node.name == name:
+                return node
+        self.fail(f"{name} not found in ep_deepep_v2.py")
+
+    def test_both_caps_exist_and_the_ladder_stops_strictly_below_the_buffer(self):
+        buf, ladder = self.consts.get("_LL_BUFFER_CAP"), self.consts.get("_LL_LADDER_CAP")
+        self.assertIsInstance(buf, int)
+        self.assertIsInstance(ladder, int)
+        # Strictly below: equality is the configuration that corrupts, and it also puts the top
+        # measured rung at 100% occupancy, which is what made capacity and last-rung
+        # indistinguishable in the original investigation.
+        self.assertLess(ladder, buf)
+        # NVSHMEM_QP_DEPTH=1024 asserts nvshmem_qp_depth >= (cap + 1) * 2 at construction.
+        self.assertLessEqual(buf, 511)
+
+    def test_buffer_cap_clamps_the_ladder_by_the_constant_not_a_literal(self):
+        returns = [
+            n.value for n in ast.walk(self._func("buffer_cap"))
+            if isinstance(n, ast.Return) and n.value is not None
+        ]
+        names = {n.id for n in returns if isinstance(n, ast.Name)}
+        self.assertIn("_LL_LADDER_CAP", names)
+        # A bare literal here would drift out of step with the constants above.
+        self.assertEqual([n for n in returns if isinstance(n, ast.Constant) and n.value is not None], [])
+
+    def test_the_low_latency_receive_is_sized_from_the_cap_not_the_ladder(self):
+        # Guards the regression that would silently re-baseline every LL row.
+        assigned = [
+            node.value for node in ast.walk(self._func("create_buffer"))
+            if isinstance(node, ast.Assign)
+            for t in node.targets
+            if isinstance(t, ast.Attribute) and t.attr == "max_tokens"
+        ]
+        self.assertTrue(
+            any(isinstance(v, ast.Name) and v.id == "_LL_BUFFER_CAP" for v in assigned),
+            "create_buffer must set self.max_tokens = _LL_BUFFER_CAP on the low-latency path",
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

From 90b16f7ae52bdef6df6d279cecb657d1bd051e4d Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 5 Aug 2026 14:08:07 +0800
Subject: [PATCH 31/34] CollectiveX: record the measured and clamped ladder in
 the artifact

b368eae1d's commit message and the methodology paragraph both claim a clamped ladder point
lands in the artifact rather than being dropped silently. That was not true: `dropped` was
printed once to stdout by rank 0 and never emitted, so an artifact from deepep-v2 low-latency
measuring 8 rungs was indistinguishable from one that measured all 9. Since the clamp is
exactly what keeps a corrupt upstream rung out of the results, its absence has to be legible
to whoever reads the record, not just to whoever tailed the log.

Emit ladder_measured, ladder_dropped and ladder_cap under workload, and guard all three.
---
 experimental/CollectiveX/bench/ep_harness.py   |  9 +++++++++
 experimental/CollectiveX/tests/test_runtime.py | 12 ++++++++++++
 2 files changed, 21 insertions(+)

diff --git a/experimental/CollectiveX/bench/ep_harness.py b/experimental/CollectiveX/bench/ep_harness.py
index 695ba1740..eb74620e8 100644
--- a/experimental/CollectiveX/bench/ep_harness.py
+++ b/experimental/CollectiveX/bench/ep_harness.py
@@ -1209,6 +1209,15 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
         },
         "workload": {
             "cross_rank_consistent": routing_consistent,
+            # The ladder actually measured, plus any requested point the backend's cap
+            # excluded. This used to be a rank-0 stdout NOTE only, which made a clamped
+            # ladder invisible to anyone reading the artifact -- so a backend measuring a
+            # shorter ladder than the sweep requested looked identical to one that measured
+            # all of it. deepep-v2's low-latency mode clamps below its receive cap to avoid a
+            # rung upstream corrupts, and that omission has to be legible downstream.
+            "ladder_measured": list(ladder),
+            "ladder_dropped": list(dropped),
+            "ladder_cap": cap,
         },
         "measurement": {
             "combine_dtype": backend.combine_dtype,
diff --git a/experimental/CollectiveX/tests/test_runtime.py b/experimental/CollectiveX/tests/test_runtime.py
index 23c859e18..19f9ef8c9 100644
--- a/experimental/CollectiveX/tests/test_runtime.py
+++ b/experimental/CollectiveX/tests/test_runtime.py
@@ -935,6 +935,18 @@ def test_the_low_latency_receive_is_sized_from_the_cap_not_the_ladder(self):
             "create_buffer must set self.max_tokens = _LL_BUFFER_CAP on the low-latency path",
         )
 
+    def test_a_clamped_ladder_is_recorded_in_the_artifact_not_only_on_stdout(self):
+        # A clamped ladder was previously visible only as a rank-0 stdout NOTE, so an artifact
+        # from a backend that measured 8 rungs was indistinguishable from one that measured 9.
+        # The emitted record must carry what ran and what was excluded.
+        harness = ast.parse((BENCH / "ep_harness.py").read_text())
+        keys = {
+            k.value for node in ast.walk(harness) if isinstance(node, ast.Dict)
+            for k in node.keys if isinstance(k, ast.Constant) and isinstance(k.value, str)
+        }
+        for required in ("ladder_measured", "ladder_dropped", "ladder_cap"):
+            self.assertIn(required, keys, f"the emitted record must include {required}")
+
 
 if __name__ == "__main__":
     unittest.main()

From 88e1481efb56cb684c317bdeb1fd6fb966858a1b Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 5 Aug 2026 23:28:12 +0800
Subject: [PATCH 32/34] CollectiveX: pin DeepEP to main for the #642
 low-latency fix, and unclamp

Fixes the defect instead of stepping around it. b368eae1d clamped the low-latency ladder to 128
so the corrupt 256 rung was not measured; the corruption itself was still in the kernels we
publish numbers from, at roughly 1.5-3.3% per invocation on every Blackwell SKU.

The fix was already upstream. DeepEP PR #642 adds a CTA-scope fence.proxy.async.shared::cta
before mbarrier_arrive(empty_barriers[stage_idx]) in LOW_LATENCY_COMBINE_RECV, so the combine
consumer's shared-memory reads retire before the stage is declared empty and the producer's next
TMA load refills it -- signalling empty too early is what let one output row be assembled from
two tiles, which matches the observed signature exactly (norm preserved to 4 s.f., 16-40% of
elements wrong). It closed #621, the same race found independently on NVL72.

COLLX_DEEPEP_V2_COMMIT was fa8a9b16, the head of the pre-merge PR #605 branch, cut before #642
landed. Its one unique commit was the #630 single-node V2 init fix, which main carries as
56169594e, so moving to main loses nothing and also picks up #715 (system-scope release before
the GIN barrier when scale-up spans NVLink and RDMA), #688 (NCCL Device API: runtime version for
ncclDevCommCreate), #178 (SM90), #641, and #640/#627 upstream's own libnccl and SO-name handling.
Verified before bumping that main still exposes every API this adapter calls: legacy Buffer
kwargs incl. allow_mnnvl, get_low_latency_rdma_size_hint, low_latency_dispatch/combine,
ElasticBuffer's full kwarg set, topk_idx_t, and the fp8 cast helpers.

rewrite_deepep_v2 now succeeds when the source already matches, because main contains #640 and
the old count(old) == 1 assertion would have aborted every leg at repository-stage. Verified
against all four source states: old form rewrites, fixed form is a no-op, absent and duplicated
forms still fail.

_LL_LADDER_CAP returns to 256. The two constants stay separate: the receive must not be sized
from max(ladder), or clamping the ladder would change the footprint that drives transport traffic
and fp8 dequant volume. The cap test now asserts ladder <= buffer -- strict inequality encoded
the workaround, not an invariant. The backend cache key includes the pin, so this forces a
rebuild rather than reusing the old build.
---
 experimental/CollectiveX/README.md            | 13 ++--
 .../CollectiveX/bench/ep_deepep_v2.py         | 64 ++++++++-----------
 experimental/CollectiveX/docs/methodology.md  | 34 ++++++----
 experimental/CollectiveX/runtime/common.sh    | 15 ++++-
 experimental/CollectiveX/runtime/stage.py     |  7 ++
 .../CollectiveX/tests/test_runtime.py         | 11 ++--
 6 files changed, 83 insertions(+), 61 deletions(-)

diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 7f5738458..17317d00a 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -96,11 +96,14 @@ scale-up domain.
 
 DeepEP V2 means the `ElasticBuffer` implementation introduced by
 [DeepEP PR #605](https://github.com/deepseek-ai/DeepEP/pull/605), not a newer legacy `Buffer` build.
-The pinned source is the [PR #630](https://github.com/deepseek-ai/DeepEP/pull/630) head, whose parent
-is the #605 merge tree, plus the exact one-line library matcher from upstream
-[PR #640](https://github.com/deepseek-ai/DeepEP/pull/640). The first fixes pure scale-up
-initialization when GIN is unavailable; the second prevents NCCL shared-memory mappings from being
-misclassified as duplicate NCCL libraries. Scale-up cases request NCCL Device API LSA and fail closed
+The pinned source is upstream `main`, which contains #605 along with
+[PR #630](https://github.com/deepseek-ai/DeepEP/pull/630) (fixes pure scale-up initialization when
+GIN is unavailable), [PR #640](https://github.com/deepseek-ai/DeepEP/pull/640) (stops NCCL
+shared-memory mappings being misclassified as duplicate NCCL libraries), and
+[PR #642](https://github.com/deepseek-ai/DeepEP/pull/642) (the low-latency combine fence that fixes
+the Blackwell top-rung corruption of
+[issue #700](https://github.com/deepseek-ai/DeepEP/issues/700)). It previously pinned the #630 head
+on the pre-merge #605 branch, which predated #642. Scale-up cases request NCCL Device API LSA and fail closed
 unless the realized LSA team covers the full EP world. x86 EP16 scale-out cases instead require the
 hybrid path with GIN, two logical scale-out domains represented by two physical RDMA ranks, and eight
 scale-up ranks per domain; GB EP16 remains MNNVL scale-up and therefore uses LSA. Whether a given
diff --git a/experimental/CollectiveX/bench/ep_deepep_v2.py b/experimental/CollectiveX/bench/ep_deepep_v2.py
index b65796825..8c055ceaf 100644
--- a/experimental/CollectiveX/bench/ep_deepep_v2.py
+++ b/experimental/CollectiveX/bench/ep_deepep_v2.py
@@ -21,18 +21,20 @@
     raise
 
 
-# The source pin in runtime/common.sh is PR #605 head at the #630 fix; #640 is NOT in
-# the fetched tree — runtime/stage.py applies it as a local rewrite before the build. This adapter no longer
-# verifies the wheel's commit tag against the pin — it checks only that the loaded
-# deep_ep exposes ElasticBuffer (the from-source PR #605 capability).
+# The source pin in runtime/common.sh is upstream main, which carries #630 (the fix our previous
+# PR #605 branch pin was cut at) as well as #640, so the stage-time rewrite is now a no-op the
+# source already satisfies. This adapter does not verify the wheel's commit tag against the pin --
+# it checks only that the loaded deep_ep exposes ElasticBuffer (the PR #605 capability).
 
 # Low-latency receive sizing. These are deliberately two numbers, not one: _LL_BUFFER_CAP
 # sizes the pre-allocated receive (and so fixes the transport footprint and the fp8 dequant
-# volume), while _LL_LADDER_CAP bounds which token counts are measured. See `buffer_cap` for
-# why the measured ladder stops below the buffer -- an upstream Blackwell combine defect at
-# the 256 rung -- and `create_buffer` for why the buffer must not follow the ladder down.
+# volume), while _LL_LADDER_CAP bounds which token counts are measured. They are equal today --
+# the ladder runs to the full receive -- but they stay separate because the buffer must not
+# follow the ladder (see `create_buffer`), and because clamping the measured ladder without
+# disturbing the footprint is the lever for a kernel defect at a specific token count, which is
+# how the Blackwell combine corruption at 256 was handled before #642 fixed it upstream.
 _LL_BUFFER_CAP = 256
-_LL_LADDER_CAP = 128
+_LL_LADDER_CAP = 256
 assert _LL_LADDER_CAP <= _LL_BUFFER_CAP <= 511, (
     "the LL receive cap must fit NVSHMEM_QP_DEPTH=1024 ((cap + 1) * 2 <= 1024 => cap <= 511) "
     "and the measured ladder must fit inside the buffer"
@@ -183,33 +185,22 @@ def __init__(self, args, rank, world_size, local_rank, device):
     def buffer_cap(self, args):
         if self.mode == "low-latency":
             # LL pre-allocates a fixed [num_local_experts, cap * num_ranks, hidden] receive
-            # buffer, so the buffer cap is a hard per-rank dispatch-slot bound. The MEASURED
-            # ladder is clamped tighter than that buffer (the harness reports every dropped
-            # point, so the omission lands in the artifact rather than being silent) because
-            # DeepEP's low-latency combine corrupts at T=256 on every Blackwell SKU -- b200,
-            # gb200 and gb300, EP8 and EP16, both precisions, MNNVL and RDMA alike -- while
-            # Hopper stays clean. It is stochastic at roughly 1.5-3.3% per oracle invocation,
-            # so a passing leg proves nothing; the gate catches it as a 0.07-6.6 relative
-            # error against a 0.03125 tolerance. Tracked upstream as DeepEP issue #700.
+            # buffer, so the cap is a hard per-rank dispatch-slot bound and the harness clamps
+            # the ladder to it, recording any dropped point in the artifact.
             #
-            # The likely fix already exists upstream and our pin simply predates it:
-            # PR #642 adds a CTA-scope `fence.proxy.async.shared::cta` before
-            # `mbarrier_arrive(empty_barriers[stage_idx])` in LOW_LATENCY_COMBINE_RECV, so the
+            # History, because the value here is load-bearing: DeepEP's low-latency combine used
+            # to corrupt the T=256 rung on every Blackwell SKU (b200, gb200, gb300; EP8 and EP16;
+            # both precisions; MNNVL and RDMA alike) while Hopper stayed clean -- stochastically,
+            # ~1.5-3.3% per oracle invocation, surfacing as one token row whose norm still matched
+            # to 4 s.f. with 16-40% of elements wrong. That is DeepEP issue #700, and it was fixed
+            # upstream by #642, which adds a CTA-scope `fence.proxy.async.shared::cta` before
+            # `mbarrier_arrive(empty_barriers[stage_idx])` in LOW_LATENCY_COMBINE_RECV so the
             # consumer's shared-memory reads retire before the stage is declared empty and the
-            # producer's next TMA load refills it. Signalling empty too early lets a row be
-            # assembled from two tiles, which is exactly the observed signature (one token row,
-            # norm preserved to 4 s.f., 16-40% of elements deviating). It closed #621, the same
-            # race reached from NVL72. COLLX_DEEPEP_V2_COMMIT is the head of PR #605, branched
-            # before #642 merged, so the fence is absent from our build; upstream main has both.
-            # An earlier on-metal test that appeared to rule fencing out used a device-scope
-            # __threadfence_system after the grid sync at internode_ll.cu:976 -- a different
-            # fence at a different site -- so it does not bear on #642.
-            #
-            # Raising this back to _LL_BUFFER_CAP is therefore gated on a pin bump, not on a
-            # new upstream release. That bump is deliberately NOT bundled here: it spans months
-            # of upstream change, re-baselines every deepep-v2 row including normal mode, and
-            # needs `rewrite_deepep_v2` made tolerant first (main already carries that
-            # 'libnccl' fix, so the rewrite's count(old) == 1 assertion would abort the stage).
+            # producer's next TMA load refills it. Our pin was the head of PR #605, branched
+            # before #642 merged, so we carried the defect and clamped this to 128 to keep the
+            # corrupt rung out of the results; the pin now tracks main and the ladder runs full.
+            # If the top rung ever reds again on Blackwell, clamping here is the containment
+            # lever -- but check the pin first rather than assuming the defect returned.
             return _LL_LADDER_CAP
         return None
 
@@ -303,11 +294,12 @@ def _create_ll_buffer(self, spec):
             raise RuntimeError(
                 "invalid DeepEP LL runtime: deep_ep.Buffer.low_latency_dispatch is absent"
             )
-        # Verified pinned signatures (commit fa8a9b16, deep_ep/buffers/legacy.py):
+        # Verified pinned signatures (commit 01dc3aaa, deep_ep/buffers/legacy.py):
         #   Buffer.get_low_latency_rdma_size_hint(num_max_dispatch_tokens_per_rank,
-        #       hidden, num_ranks, num_experts) -> int   (staticmethod, line 175)
+        #       hidden, num_ranks, num_experts) -> int   (staticmethod, line 176)
         #   Buffer(group, num_nvl_bytes=0, num_rdma_bytes=0, low_latency_mode=False,
-        #       num_qps_per_rank=24, allow_nvlink_for_low_latency_mode=True, ...)  (line 33)
+        #       num_qps_per_rank=24, allow_nvlink_for_low_latency_mode=True,
+        #       allow_mnnvl=False, explicitly_destroy=False, ...)  (line 33)
         num_rdma_bytes = deep_ep.Buffer.get_low_latency_rdma_size_hint(
             self.max_tokens, args.hidden, world_size, args.experts
         )
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 0a4e4456b..2be89463f 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -63,18 +63,23 @@ cells carry the control alone; the per-backend precision set lives in `sweep_mat
   live with the workload in `configs/sweep.json`.
 
 A backend may clamp the ladder below that, and every clamped point is reported in the artifact
-rather than dropped silently. Exactly one backend clamps today: DeepEP V2 in `low-latency` mode
-pre-allocates a fixed receive, so its ladder cannot exceed that buffer, and it is currently held at
-**T=128**, one rung below the 256-slot receive, because DeepEP's low-latency combine corrupts the
-256 rung on every Blackwell SKU we run — B200, GB200 and GB300, EP8 and EP16, both precisions,
-MNNVL and RDMA alike, while Hopper stays clean. It is stochastic at roughly 1.5-3.3% per
-invocation and shows up as one wrong token row whose norm still matches, so it is a correctness
-gate failure rather than a crash (upstream DeepEP issue #700). The receive stays sized at 256
-even though the ladder stops at 128: its footprint drives both the transport's memory traffic and
-the FP8 dequant volume, so shrinking it with the ladder would move every retained rung and break
-comparability with the published series. The likely upstream fix (DeepEP PR #642, a CTA-scope
-fence so the combine consumer's shared-memory reads retire before the stage is recycled) landed
-after the commit we pin, so raising this back to 256 is gated on a pin bump.
+rather than dropped silently — `workload.ladder_measured`, `ladder_dropped` and `ladder_cap`
+record what ran and what did not. DeepEP V2 in `low-latency` mode pre-allocates a fixed receive,
+so its ladder cannot exceed that buffer; both are 256, so the decode ladder runs to its full
+extent and only the 512 point is dropped.
+
+That clamp was briefly load-bearing and the episode is worth recording, because it is the kind of
+defect that reads as a measurement bug. DeepEP's low-latency combine corrupted the 256 rung on
+every Blackwell SKU — B200, GB200 and GB300, EP8 and EP16, both precisions, MNNVL and RDMA alike
+— while Hopper stayed clean, stochastically at roughly 1.5-3.3% per invocation, surfacing as one
+wrong token row whose norm still matched to 4 significant figures. Upstream fixed it in PR #642
+with a CTA-scope fence so the combine consumer's shared-memory reads retire before its staging
+buffer is recycled; the commit we pinned was a pre-merge branch head that predated the fix, so we
+carried the defect and clamped the measured ladder to 128 to keep the corrupt rung out of the
+results. The pin now tracks upstream main and the rung is measured again. The receive is sized
+from a constant rather than from `max(ladder)` so that clamping the ladder cannot change the
+footprint, since that footprint drives both the transport's memory traffic and the FP8 dequant
+volume and would otherwise shift every remaining rung.
 
 `sweep_matrix.py` materializes the requested SKUs, backends, EP sizes, and token ladders into a
 matrix document, then extracts strict per-shard controls. `--only-sku`, `--exclude-skus`,
@@ -92,8 +97,9 @@ Physical host count does not define scope. Both GB cells remain inside one 72-GP
 domain.
 
 Unsupported combinations are explicitly classified in the matrix, not silently skipped coverage. DeepEP V2 is the
-`ElasticBuffer` introduced by PR #605, pinned with upstream PR #630's minimal pure-scale-up fix and
-the exact upstream PR #640 library matcher that excludes NCCL shared-memory mappings. Scale-up cases
+`ElasticBuffer` introduced by PR #605, pinned at upstream main, which carries that PR plus #630's
+minimal pure-scale-up fix, the #640 library matcher that excludes NCCL shared-memory mappings, and
+the #642 low-latency combine fence. Scale-up cases
 request NCCL Device API LSA and fail closed unless the realized LSA team covers the full EP world.
 x86 EP16 scale-out uses the hybrid path with GIN and requires two logical scale-out domains
 represented by two physical RDMA ranks, with eight scale-up ranks per domain. GB EP16 remains MNNVL
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index eafec3260..e1147573a 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -12,7 +12,20 @@ collx_log() { printf '[collectivex] %s\n' "$*" >&2; }
 collx_die() { printf '[collectivex] FATAL: %s\n' "$*" >&2; exit 1; }
 
 COLLX_DEEPEP_V2_REPO="https://github.com/deepseek-ai/DeepEP"
-COLLX_DEEPEP_V2_COMMIT="fa8a9b16898204afd347c663b89e65ef87dc6ce6"
+# Upstream main. This replaced the head of PR #605 (fa8a9b16), which was a pre-merge branch
+# commit: #605 merged on 2026-04-29 and useful fixes landed on main afterwards that the branch
+# never received. Its one unique commit was the #630 single-node V2 init fix, which main carries
+# as 56169594e, so nothing was lost by moving. What was gained, and why this bump happened:
+#   #642 fence.proxy.async.shared::cta in LOW_LATENCY_COMBINE_RECV -- the fix for the Blackwell
+#        low-latency combine corruption at the top ladder rung (DeepEP issue #700)
+#   #715 system-scope release before the GIN barrier when scale-up spans NVLink and RDMA
+#   #688 NCCL Device API compat: use the runtime version for ncclDevCommCreate
+#   #178 SM90 compatibility; #641 internode dispatch args
+#   #640/#627 match "libnccl" and resolve real NVSHMEM/NCCL SO names for pip-wheel installs,
+#        which is the upstream form of the rewrite collx_prepare_deepep_source applies below
+# The backend cache directory is keyed on this value, so changing it forces a rebuild rather
+# than silently reusing a build of the old source.
+COLLX_DEEPEP_V2_COMMIT="01dc3aaac82068020353dce2c302e38153c0bfaa"
 
 COLLX_UCCL_REPO="https://github.com/uccl-project/uccl"
 COLLX_UCCL_COMMIT="fc1b582031221645ea9fce58aeb57187713145e3"
diff --git a/experimental/CollectiveX/runtime/stage.py b/experimental/CollectiveX/runtime/stage.py
index 119b77496..8929bbe16 100644
--- a/experimental/CollectiveX/runtime/stage.py
+++ b/experimental/CollectiveX/runtime/stage.py
@@ -61,10 +61,17 @@ def validate_cleanup(args) -> None:
 
 
 def rewrite_deepep_v2(args) -> None:
+    # Narrows DeepEP's NCCL library scan from 'nccl' to 'libnccl'; upstream landed the same
+    # change as #640, so a current pin already satisfies it and there is nothing to rewrite.
+    # Succeeding in that case is the point: failing on an already-correct source would abort
+    # every leg at repository-stage the moment the pin moved forward.
     path = Path(args.path)
     old = "for so in [line.strip().split(' ')[-1] for line in f if 'nccl' in line]:"
     new = "for so in [line.strip().split(' ')[-1] for line in f if 'libnccl' in line]:"
     text = path.read_text()
+    if text.count(new) >= 1 and text.count(old) == 0:
+        return
+    # Exactly one un-rewritten occurrence, or the source is not what we think it is.
     if text.count(old) != 1: raise SystemExit(1)
     path.write_text(text.replace(old, new))
 
diff --git a/experimental/CollectiveX/tests/test_runtime.py b/experimental/CollectiveX/tests/test_runtime.py
index 19f9ef8c9..dc7ac3f01 100644
--- a/experimental/CollectiveX/tests/test_runtime.py
+++ b/experimental/CollectiveX/tests/test_runtime.py
@@ -901,14 +901,15 @@ def _func(self, name):
                 return node
         self.fail(f"{name} not found in ep_deepep_v2.py")
 
-    def test_both_caps_exist_and_the_ladder_stops_strictly_below_the_buffer(self):
+    def test_both_caps_exist_and_the_ladder_fits_inside_the_buffer(self):
         buf, ladder = self.consts.get("_LL_BUFFER_CAP"), self.consts.get("_LL_LADDER_CAP")
         self.assertIsInstance(buf, int)
         self.assertIsInstance(ladder, int)
-        # Strictly below: equality is the configuration that corrupts, and it also puts the top
-        # measured rung at 100% occupancy, which is what made capacity and last-rung
-        # indistinguishable in the original investigation.
-        self.assertLess(ladder, buf)
+        # Equality is expected while the ladder runs to the full receive; what must never happen
+        # is a ladder larger than the buffer, which would dispatch past the allocated slots.
+        # (This was `assertLess` while the ladder was clamped to 128 to dodge the pre-#642
+        # Blackwell combine defect -- that was a workaround, not an invariant.)
+        self.assertLessEqual(ladder, buf)
         # NVSHMEM_QP_DEPTH=1024 asserts nvshmem_qp_depth >= (cap + 1) * 2 at construction.
         self.assertLessEqual(buf, 511)
 

From ca72509db89e4f8ac117e86f592ee9c842a69cbe Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 5 Aug 2026 23:34:21 +0800
Subject: [PATCH 33/34] CollectiveX: exclude b300-018, which has two dead RDMA
 rails

The network profile gate failed the leg closed with network-profile-rdma-port-9=inactive nine
seconds in, before any build or DeepEP code ran. Confirmed on the node: mlx5_10 and mlx5_11 are
both DOWN while the other 22 devices are ACTIVE, and rdma_devices[9] is mlx5_11 exactly. Slurm
still reports the node idle with no reason, so it keeps getting allocated and every EP16 leg that
lands on it dies at pre-flight. Same treatment as gpu-2-6 on b200-dgxc.

Remove this once the rails are back up; the gate itself needs no change -- it did its job.
---
 experimental/CollectiveX/configs/platform_config.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/experimental/CollectiveX/configs/platform_config.json b/experimental/CollectiveX/configs/platform_config.json
index c6f5d1fb5..1d3739bc3 100644
--- a/experimental/CollectiveX/configs/platform_config.json
+++ b/experimental/CollectiveX/configs/platform_config.json
@@ -85,7 +85,8 @@
         "partition": "batch_1",
         "account": "benchmark",
         "qos": "batch_1_qos",
-        "squash_dir": "/data/home/sa-shared/sqsh"
+        "squash_dir": "/data/home/sa-shared/sqsh",
+        "exclude_nodes": "b300-018"
       },
       "network": {
         "socket_ifname": "bond0",

From 1b3d223e4a10941b58ba0cade2518a96ce5dfed9 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Thu, 6 Aug 2026 00:14:51 +0800
Subject: [PATCH 34/34] CollectiveX: raise the per-case hang guard to 5400s for
 the post-#715 GIN cost

Moving the DeepEP pin to main brought #715, which adds a system-scope release before every GIN
barrier when scale-up spans NVLink and RDMA. It is a correctness fix and it is not free: on the
multi-node EP16 path it roughly doubles decode and costs several times more on prefill, which
issues far more barriers. Measured b200 EP16 bf16 decode 185s -> 287s and h200 127s -> 270s,
with both prefill cases running past the 1800s guard and being killed.

Single-node EP8 is byte-for-byte unaffected (b200 74/173s -> 77/173s, h200 74/296s -> 69/273s),
which is what localizes the cost to the RDMA+GIN path rather than to the kernels generally.

The guard exists to catch hangs, not to bound legitimate work, so truncating the measurement is
the wrong failure mode -- raise it and let the slower-but-correct path report a real number.
5400s stays well inside the 300-minute allocation.
---
 experimental/CollectiveX/runtime/common.sh | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index e1147573a..aa124f7de 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -941,8 +941,14 @@ collx_run_shard() {
     collx_log "EP${NGPUS}[$((ci + 1))/$expected_cases] $COLLX_BENCH"
     runtime_log="$(collx_private_log_path "runtime-c$(printf '%03d' "$ci")")"
     # A hang guard, NOT a work budget: at 900 it killed FP8 prefill cases that had already
-    # written complete, all-rungs-passed artifacts. 1800 is what the AMD launcher already used.
-    if ! timeout -k 30 "${COLLX_RUN_TIMEOUT:-1800}" \
+    # written complete, all-rungs-passed artifacts, and at 1800 it killed multi-node EP16 prefill
+    # once the DeepEP pin moved to main. That bump brought #715, which adds a system-scope release
+    # before every GIN barrier when scale-up spans NVLink and RDMA -- correct, but it roughly
+    # doubles EP16 decode and costs several times more on prefill, which issues far more
+    # barriers. Single-node EP8 uses neither RDMA nor GIN and is unchanged, which is what
+    # localizes the cost. 5400 keeps the guard well inside the 300-minute allocation while
+    # leaving room to actually measure that cost instead of truncating it.
+    if ! timeout -k 30 "${COLLX_RUN_TIMEOUT:-5400}" \
       srun --jobid="$JOB_ID" --nodes="$NODES" \
       --ntasks="$NGPUS" --ntasks-per-node="$GPN" --chdir=/tmp \
       --container-name="$container_name" --container-image="$SQUASH_FILE" \