From 486d9cc4e4ed2482165546ec7897e054e7c523f0 Mon Sep 17 00:00:00 2001 From: Brad Hilton Date: Thu, 3 Sep 2026 19:27:06 +0000 Subject: [PATCH 1/2] Decide calibration warm-ups from world-wide compile telemetry (#840) The cost-calibration harness stopped warming a candidate up when the local rank's forward was compile-free. Compile telemetry is per process, so a CP rank whose local shapes still recompiled ran one more warm-up of the previous layout while its peers moved to the next candidate; the context-parallel all-to-alls then paired different layouts and deadlocked. This is the "deterministic CP4 hang" of issue #840 on Ellavox groups 1 and 4: rank 3 recompiled on its second depth_one warm-up, ranks 0-2 did not. Every recorded compile status is now the gathered union over all ranks (`_world_compile_statuses`), and warm-up completion is a pure function of that (`_warmup_complete`); the tp2-public compile-free gate uses the same world-wide view. Regression test on CPU. Diagnostics kept as dev tools: `trainer_rank_collective_trace.py` runs a calibration cell with every torch.distributed collective logged per rank and a stall watchdog; `trainer_rank_collective_diff.py` finds the first collective where ranks disagree. The CP4 recipe gains `CELL_SET=ellavox` to re-measure single Ellavox groups. Co-Authored-By: Claude Fable 5.1 --- dev/trainer_rank_collective_diff.py | 105 ++++++++ dev/trainer_rank_collective_trace.py | 240 ++++++++++++++++++ ...trainer_rank_cost_calibration_cp4.sky.yaml | 11 +- dev/trainer_rank_landing_acceptance.py | 48 +++- .../test_trainer_rank_calibration_harness.py | 68 +++++ 5 files changed, 457 insertions(+), 15 deletions(-) create mode 100644 dev/trainer_rank_collective_diff.py create mode 100644 dev/trainer_rank_collective_trace.py create mode 100644 tests/unit/test_trainer_rank_calibration_harness.py diff --git a/dev/trainer_rank_collective_diff.py b/dev/trainer_rank_collective_diff.py new file mode 100644 index 000000000..c10166d00 --- /dev/null +++ b/dev/trainer_rank_collective_diff.py @@ -0,0 +1,105 @@ +"""Compare per-rank collective traces from ``trainer_rank_collective_trace.py``. + +Prints the layout label each rank ran per forward, then the first collective at +which the ranks disagree on operation, participation or split sizes (rank a's +send to b must equal rank b's expected receive from a), and any stall dumps. + +Usage: python dev/trainer_rank_collective_diff.py +""" + +from __future__ import annotations + +import collections +import json +from pathlib import Path +import sys + + +def _load(root: Path) -> dict[int, list[dict]]: + logs: dict[int, list[dict]] = {} + for path in sorted(root.glob("collectives.rank*.jsonl")): + rank = int(path.stem.split("rank")[1]) + logs[rank] = [ + json.loads(line) for line in path.read_text().splitlines() if line.strip() + ] + return logs + + +def main() -> None: + root = Path(sys.argv[1]) + logs = _load(root) + ranks = sorted(logs) + print("records per rank:", {rank: len(logs[rank]) for rank in ranks}) + for rank in ranks: + forwards = collections.OrderedDict() + for record in logs[rank]: + if record["op"] == "forward_start": + forwards[record["forward_index"]] = record["anchor"] + print( + f"rank {rank} forwards: " + + ", ".join(f"f{index}={anchor}" for index, anchor in forwards.items()) + ) + sequences = { + rank: [r for r in logs[rank] if not r["op"].startswith("forward_")] + for rank in ranks + } + length = max(len(sequence) for sequence in sequences.values()) + for index in range(length): + records = { + rank: sequences[rank][index] if index < len(sequences[rank]) else None + for rank in ranks + } + signatures = { + rank: ( + (record["op"], record["caller"].split("/")[0], record["group_size"]) + if record + else None + ) + for rank, record in records.items() + } + reason = None + if len(set(signatures.values())) != 1: + reason = f"different operations: {signatures}" + else: + mismatches = [] + for a in ranks: + for b in ranks: + if a == b or records[a] is None or records[b] is None: + continue + sent, expected = ( + records[a].get("in_splits"), + records[b].get("out_splits"), + ) + if sent is None or expected is None: + continue + if sent[b] != expected[a]: + mismatches.append( + f"rank{a}->rank{b}: sends {sent[b]}, rank{b} expects {expected[a]}" + ) + if mismatches: + reason = "split mismatch: " + "; ".join(mismatches[:4]) + if reason is None: + continue + print(f"\nFIRST DIVERGENCE at collective #{index + 1}: {reason}") + for rank in ranks: + for j in range(max(0, index - 3), min(len(sequences[rank]), index + 4)): + record = sequences[rank][j] + detail = ( + f"in={record.get('in_splits')} out={record.get('out_splits')}" + if record["op"] == "all_to_all_single" + else f"numel={record.get('numel')}" + ) + print( + f" rank{rank} #{j + 1} f{record['forward_index']} {record['anchor']} " + f"{record['op']} {record['caller'][:70]} {detail}" + ) + break + else: + print("\nno divergence in the common prefix of the traces") + for path in sorted(root.glob("stall.rank*.txt")): + print(f"\n=== {path.name}") + print(path.read_text()[:1200]) + + +if __name__ == "__main__": + main() diff --git a/dev/trainer_rank_collective_trace.py b/dev/trainer_rank_collective_trace.py new file mode 100644 index 000000000..6c1a05a16 --- /dev/null +++ b/dev/trainer_rank_collective_trace.py @@ -0,0 +1,240 @@ +"""Run one cost-calibration cell with every collective traced per rank. + +Diagnostic for hangs and desynchronization in distributed TrainerRank runs +(used to root-cause issue #840). Runs +``dev/trainer_rank_landing_acceptance.py --phase cost-calibrate`` unchanged +while logging every ``torch.distributed`` collective to a per-rank JSONL: +operation, calling frames, split sizes, element counts, the forced-layout +label in effect and a running forward index. A stall watchdog dumps all thread +stacks and exits when no collective completes for ``--stall-seconds``, so a +deadlock surfaces in minutes instead of the NCCL timeout. + +Usage (from the repo root, e.g. 4 GPUs at CP4): + torchrun --standalone --nproc-per-node=4 dev/trainer_rank_collective_trace.py \ + --cell cal-ellavox --group 4 --repeat 1 --log-dir + python dev/trainer_rank_collective_diff.py # first divergent collective +""" + +from __future__ import annotations + +import argparse +import faulthandler +import functools +import inspect +import json +import os +from pathlib import Path +import sys +import threading +import time + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "dev")) + +COLLECTIVES = ( + "all_to_all_single", + "all_to_all", + "all_reduce", + "all_gather", + "all_gather_into_tensor", + "all_gather_object", + "reduce_scatter_tensor", + "reduce_scatter", + "broadcast", + "broadcast_object_list", + "barrier", + "send", + "recv", +) + + +class _Logger: + def __init__(self, log_path: Path, rank: int) -> None: + self.rank = rank + self.handle = open(log_path, "a", encoding="utf-8") + self.seq = 0 + self.forward_index = 0 + self.last_event = time.time() + self.lock = threading.Lock() + + def write(self, record: dict[str, object]) -> None: + with self.lock: + self.seq += 1 + record = {"seq": self.seq, "rank": self.rank, "t": time.time(), **record} + self.handle.write(json.dumps(record, default=str) + "\n") + self.handle.flush() + self.last_event = record["t"] + + def anchor(self) -> str: + return os.environ.get("ART_TRAINER_RANK_TEST_ANCHOR", "automatic") + + +def _numel(value: object) -> int | None: + if hasattr(value, "numel"): + return int(value.numel()) + if isinstance(value, (list, tuple)): + return sum(_numel(v) or 0 for v in value) + return None + + +def _shape(value: object) -> object: + if hasattr(value, "shape"): + return list(value.shape) + if isinstance(value, (list, tuple)): + return [_shape(v) for v in value[:4]] + return None + + +def _install(logger: _Logger) -> None: + import torch.distributed as dist + + import art.megatron.gdn.layout as gdn_layout + from art.trainer_rank import TrainerRank + + def wrap(name: str): + original = getattr(dist, name) + signature = inspect.signature(original) + + def logged(*args, **kwargs): + bound = signature.bind_partial(*args, **kwargs) + params = bound.arguments + group = params.get("group") + frames = inspect.stack()[1:7] + record: dict[str, object] = { + "op": name, + "forward_index": logger.forward_index, + "anchor": logger.anchor(), + "caller": "/".join(f.function for f in frames), + "group_size": dist.get_world_size(group) + if dist.is_initialized() + else None, + "async_op": bool(params.get("async_op", False)), + } + if name == "all_to_all_single": + record["in_splits"] = ( + list(map(int, params["input_split_sizes"])) + if params.get("input_split_sizes") is not None + else None + ) + record["out_splits"] = ( + list(map(int, params["output_split_sizes"])) + if params.get("output_split_sizes") is not None + else None + ) + record["numel_in"] = _numel(params.get("input")) + record["numel_out"] = _numel(params.get("output")) + record["shape_in"] = _shape(params.get("input")) + record["shape_out"] = _shape(params.get("output")) + else: + first = next(iter(params.values()), None) + record["numel"] = _numel(first) + record["shape"] = _shape(first) + logger.write(record) + return original(*args, **kwargs) + + logged.__name__ = name + return logged + + for name in COLLECTIVES: + if hasattr(dist, name): + setattr(dist, name, wrap(name)) + # gdn.layout imported the function by name; rebind it to the logged one. + gdn_layout.all_to_all_single = dist.all_to_all_single + + original_forward = TrainerRank.dp_rank_forward + + @functools.wraps(original_forward) + def logged_forward(self, *args, **kwargs): + logger.forward_index += 1 + logger.write( + { + "op": "forward_start", + "forward_index": logger.forward_index, + "anchor": logger.anchor(), + } + ) + outputs = original_forward(self, *args, **kwargs) + telemetry = self.last_forward_telemetry() + logger.write( + { + "op": "forward_end", + "forward_index": logger.forward_index, + "anchor": logger.anchor(), + "selected_max_depth": telemetry.get("selected_max_depth"), + "subforward_count": telemetry.get("subforward_count"), + } + ) + return outputs + + TrainerRank.dp_rank_forward = logged_forward # type: ignore[method-assign] + + +def _start_watchdog(logger: _Logger, log_dir: Path, stall_seconds: float) -> None: + def watch() -> None: + while True: + time.sleep(5) + idle = time.time() - logger.last_event + if logger.seq > 0 and idle > stall_seconds: + path = log_dir / f"stall.rank{logger.rank}.txt" + with open(path, "w", encoding="utf-8") as handle: + handle.write( + f"rank {logger.rank}: no collective for {idle:.0f}s after seq " + f"{logger.seq} (forward_index {logger.forward_index}, anchor " + f"{logger.anchor()}); dumping all thread stacks\n" + ) + handle.flush() + faulthandler.dump_traceback(handle, all_threads=True) + print( + f"repro: rank {logger.rank} stalled after collective #{logger.seq} " + f"(forward {logger.forward_index}, anchor {logger.anchor()}); see {path}", + flush=True, + ) + os._exit(3) + + threading.Thread(target=watch, name="stall-watchdog", daemon=True).start() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cell", default="cal-ellavox") + parser.add_argument("--group", type=int, default=0) + parser.add_argument("--repeat", type=int, default=1) + parser.add_argument("--model", default="Qwen/Qwen3.5-4B") + parser.add_argument("--layers", type=int, default=0) + parser.add_argument("--log-dir", required=True) + parser.add_argument("--stall-seconds", type=float, default=300.0) + arguments = parser.parse_args() + + rank = int(os.environ.get("RANK", "0")) + log_dir = Path(arguments.log_dir) + log_dir.mkdir(parents=True, exist_ok=True) + logger = _Logger(log_dir / f"collectives.rank{rank}.jsonl", rank) + _install(logger) + _start_watchdog(logger, log_dir, arguments.stall_seconds) + + import trainer_rank_landing_acceptance as driver + + sys.argv = [ + "trainer_rank_landing_acceptance.py", + "--phase", + "cost-calibrate", + "--cell", + arguments.cell, + "--model", + arguments.model, + "--layers", + str(arguments.layers), + "--group", + str(arguments.group), + "--repeat", + str(arguments.repeat), + "--evidence", + str(log_dir / "evidence.jsonl"), + ] + driver.main() + if rank == 0: + print("repro: harness completed without hanging", flush=True) + + +if __name__ == "__main__": + main() diff --git a/dev/trainer_rank_cost_calibration_cp4.sky.yaml b/dev/trainer_rank_cost_calibration_cp4.sky.yaml index 238ef8c2d..5999cf1f7 100644 --- a/dev/trainer_rank_cost_calibration_cp4.sky.yaml +++ b/dev/trainer_rank_cost_calibration_cp4.sky.yaml @@ -25,7 +25,8 @@ envs: RUN_ID: unset REPEAT: "8" # "main" = GRPO/hetero/attention cells; "hetero-ellavox" = heterogeneous - # variants and Ellavox groups (second campaign). + # variants and Ellavox groups (second campaign); "ellavox" = the Ellavox + # groups in ELLAVOX_GROUPS only (e.g. re-measuring single groups). CELL_SET: main ELLAVOX_GROUPS: "0 1 2 3 4 5 6 7" @@ -76,9 +77,11 @@ run: | run_cell cal-grpo-g8-long Qwen/Qwen3-4B 2 run_cell cal-grpo-g8 Qwen/Qwen3-4B 0 else - run_cell cal-hetero2 Qwen/Qwen3.5-4B 0 - run_cell cal-hetero3 Qwen/Qwen3.5-4B 0 - run_cell cal-hetero2 Qwen/Qwen3-4B 0 + if [ "${CELL_SET}" = "hetero-ellavox" ]; then + run_cell cal-hetero2 Qwen/Qwen3.5-4B 0 + run_cell cal-hetero3 Qwen/Qwen3.5-4B 0 + run_cell cal-hetero2 Qwen/Qwen3-4B 0 + fi for g in ${ELLAVOX_GROUPS}; do run_cell cal-ellavox Qwen/Qwen3.5-4B 0 "$g" done diff --git a/dev/trainer_rank_landing_acceptance.py b/dev/trainer_rank_landing_acceptance.py index 2b22c5a41..a7946a1ca 100644 --- a/dev/trainer_rank_landing_acceptance.py +++ b/dev/trainer_rank_landing_acceptance.py @@ -1090,6 +1090,22 @@ def _gather_objects(value: Any, group: Any = None) -> list[Any]: return values +def _merge_rank_statuses(gathered: list[list[str]]) -> list[str]: + return sorted({str(status) for statuses in gathered for status in statuses}) + + +def _world_compile_statuses(watch: Any) -> list[str]: + """Compile statuses of the last forward on every rank (sorted, unique). + + Compile telemetry is per process: a rank whose local shapes differ can + recompile while its peers do not. Any decision that steers the next + forward (warm-up completion, compile-free gates) must be taken from the + world-wide view, or the ranks run different layouts and their collectives + deadlock (issue #840). + """ + return _merge_rank_statuses(_gather_objects(watch.take())) + + def _plan_fingerprint( rank: Any, requests: list[Any], checkpoint: Any ) -> dict[str, Any]: @@ -1335,7 +1351,7 @@ def run(arm: str) -> dict[str, Any]: "planning_ms": float(telemetry["planning_ms"]), "packed_tokens": int(fingerprint["packed_tokens"]), "group_lengths": list(fingerprint["group_lengths"]), - "compile_statuses": watch.take(), + "compile_statuses": _world_compile_statuses(watch), } del outputs, loss rank.zero_grad() @@ -1904,6 +1920,20 @@ def stream(arm: str, cap_bytes: int | None) -> dict[str, Any]: CALIBRATION_MIN_WARMUPS = 2 +def _warmup_complete(attempt: int, statuses: list[str]) -> bool: + """Whether a candidate's warm-up may stop after 0-based ``attempt``. + + ``statuses`` must be the world-wide compile statuses of that attempt + (``_world_compile_statuses``); the decision selects every rank's next + forward, so it has to be identical on all ranks. + """ + return ( + attempt + 1 >= CALIBRATION_MIN_WARMUPS + and bool(statuses) + and all(status == "none" for status in statuses) + ) + + def _gdn_layer_count(model: Any) -> int: try: from megatron.core.ssm.gated_delta_net import GatedDeltaNet @@ -2000,9 +2030,10 @@ def phase_cost_calibrate( ) -> None: """Time every mandatory candidate layout of one cell (GPU). - Per candidate: warm up until a forward is compile-free (bounded), then - ``repeat`` measured rounds in a rotating candidate order so drift is - balanced across candidates. Each measured row records max-rank forward + + Per candidate: warm up until a forward is compile-free on every rank + (bounded; the decision is taken from world-wide compile telemetry so all + ranks run the same layout sequence), then ``repeat`` measured rounds in a + rotating candidate order so drift is balanced across candidates. Each measured row records max-rank forward + backward wall time, compile status, plan-cache planning time, peak memory, subforward count (split rows are excluded from fitting), the candidate's layout features and labels, and the topology and model facts. @@ -2194,7 +2225,7 @@ def run(label: str) -> dict[str, object]: "admission_failed": False, "ms_max_rank": float(ms.item()), "ms_local": local_ms, - "compile_statuses": watch.take(), + "compile_statuses": _world_compile_statuses(watch), "planning_ms": float(telemetry["planning_ms"]), "selected_max_depth": int(telemetry["selected_max_depth"]), "subforward_count": int(telemetry["subforward_count"]), @@ -2224,12 +2255,7 @@ def run(label: str) -> dict[str, object]: ) if result.get("admission_failed"): break - statuses = cast(list, result["compile_statuses"]) - if ( - attempt + 1 >= CALIBRATION_MIN_WARMUPS - and statuses - and all(status == "none" for status in statuses) - ): + if _warmup_complete(attempt, cast(list, result["compile_statuses"])): live.append(label) break else: diff --git a/tests/unit/test_trainer_rank_calibration_harness.py b/tests/unit/test_trainer_rank_calibration_harness.py new file mode 100644 index 000000000..39cda38e2 --- /dev/null +++ b/tests/unit/test_trainer_rank_calibration_harness.py @@ -0,0 +1,68 @@ +"""The calibration harness steers every rank's next forward from world-wide +compile telemetry. + +Issue #840: the warm-up loop stopped when the *local* rank's forward was +compile-free. One CP rank whose local shapes still recompiled ran an extra +warm-up of the previous layout while its peers moved on to the next one, so the +context-parallel all-to-alls paired different layouts and deadlocked. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import sys + +_DRIVER = ( + Path(__file__).resolve().parents[2] / "dev" / "trainer_rank_landing_acceptance.py" +) +_spec = importlib.util.spec_from_file_location( + "trainer_rank_landing_acceptance", _DRIVER +) +assert _spec is not None and _spec.loader is not None +driver = importlib.util.module_from_spec(_spec) +sys.modules[_spec.name] = driver +_spec.loader.exec_module(driver) + + +class _Watch: + def __init__(self, statuses: list[str]) -> None: + self._statuses = list(statuses) + + def take(self) -> list[str]: + statuses, self._statuses = self._statuses, [] + return statuses + + +def test_world_compile_statuses_merges_every_rank(monkeypatch) -> None: + # Rank 3 recompiled while ranks 0-2 were compile-free: the warm-up must + # continue on all four ranks. + per_rank = [["none"], ["none"], ["none"], ["recompile"]] + gathered: list[list[str]] = [] + + def fake_gather(value, group=None): + gathered.append(value) + return per_rank + + monkeypatch.setattr(driver, "_gather_objects", fake_gather) + statuses = driver._world_compile_statuses(_Watch(["none"])) + assert gathered == [["none"]], "the local statuses must be gathered" + assert statuses == ["none", "recompile"] + assert not driver._warmup_complete(attempt=1, statuses=statuses) + + +def test_warmup_complete_needs_min_warmups_and_no_compile_anywhere() -> None: + assert not driver._warmup_complete(0, ["none"]) + assert driver._warmup_complete(1, ["none"]) + assert not driver._warmup_complete(1, []) + assert not driver._warmup_complete(7, ["none", "recompile"]) + assert driver._merge_rank_statuses([["recompile", "none"], ["none"]]) == [ + "none", + "recompile", + ] + + +def test_every_recorded_compile_status_is_world_wide() -> None: + source = _DRIVER.read_text() + assert '"compile_statuses": watch.take()' not in source + assert source.count('"compile_statuses": _world_compile_statuses(watch)') == 2 From 4a08d15fa760d8e86412717d8193a8fe730f694a Mon Sep 17 00:00:00 2001 From: Brad Hilton Date: Thu, 3 Sep 2026 20:06:44 +0000 Subject: [PATCH 2/2] Fold the re-measured Ellavox CP4 cells into the cost certificate Groups 1 and 4 at CP4 were excluded from the calibration certificate while their hang (issue #840) was open. Re-measured with the fixed harness (campaign tr-cost-cp4-840, every candidate compile-free on all ranks, 8 measured rounds), both cells are now certified as held-out validation cells: group 1 is an odd Ellavox group (pre-registered holdout) and group 4 is held out by argument, so the 45 training cells and therefore the shipped table are unchanged (same hash). Metrics on all 58 cells / 3,849 pairs: 98.1% pairwise, p95 regret 2.9%, max 4.2%, no clear misses; shipped-table regret on the two new cells 0% and 2.8%. Manifest: no exclusions, the two cells attributed to the new campaign; the certificate/manifest tests assert 58 identities; README and design brief updated with the root cause. Co-Authored-By: Claude Fable 5.1 --- ...ner_rank_cost_calibration_certificate.json | 12 ++++---- ...rainer_rank_cost_calibration_manifest.json | 9 +++--- dev/trainer_rank_landing_acceptance_README.md | 6 ++-- dev/trainer_rank_planner_design.md | 28 ++++++++++++------- tests/unit/test_planner_cost_certificate.py | 5 ++-- .../test_planner_cost_manifest_recipes.py | 3 +- 6 files changed, 37 insertions(+), 26 deletions(-) diff --git a/dev/trainer_rank_cost_calibration_certificate.json b/dev/trainer_rank_cost_calibration_certificate.json index 3c2cca8eb..8414a2301 100644 --- a/dev/trainer_rank_cost_calibration_certificate.json +++ b/dev/trainer_rank_cost_calibration_certificate.json @@ -17,8 +17,10 @@ {"candidates":[{"features":{"max_depth":2,"packed_tokens":9433,"segment_count":14,"segments_below":[1,3,6,9,13,13,13,14]},"label":"depth_one","median_ms":668.4934997558594,"n":8,"spread_pct":0.8820099524336579},{"features":{"max_depth":6,"packed_tokens":6738,"segment_count":17,"segments_below":[6,11,15,16,16,16,16,17]},"label":"minimum_effective_span_105","median_ms":1085.5021362304688,"n":8,"spread_pct":8.409486559556504},{"features":{"max_depth":6,"packed_tokens":6778,"segment_count":17,"segments_below":[5,10,15,16,16,16,16,17]},"label":"minimum_effective_span_113","median_ms":1086.4219970703125,"n":8,"spread_pct":7.212553212263997},{"features":{"max_depth":6,"packed_tokens":6877,"segment_count":17,"segments_below":[5,8,16,16,16,16,16,17]},"label":"minimum_effective_span_133","median_ms":1083.8602905273438,"n":8,"spread_pct":12.719941421222373},{"features":{"max_depth":5,"packed_tokens":7090,"segment_count":16,"segments_below":[3,8,14,15,15,15,15,16]},"label":"minimum_effective_span_164","median_ms":994.7117309570312,"n":8,"spread_pct":13.19183227071727},{"features":{"max_depth":5,"packed_tokens":7134,"segment_count":16,"segments_below":[4,7,14,15,15,15,15,16]},"label":"minimum_effective_span_195","median_ms":984.8882446289062,"n":8,"spread_pct":6.894005971217017},{"features":{"max_depth":4,"packed_tokens":7415,"segment_count":16,"segments_below":[3,5,13,15,15,15,15,16]},"label":"minimum_effective_span_197","median_ms":887.2643127441406,"n":8,"spread_pct":10.572528734299866},{"features":{"max_depth":4,"packed_tokens":7626,"segment_count":15,"segments_below":[3,6,10,14,14,14,14,15]},"label":"minimum_effective_span_245","median_ms":879.5194091796875,"n":8,"spread_pct":11.191289743280485},{"features":{"max_depth":4,"packed_tokens":7702,"segment_count":15,"segments_below":[3,5,9,14,14,14,14,15]},"label":"minimum_effective_span_281","median_ms":888.7151489257812,"n":8,"spread_pct":91.34438439776524},{"features":{"max_depth":12,"packed_tokens":6323,"segment_count":23,"segments_below":[16,20,22,22,22,22,22,23]},"label":"minimum_effective_span_32","median_ms":1652.6224365234375,"n":8,"spread_pct":9.261395998296091},{"features":{"max_depth":3,"packed_tokens":8041,"segment_count":15,"segments_below":[2,4,8,14,14,14,14,15]},"label":"minimum_effective_span_340","median_ms":792.2379760742188,"n":8,"spread_pct":6.9943845263187825},{"features":{"max_depth":3,"packed_tokens":8113,"segment_count":15,"segments_below":[1,4,8,14,14,14,14,15]},"label":"minimum_effective_span_349","median_ms":792.230224609375,"n":8,"spread_pct":14.296258060539756},{"features":{"max_depth":11,"packed_tokens":6359,"segment_count":22,"segments_below":[15,19,21,21,21,21,21,22]},"label":"minimum_effective_span_37","median_ms":1584.7108764648438,"n":8,"spread_pct":10.150299822025987},{"features":{"max_depth":9,"packed_tokens":6437,"segment_count":20,"segments_below":[9,17,19,19,19,19,19,20]},"label":"minimum_effective_span_40","median_ms":1394.2275390625,"n":8,"spread_pct":12.65724713155316},{"features":{"max_depth":3,"packed_tokens":8391,"segment_count":15,"segments_below":[1,3,9,12,14,14,14,15]},"label":"minimum_effective_span_441","median_ms":787.9234924316406,"n":8,"spread_pct":5.705350432944337},{"features":{"max_depth":2,"packed_tokens":13958,"segment_count":14,"segments_below":[1,3,6,9,12,12,12,14]},"label":"minimum_effective_span_4955","median_ms":814.9677734375,"n":8,"spread_pct":4.857886563286748},{"features":{"max_depth":2,"packed_tokens":18411,"segment_count":14,"segments_below":[2,4,6,8,11,11,11,14]},"label":"minimum_effective_span_4994","median_ms":1145.12451171875,"n":8,"spread_pct":1.8551923810823316},{"features":{"max_depth":2,"packed_tokens":23107,"segment_count":14,"segments_below":[2,3,5,7,10,10,10,14]},"label":"minimum_effective_span_5048","median_ms":1164.4971313476562,"n":8,"spread_pct":2.643549459118164},{"features":{"max_depth":2,"packed_tokens":27945,"segment_count":14,"segments_below":[2,3,5,6,9,9,9,14]},"label":"minimum_effective_span_5087","median_ms":1460.0728759765625,"n":8,"spread_pct":0.7123450484607493},{"features":{"max_depth":2,"packed_tokens":32845,"segment_count":14,"segments_below":[1,3,4,6,8,8,8,14]},"label":"minimum_effective_span_5118","median_ms":1565.3167724609375,"n":8,"spread_pct":1.883463989721654},{"features":{"max_depth":2,"packed_tokens":37693,"segment_count":14,"segments_below":[2,2,3,6,7,7,7,14]},"label":"minimum_effective_span_5149","median_ms":1820.8956298828125,"n":8,"spread_pct":1.1516160629068866},{"features":{"max_depth":2,"packed_tokens":42711,"segment_count":14,"segments_below":[1,2,3,6,6,6,6,14]},"label":"minimum_effective_span_5199","median_ms":1939.9025268554688,"n":8,"spread_pct":1.0320934257334569},{"features":{"max_depth":3,"packed_tokens":8746,"segment_count":14,"segments_below":[2,4,7,10,13,13,13,14]},"label":"minimum_effective_span_522","median_ms":687.1361389160156,"n":8,"spread_pct":6.190048115397722},{"features":{"max_depth":2,"packed_tokens":47673,"segment_count":14,"segments_below":[1,1,2,5,5,5,5,14]},"label":"minimum_effective_span_5235","median_ms":2217.032958984375,"n":8,"spread_pct":0.7951906443443744},{"features":{"max_depth":2,"packed_tokens":52699,"segment_count":14,"segments_below":[0,1,2,4,4,4,4,14]},"label":"minimum_effective_span_5303","median_ms":2339.1036376953125,"n":8,"spread_pct":0.7962451812674262},{"features":{"max_depth":2,"packed_tokens":57931,"segment_count":14,"segments_below":[0,0,3,3,3,3,3,14]},"label":"minimum_effective_span_5395","median_ms":2610.714599609375,"n":8,"spread_pct":1.125619734703937},{"features":{"max_depth":2,"packed_tokens":63240,"segment_count":13,"segments_below":[1,1,1,1,1,1,1,13]},"label":"minimum_effective_span_5476","median_ms":2698.513427734375,"n":8,"spread_pct":0.8406592496899741},{"features":{"max_depth":8,"packed_tokens":6558,"segment_count":19,"segments_below":[6,15,18,18,18,18,18,19]},"label":"minimum_effective_span_71","median_ms":1298.7864379882812,"n":8,"spread_pct":8.796344356353167},{"features":{"max_depth":7,"packed_tokens":6639,"segment_count":18,"segments_below":[6,14,16,17,17,17,17,18]},"label":"minimum_effective_span_82","median_ms":1185.1766357421875,"n":8,"spread_pct":8.777246848197136},{"features":{"max_depth":7,"packed_tokens":6657,"segment_count":18,"segments_below":[6,14,16,17,17,17,17,18]},"label":"minimum_effective_span_87","median_ms":1178.5443725585938,"n":8,"spread_pct":9.573018059563202},{"features":{"max_depth":7,"packed_tokens":6668,"segment_count":18,"segments_below":[6,12,17,17,17,17,17,18]},"label":"minimum_effective_span_93","median_ms":1183.8182983398438,"n":8,"spread_pct":8.90212711810708},{"features":{"max_depth":7,"packed_tokens":6645,"segment_count":18,"segments_below":[7,12,17,17,17,17,17,18]},"label":"minimum_effective_span_94","median_ms":1190.3358154296875,"n":8,"spread_pct":8.795771528743398},{"features":{"max_depth":1,"packed_tokens":68881,"segment_count":13,"segments_below":[0,0,0,0,0,0,0,13]},"label":"no_sharing","median_ms":2976.0057373046875,"n":8,"spread_pct":1.1236681861301239},{"features":{"max_depth":11,"packed_tokens":6620,"segment_count":23,"segments_below":[16,19,20,22,22,22,22,23]},"label":"uniform_depth_10","median_ms":1517.1295776367188,"n":8,"spread_pct":11.455396786960561},{"features":{"max_depth":12,"packed_tokens":6458,"segment_count":24,"segments_below":[16,20,23,23,23,23,23,24]},"label":"uniform_depth_11","median_ms":1620.676025390625,"n":8,"spread_pct":9.90263454935451},{"features":{"max_depth":13,"packed_tokens":6292,"segment_count":24,"segments_below":[17,21,23,23,23,23,23,24]},"label":"uniform_depth_12","median_ms":1728.6766357421875,"n":8,"spread_pct":11.51338374624567},{"features":{"max_depth":3,"packed_tokens":9004,"segment_count":15,"segments_below":[3,5,8,11,14,14,14,15]},"label":"uniform_depth_2","median_ms":761.2681884765625,"n":8,"spread_pct":1.4101049194323116},{"features":{"max_depth":4,"packed_tokens":8464,"segment_count":16,"segments_below":[6,8,10,12,15,15,15,16]},"label":"uniform_depth_3","median_ms":844.6612854003906,"n":8,"spread_pct":2.544864287019868},{"features":{"max_depth":5,"packed_tokens":8113,"segment_count":17,"segments_below":[8,9,11,13,16,16,16,17]},"label":"uniform_depth_4","median_ms":954.4981994628906,"n":8,"spread_pct":1.7455640002779038},{"features":{"max_depth":6,"packed_tokens":7865,"segment_count":18,"segments_below":[10,11,13,14,17,17,17,18]},"label":"uniform_depth_5","median_ms":1050.383056640625,"n":8,"spread_pct":4.21597961495167},{"features":{"max_depth":7,"packed_tokens":7648,"segment_count":19,"segments_below":[11,13,14,16,18,18,18,19]},"label":"uniform_depth_6","median_ms":1137.7999877929688,"n":8,"spread_pct":6.286664075703335},{"features":{"max_depth":8,"packed_tokens":7348,"segment_count":20,"segments_below":[14,14,15,18,19,19,19,20]},"label":"uniform_depth_7","median_ms":1247.5393676757812,"n":8,"spread_pct":10.047512963140088},{"features":{"max_depth":9,"packed_tokens":7168,"segment_count":21,"segments_below":[15,16,17,20,20,20,20,21]},"label":"uniform_depth_8","median_ms":1322.2769775390625,"n":8,"spread_pct":8.954668317563375},{"features":{"max_depth":10,"packed_tokens":6896,"segment_count":22,"segments_below":[16,17,18,21,21,21,21,22]},"label":"uniform_depth_9","median_ms":1416.5543212890625,"n":8,"spread_pct":11.92124065215535}],"cell":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp2|g6","device":"NVIDIA H200","facts":{"cp":2.0,"gdn_layers":24.0,"layers":32.0,"tp":1.0,"uses_gdn":1.0},"hidden_size":2560,"param_dtype":"torch.bfloat16","requests_sha256":"22fe33a263cacb6bc86ab088fe07af0a0fb8e68bfdc85c518a57ec37e403d138","source":"797746b9f425d844419fe609c164c2e4940507f3bbb2f82b815a73740d5abc2d","workload":{"group":6,"kind":"ellavox"}}, {"candidates":[{"features":{"max_depth":2,"packed_tokens":12674,"segment_count":9,"segments_below":[1,4,8,8,8,8,8,8]},"label":"depth_one","median_ms":1063.86669921875,"n":8,"spread_pct":1.5990460799734165},{"features":{"max_depth":2,"packed_tokens":24329,"segment_count":9,"segments_below":[2,4,7,7,7,7,7,7]},"label":"minimum_effective_span_11722","median_ms":1252.4087524414062,"n":8,"spread_pct":6.7657027120346465},{"features":{"max_depth":2,"packed_tokens":36051,"segment_count":9,"segments_below":[1,3,6,6,6,6,6,6]},"label":"minimum_effective_span_11733","median_ms":1997.3419189453125,"n":8,"spread_pct":1.1945441405432329},{"features":{"max_depth":2,"packed_tokens":47693,"segment_count":9,"segments_below":[1,2,5,5,5,5,5,5]},"label":"minimum_effective_span_11735","median_ms":2256.8770751953125,"n":8,"spread_pct":0.7773549346271652},{"features":{"max_depth":2,"packed_tokens":59300,"segment_count":9,"segments_below":[1,4,4,4,4,4,4,4]},"label":"minimum_effective_span_11758","median_ms":2981.688720703125,"n":8,"spread_pct":1.0352250147936661},{"features":{"max_depth":2,"packed_tokens":71013,"segment_count":9,"segments_below":[2,3,3,3,3,3,3,3]},"label":"minimum_effective_span_11808","median_ms":3237.851806640625,"n":8,"spread_pct":0.8625012922024264},{"features":{"max_depth":2,"packed_tokens":82862,"segment_count":9,"segments_below":[1,2,2,2,2,2,2,2]},"label":"minimum_effective_span_11855","median_ms":3907.3223876953125,"n":8,"spread_pct":0.42582684722334824},{"features":{"max_depth":6,"packed_tokens":12273,"segment_count":13,"segments_below":[9,12,12,12,12,12,12,12]},"label":"minimum_effective_span_12","median_ms":1268.52685546875,"n":8,"spread_pct":20.134583399250868},{"features":{"max_depth":3,"packed_tokens":12536,"segment_count":10,"segments_below":[2,6,9,9,9,9,9,9]},"label":"minimum_effective_span_134","median_ms":1062.8412475585938,"n":8,"spread_pct":51.46326223005927},{"features":{"max_depth":5,"packed_tokens":12286,"segment_count":12,"segments_below":[8,11,11,11,11,11,11,11]},"label":"minimum_effective_span_14","median_ms":1195.7191772460938,"n":8,"spread_pct":7.734133347094992},{"features":{"max_depth":7,"packed_tokens":12259,"segment_count":14,"segments_below":[11,13,13,13,13,13,13,13]},"label":"minimum_effective_span_3","median_ms":1364.6408081054688,"n":8,"spread_pct":11.493865246759869},{"features":{"max_depth":4,"packed_tokens":12322,"segment_count":11,"segments_below":[5,10,10,10,10,10,10,10]},"label":"minimum_effective_span_37","median_ms":1149.483642578125,"n":8,"spread_pct":13.78543354958334},{"features":{"max_depth":4,"packed_tokens":12364,"segment_count":11,"segments_below":[4,10,10,10,10,10,10,10]},"label":"minimum_effective_span_48","median_ms":1134.5525512695312,"n":8,"spread_pct":17.660050087933357},{"features":{"max_depth":3,"packed_tokens":12416,"segment_count":10,"segments_below":[2,9,9,9,9,9,9,9]},"label":"minimum_effective_span_53","median_ms":1063.8711547851562,"n":8,"spread_pct":2.676050237328723},{"features":{"max_depth":6,"packed_tokens":12264,"segment_count":13,"segments_below":[10,12,12,12,12,12,12,12]},"label":"minimum_effective_span_6","median_ms":1255.8862915039062,"n":8,"spread_pct":9.430289718226783},{"features":{"max_depth":3,"packed_tokens":12408,"segment_count":10,"segments_below":[3,7,9,9,9,9,9,9]},"label":"minimum_effective_span_87","median_ms":1059.7821044921875,"n":8,"spread_pct":5.467144528138673},{"features":{"max_depth":1,"packed_tokens":94721,"segment_count":8,"segments_below":[0,0,0,0,0,0,0,0]},"label":"no_sharing","median_ms":4182.67724609375,"n":8,"spread_pct":0.6651433793327506},{"features":{"max_depth":3,"packed_tokens":12608,"segment_count":10,"segments_below":[4,6,9,9,9,9,9,9]},"label":"uniform_depth_2","median_ms":1061.4689331054688,"n":8,"spread_pct":2.1436358249075234},{"features":{"max_depth":4,"packed_tokens":12598,"segment_count":11,"segments_below":[5,7,10,10,10,10,10,10]},"label":"uniform_depth_3","median_ms":1119.1444702148438,"n":8,"spread_pct":4.975222870907457},{"features":{"max_depth":5,"packed_tokens":12506,"segment_count":12,"segments_below":[6,8,11,11,11,11,11,11]},"label":"uniform_depth_4","median_ms":1178.3833618164062,"n":8,"spread_pct":8.540795280047018},{"features":{"max_depth":6,"packed_tokens":12356,"segment_count":13,"segments_below":[8,12,12,12,12,12,12,12]},"label":"uniform_depth_5","median_ms":1257.1089477539062,"n":8,"spread_pct":8.815596728624362},{"features":{"max_depth":7,"packed_tokens":12262,"segment_count":14,"segments_below":[11,13,13,13,13,13,13,13]},"label":"uniform_depth_6","median_ms":1350.4036254882812,"n":8,"spread_pct":9.75590839200863},{"features":{"max_depth":8,"packed_tokens":12257,"segment_count":15,"segments_below":[12,14,14,14,14,14,14,14]},"label":"uniform_depth_7","median_ms":1446.935791015625,"n":8,"spread_pct":12.13752493187972}],"cell":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp2|g7","device":"NVIDIA H200","facts":{"cp":2.0,"gdn_layers":24.0,"layers":32.0,"tp":1.0,"uses_gdn":1.0},"hidden_size":2560,"param_dtype":"torch.bfloat16","requests_sha256":"d04b77db2ed7d695d20c6f183215ead28d0337ae88b9ddbc3674926310d364b9","source":"797746b9f425d844419fe609c164c2e4940507f3bbb2f82b815a73740d5abc2d","workload":{"group":7,"kind":"ellavox"}}, {"candidates":[{"features":{"max_depth":2,"packed_tokens":12659,"segment_count":5,"segments_below":[1,3,4,4,4,4,4,4]},"label":"depth_one","median_ms":955.1390380859375,"n":8,"spread_pct":0.8377597083212612},{"features":{"max_depth":2,"packed_tokens":24836,"segment_count":5,"segments_below":[2,2,3,3,3,3,3,3]},"label":"minimum_effective_span_12268","median_ms":1018.6960754394531,"n":8,"spread_pct":0.6115582050226549},{"features":{"max_depth":2,"packed_tokens":37143,"segment_count":5,"segments_below":[1,2,2,2,2,2,2,2]},"label":"minimum_effective_span_12313","median_ms":1110.6633911132812,"n":8,"spread_pct":1.072357152996116},{"features":{"max_depth":3,"packed_tokens":12609,"segment_count":6,"segments_below":[3,5,5,5,5,5,5,5]},"label":"minimum_effective_span_46","median_ms":961.5490112304688,"n":8,"spread_pct":2.4396377838569316},{"features":{"max_depth":1,"packed_tokens":49460,"segment_count":4,"segments_below":[0,0,0,0,0,0,0,0]},"label":"no_sharing","median_ms":1183.0565795898438,"n":8,"spread_pct":0.6518944576692424},{"features":{"max_depth":3,"packed_tokens":12569,"segment_count":6,"segments_below":[4,4,5,5,5,5,5,5]},"label":"uniform_depth_2","median_ms":956.9637756347656,"n":8,"spread_pct":0.7843601473305404},{"features":{"max_depth":4,"packed_tokens":12564,"segment_count":7,"segments_below":[5,6,6,6,6,6,6,6]},"label":"uniform_depth_3","median_ms":993.3770446777344,"n":8,"spread_pct":1.5156103292622936}],"cell":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g0","device":"NVIDIA H200","facts":{"cp":4.0,"gdn_layers":24.0,"layers":32.0,"tp":1.0,"uses_gdn":1.0},"hidden_size":2560,"param_dtype":"torch.bfloat16","requests_sha256":"f8b37d8f36ed38a4d2ea74f20bfe0d15bedbdb66ffaae41eb728a1756d67d5ff","source":"b6f0643a1534e30cb7369eca048cfaff34a7cd158047dd1c599068464a415e97","workload":{"group":0,"kind":"ellavox"}}, + {"candidates":[{"features":{"max_depth":2,"packed_tokens":12382,"segment_count":3,"segments_below":[1,2,2,2,2,2,2,2]},"label":"depth_one","median_ms":929.7828063964844,"n":8,"spread_pct":0.97942296684856},{"features":{"max_depth":1,"packed_tokens":24645,"segment_count":2,"segments_below":[0,0,0,0,0,0,0,0]},"label":"no_sharing","median_ms":1001.3226318359375,"n":8,"spread_pct":0.8318404551597101}],"cell":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g1","device":"NVIDIA H200","facts":{"cp":4.0,"gdn_layers":24.0,"layers":32.0,"tp":1.0,"uses_gdn":1.0},"hidden_size":2560,"param_dtype":"torch.bfloat16","requests_sha256":"515df421b93fe5f98e250ae3d849975dccadfd7dc00046242b5875ddf392c74f","source":"b2f0b56f18bce1f8ab5b68d4101827fb697d96e0d0f9148deaaf91fde626f85d","workload":{"group":1,"kind":"ellavox"}}, {"candidates":[{"features":{"max_depth":1,"packed_tokens":12546,"segment_count":1,"segments_below":[0,0,0,0,0,0,0,0]},"label":"no_sharing","median_ms":938.0733947753906,"n":8,"spread_pct":0.5994510714133858}],"cell":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g2","device":"NVIDIA H200","facts":{"cp":4.0,"gdn_layers":24.0,"layers":32.0,"tp":1.0,"uses_gdn":1.0},"hidden_size":2560,"param_dtype":"torch.bfloat16","requests_sha256":"8aa85999c5d3c576d127bd2e3a44436cbd0e9062c7c92b16bcbaab0814703094","source":"b6f0643a1534e30cb7369eca048cfaff34a7cd158047dd1c599068464a415e97","workload":{"group":2,"kind":"ellavox"}}, {"candidates":[{"features":{"max_depth":2,"packed_tokens":47253,"segment_count":6,"segments_below":[0,0,0,1,1,1,1,3]},"label":"depth_one","median_ms":1400.5692749023438,"n":8,"spread_pct":0.6754542776032713},{"features":{"max_depth":2,"packed_tokens":23150,"segment_count":7,"segments_below":[0,0,0,1,3,5,5,6]},"label":"minimum_effective_span_1366","median_ms":947.1145629882812,"n":8,"spread_pct":0.8672581484482867},{"features":{"max_depth":3,"packed_tokens":21785,"segment_count":7,"segments_below":[0,0,0,1,4,5,5,6]},"label":"minimum_effective_span_305","median_ms":936.1377258300781,"n":8,"spread_pct":0.9790528626541672},{"features":{"max_depth":2,"packed_tokens":28799,"segment_count":6,"segments_below":[0,0,0,1,3,3,3,4]},"label":"minimum_effective_span_7015","median_ms":972.6693115234375,"n":8,"spread_pct":1.3475848153143526},{"features":{"max_depth":2,"packed_tokens":37178,"segment_count":6,"segments_below":[0,0,0,1,2,2,2,3]},"label":"minimum_effective_span_8380","median_ms":981.3615417480469,"n":8,"spread_pct":1.3315253431923837},{"features":{"max_depth":1,"packed_tokens":48469,"segment_count":5,"segments_below":[0,0,0,0,0,0,0,1]},"label":"no_sharing","median_ms":1398.450927734375,"n":8,"spread_pct":0.5363862602576267},{"features":{"max_depth":3,"packed_tokens":22846,"segment_count":8,"segments_below":[0,0,0,2,4,6,6,7]},"label":"uniform_depth_2","median_ms":968.9115905761719,"n":8,"spread_pct":0.8283837483635463},{"features":{"max_depth":4,"packed_tokens":21481,"segment_count":8,"segments_below":[0,0,0,2,5,6,6,7]},"label":"uniform_depth_3","median_ms":908.8069152832031,"n":8,"spread_pct":1.407182758371157}],"cell":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g3","device":"NVIDIA H200","facts":{"cp":4.0,"gdn_layers":24.0,"layers":32.0,"tp":1.0,"uses_gdn":1.0},"hidden_size":2560,"param_dtype":"torch.bfloat16","requests_sha256":"62112dd684879b09adbef34aa98e5ea3966b7de9b9e8067de5920b1d73f07197","source":"b6f0643a1534e30cb7369eca048cfaff34a7cd158047dd1c599068464a415e97","workload":{"group":3,"kind":"ellavox"}}, + {"candidates":[{"features":{"max_depth":2,"packed_tokens":30477,"segment_count":4,"segments_below":[0,0,0,1,1,1,1,2]},"label":"depth_one","median_ms":1026.1570434570312,"n":8,"spread_pct":1.5202862955349385},{"features":{"max_depth":2,"packed_tokens":19821,"segment_count":4,"segments_below":[0,0,0,2,2,2,2,3]},"label":"minimum_effective_span_495","median_ms":911.28955078125,"n":8,"spread_pct":0.42354048419323365},{"features":{"max_depth":1,"packed_tokens":31465,"segment_count":3,"segments_below":[0,0,0,0,0,0,0,1]},"label":"no_sharing","median_ms":1025.56787109375,"n":8,"spread_pct":0.22727023852543585},{"features":{"max_depth":3,"packed_tokens":19327,"segment_count":5,"segments_below":[0,0,0,3,3,3,3,4]},"label":"uniform_depth_2","median_ms":886.4833984375,"n":8,"spread_pct":0.5654309128303878}],"cell":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g4","device":"NVIDIA H200","facts":{"cp":4.0,"gdn_layers":24.0,"layers":32.0,"tp":1.0,"uses_gdn":1.0},"hidden_size":2560,"param_dtype":"torch.bfloat16","requests_sha256":"28b4492a5c1ce50958108e27b72b7afdf09941e4670076dfa879b517d9cc7575","source":"b2f0b56f18bce1f8ab5b68d4101827fb697d96e0d0f9148deaaf91fde626f85d","workload":{"group":4,"kind":"ellavox"}}, {"candidates":[{"features":{"max_depth":2,"packed_tokens":13557,"segment_count":6,"segments_below":[0,1,3,5,5,5,5,5]},"label":"depth_one","median_ms":982.8792724609375,"n":8,"spread_pct":0.7863448764597297},{"features":{"max_depth":2,"packed_tokens":25902,"segment_count":6,"segments_below":[0,1,2,4,4,4,4,4]},"label":"minimum_effective_span_12406","median_ms":1065.890869140625,"n":8,"spread_pct":0.5699297439765417},{"features":{"max_depth":2,"packed_tokens":37995,"segment_count":6,"segments_below":[1,2,3,3,3,3,3,3]},"label":"minimum_effective_span_12426","median_ms":1152.84423828125,"n":8,"spread_pct":57.03694370029966},{"features":{"max_depth":2,"packed_tokens":50582,"segment_count":6,"segments_below":[1,2,2,2,2,2,2,2]},"label":"minimum_effective_span_12592","median_ms":1291.6544799804688,"n":8,"spread_pct":0.9010197157070582},{"features":{"max_depth":3,"packed_tokens":13367,"segment_count":7,"segments_below":[1,3,5,6,6,6,6,6]},"label":"minimum_effective_span_187","median_ms":994.9620056152344,"n":8,"spread_pct":1.372282202373622},{"features":{"max_depth":3,"packed_tokens":13185,"segment_count":7,"segments_below":[1,3,6,6,6,6,6,6]},"label":"minimum_effective_span_21","median_ms":988.8197631835938,"n":8,"spread_pct":0.9538095688101772},{"features":{"max_depth":1,"packed_tokens":63177,"segment_count":5,"segments_below":[0,0,0,0,0,0,0,0]},"label":"no_sharing","median_ms":1572.5120239257812,"n":8,"spread_pct":0.2923222009035308},{"features":{"max_depth":3,"packed_tokens":13497,"segment_count":7,"segments_below":[1,2,4,6,6,6,6,6]},"label":"uniform_depth_2","median_ms":988.2910766601562,"n":8,"spread_pct":1.472805266696644},{"features":{"max_depth":4,"packed_tokens":13165,"segment_count":8,"segments_below":[2,4,7,7,7,7,7,7]},"label":"uniform_depth_3","median_ms":1039.166259765625,"n":8,"spread_pct":1.535502211362822},{"features":{"max_depth":5,"packed_tokens":13161,"segment_count":9,"segments_below":[3,5,8,8,8,8,8,8]},"label":"uniform_depth_4","median_ms":1116.9856567382812,"n":8,"spread_pct":2.8821312941537243}],"cell":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g5","device":"NVIDIA H200","facts":{"cp":4.0,"gdn_layers":24.0,"layers":32.0,"tp":1.0,"uses_gdn":1.0},"hidden_size":2560,"param_dtype":"torch.bfloat16","requests_sha256":"7241823ac1b67d127c5921d771dda170dcc9d4886feffaff3c2520113cd01a08","source":"b6f0643a1534e30cb7369eca048cfaff34a7cd158047dd1c599068464a415e97","workload":{"group":5,"kind":"ellavox"}}, {"candidates":[{"features":{"max_depth":2,"packed_tokens":9433,"segment_count":14,"segments_below":[1,3,6,9,13,13,13,14]},"label":"depth_one","median_ms":532.6179809570312,"n":8,"spread_pct":7.694432981799052},{"features":{"max_depth":6,"packed_tokens":6738,"segment_count":17,"segments_below":[6,11,15,16,16,16,16,17]},"label":"minimum_effective_span_105","median_ms":891.9311218261719,"n":8,"spread_pct":7.374314238107294},{"features":{"max_depth":6,"packed_tokens":6778,"segment_count":17,"segments_below":[5,10,15,16,16,16,16,17]},"label":"minimum_effective_span_113","median_ms":888.4822998046875,"n":8,"spread_pct":6.924011399684575},{"features":{"max_depth":6,"packed_tokens":6877,"segment_count":17,"segments_below":[5,8,16,16,16,16,16,17]},"label":"minimum_effective_span_133","median_ms":905.1231079101562,"n":8,"spread_pct":6.022352552207125},{"features":{"max_depth":5,"packed_tokens":7090,"segment_count":16,"segments_below":[3,8,14,15,15,15,15,16]},"label":"minimum_effective_span_164","median_ms":811.3995361328125,"n":8,"spread_pct":5.291135755534277},{"features":{"max_depth":5,"packed_tokens":7134,"segment_count":16,"segments_below":[4,7,14,15,15,15,15,16]},"label":"minimum_effective_span_195","median_ms":804.4127197265625,"n":8,"spread_pct":6.203347047057483},{"features":{"max_depth":4,"packed_tokens":7415,"segment_count":16,"segments_below":[3,5,13,15,15,15,15,16]},"label":"minimum_effective_span_197","median_ms":722.187255859375,"n":8,"spread_pct":5.794639020796943},{"features":{"max_depth":4,"packed_tokens":7626,"segment_count":15,"segments_below":[3,6,10,14,14,14,14,15]},"label":"minimum_effective_span_245","median_ms":648.9917907714844,"n":8,"spread_pct":7.18088166633539},{"features":{"max_depth":4,"packed_tokens":7702,"segment_count":15,"segments_below":[3,5,9,14,14,14,14,15]},"label":"minimum_effective_span_281","median_ms":648.9981384277344,"n":8,"spread_pct":6.823138748026168},{"features":{"max_depth":12,"packed_tokens":6323,"segment_count":23,"segments_below":[16,20,22,22,22,22,22,23]},"label":"minimum_effective_span_32","median_ms":1454.529541015625,"n":8,"spread_pct":5.0826628764233766},{"features":{"max_depth":3,"packed_tokens":8041,"segment_count":15,"segments_below":[2,4,8,14,14,14,14,15]},"label":"minimum_effective_span_340","median_ms":601.324951171875,"n":8,"spread_pct":158.67452934945496},{"features":{"max_depth":3,"packed_tokens":8113,"segment_count":15,"segments_below":[1,4,8,14,14,14,14,15]},"label":"minimum_effective_span_349","median_ms":589.7940063476562,"n":8,"spread_pct":8.308968523318141},{"features":{"max_depth":11,"packed_tokens":6359,"segment_count":22,"segments_below":[15,19,21,21,21,21,21,22]},"label":"minimum_effective_span_37","median_ms":1369.0506591796875,"n":8,"spread_pct":5.966137396867109},{"features":{"max_depth":9,"packed_tokens":6437,"segment_count":20,"segments_below":[9,17,19,19,19,19,19,20]},"label":"minimum_effective_span_40","median_ms":1182.49609375,"n":8,"spread_pct":7.027102692596104},{"features":{"max_depth":3,"packed_tokens":8391,"segment_count":15,"segments_below":[1,3,9,12,14,14,14,15]},"label":"minimum_effective_span_441","median_ms":579.4763793945312,"n":8,"spread_pct":6.3580264923388015},{"features":{"max_depth":2,"packed_tokens":13958,"segment_count":14,"segments_below":[1,3,6,9,12,12,12,14]},"label":"minimum_effective_span_4955","median_ms":571.3086547851562,"n":8,"spread_pct":1.598994307994352},{"features":{"max_depth":2,"packed_tokens":18411,"segment_count":14,"segments_below":[2,4,6,8,11,11,11,14]},"label":"minimum_effective_span_4994","median_ms":586.8535461425781,"n":8,"spread_pct":3.051843375905492},{"features":{"max_depth":2,"packed_tokens":23107,"segment_count":14,"segments_below":[2,3,5,7,10,10,10,14]},"label":"minimum_effective_span_5048","median_ms":657.2309265136719,"n":8,"spread_pct":1.693989494776293},{"features":{"max_depth":2,"packed_tokens":27945,"segment_count":14,"segments_below":[2,3,5,6,9,9,9,14]},"label":"minimum_effective_span_5087","median_ms":867.4882507324219,"n":8,"spread_pct":0.2590951090125521},{"features":{"max_depth":2,"packed_tokens":32845,"segment_count":14,"segments_below":[1,3,4,6,8,8,8,14]},"label":"minimum_effective_span_5118","median_ms":1052.4736938476562,"n":8,"spread_pct":0.6342015697416724},{"features":{"max_depth":2,"packed_tokens":37693,"segment_count":14,"segments_below":[2,2,3,6,7,7,7,14]},"label":"minimum_effective_span_5149","median_ms":1059.2788696289062,"n":8,"spread_pct":3.654887793157392},{"features":{"max_depth":2,"packed_tokens":42711,"segment_count":14,"segments_below":[1,2,3,6,6,6,6,14]},"label":"minimum_effective_span_5199","median_ms":1069.5327758789062,"n":8,"spread_pct":0.8512245890810625},{"features":{"max_depth":3,"packed_tokens":8746,"segment_count":14,"segments_below":[2,4,7,10,13,13,13,14]},"label":"minimum_effective_span_522","median_ms":588.8303527832031,"n":8,"spread_pct":8.097231206367438},{"features":{"max_depth":2,"packed_tokens":47673,"segment_count":14,"segments_below":[1,1,2,5,5,5,5,14]},"label":"minimum_effective_span_5235","median_ms":1366.17431640625,"n":8,"spread_pct":0.667941420874398},{"features":{"max_depth":2,"packed_tokens":52699,"segment_count":14,"segments_below":[0,1,2,4,4,4,4,14]},"label":"minimum_effective_span_5303","median_ms":1383.7882690429688,"n":8,"spread_pct":0.5156760190450004},{"features":{"max_depth":2,"packed_tokens":57931,"segment_count":14,"segments_below":[0,0,3,3,3,3,3,14]},"label":"minimum_effective_span_5395","median_ms":1439.2272338867188,"n":8,"spread_pct":0.7374121931193492},{"features":{"max_depth":2,"packed_tokens":63240,"segment_count":13,"segments_below":[1,1,1,1,1,1,1,13]},"label":"minimum_effective_span_5476","median_ms":1411.630859375,"n":8,"spread_pct":0.44204121728663237},{"features":{"max_depth":8,"packed_tokens":6558,"segment_count":19,"segments_below":[6,15,18,18,18,18,18,19]},"label":"minimum_effective_span_71","median_ms":1082.0573120117188,"n":8,"spread_pct":5.620284303847149},{"features":{"max_depth":7,"packed_tokens":6639,"segment_count":18,"segments_below":[6,14,16,17,17,17,17,18]},"label":"minimum_effective_span_82","median_ms":987.9519958496094,"n":8,"spread_pct":6.582547526718928},{"features":{"max_depth":7,"packed_tokens":6657,"segment_count":18,"segments_below":[6,14,16,17,17,17,17,18]},"label":"minimum_effective_span_87","median_ms":991.8036499023438,"n":8,"spread_pct":5.959342907698482},{"features":{"max_depth":7,"packed_tokens":6668,"segment_count":18,"segments_below":[6,12,17,17,17,17,17,18]},"label":"minimum_effective_span_93","median_ms":991.577392578125,"n":8,"spread_pct":5.675137098328918},{"features":{"max_depth":7,"packed_tokens":6645,"segment_count":18,"segments_below":[7,12,17,17,17,17,17,18]},"label":"minimum_effective_span_94","median_ms":985.685546875,"n":8,"spread_pct":8.272945394128056},{"features":{"max_depth":1,"packed_tokens":68881,"segment_count":13,"segments_below":[0,0,0,0,0,0,0,13]},"label":"no_sharing","median_ms":1718.9435424804688,"n":8,"spread_pct":0.6078647890892683},{"features":{"max_depth":11,"packed_tokens":6620,"segment_count":23,"segments_below":[16,19,20,22,22,22,22,23]},"label":"uniform_depth_10","median_ms":1363.5620727539062,"n":8,"spread_pct":6.273367489084111},{"features":{"max_depth":12,"packed_tokens":6458,"segment_count":24,"segments_below":[16,20,23,23,23,23,23,24]},"label":"uniform_depth_11","median_ms":1465.7698974609375,"n":8,"spread_pct":6.364117953090825},{"features":{"max_depth":13,"packed_tokens":6292,"segment_count":24,"segments_below":[17,21,23,23,23,23,23,24]},"label":"uniform_depth_12","median_ms":1546.1495361328125,"n":8,"spread_pct":6.410226955397406},{"features":{"max_depth":3,"packed_tokens":9004,"segment_count":15,"segments_below":[3,5,8,11,14,14,14,15]},"label":"uniform_depth_2","median_ms":692.7481994628906,"n":8,"spread_pct":1.5841957168847736},{"features":{"max_depth":4,"packed_tokens":8464,"segment_count":16,"segments_below":[6,8,10,12,15,15,15,16]},"label":"uniform_depth_3","median_ms":790.9555053710938,"n":8,"spread_pct":2.109928879625496},{"features":{"max_depth":5,"packed_tokens":8113,"segment_count":17,"segments_below":[8,9,11,13,16,16,16,17]},"label":"uniform_depth_4","median_ms":889.405029296875,"n":8,"spread_pct":2.469610922637176},{"features":{"max_depth":6,"packed_tokens":7865,"segment_count":18,"segments_below":[10,11,13,14,17,17,17,18]},"label":"uniform_depth_5","median_ms":876.6935119628906,"n":8,"spread_pct":7.109684092517226},{"features":{"max_depth":7,"packed_tokens":7648,"segment_count":19,"segments_below":[11,13,14,16,18,18,18,19]},"label":"uniform_depth_6","median_ms":970.5019226074219,"n":8,"spread_pct":2.505524362491257},{"features":{"max_depth":8,"packed_tokens":7348,"segment_count":20,"segments_below":[14,14,15,18,19,19,19,20]},"label":"uniform_depth_7","median_ms":1069.3145751953125,"n":8,"spread_pct":2.7593359456381834},{"features":{"max_depth":9,"packed_tokens":7168,"segment_count":21,"segments_below":[15,16,17,20,20,20,20,21]},"label":"uniform_depth_8","median_ms":1053.6236572265625,"n":8,"spread_pct":1.7719493679098766},{"features":{"max_depth":10,"packed_tokens":6896,"segment_count":22,"segments_below":[16,17,18,21,21,21,21,22]},"label":"uniform_depth_9","median_ms":1129.0491333007812,"n":8,"spread_pct":4.152794026231866}],"cell":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g6","device":"NVIDIA H200","facts":{"cp":4.0,"gdn_layers":24.0,"layers":32.0,"tp":1.0,"uses_gdn":1.0},"hidden_size":2560,"param_dtype":"torch.bfloat16","requests_sha256":"22fe33a263cacb6bc86ab088fe07af0a0fb8e68bfdc85c518a57ec37e403d138","source":"b6f0643a1534e30cb7369eca048cfaff34a7cd158047dd1c599068464a415e97","workload":{"group":6,"kind":"ellavox"}}, {"candidates":[{"features":{"max_depth":2,"packed_tokens":12674,"segment_count":9,"segments_below":[1,4,8,8,8,8,8,8]},"label":"depth_one","median_ms":948.4121398925781,"n":8,"spread_pct":1.5174095502772433},{"features":{"max_depth":2,"packed_tokens":24329,"segment_count":9,"segments_below":[2,4,7,7,7,7,7,7]},"label":"minimum_effective_span_11722","median_ms":998.7579345703125,"n":8,"spread_pct":1.0313530294280309},{"features":{"max_depth":2,"packed_tokens":36051,"segment_count":9,"segments_below":[1,3,6,6,6,6,6,6]},"label":"minimum_effective_span_11733","median_ms":1075.5050659179688,"n":8,"spread_pct":0.6536037103298181},{"features":{"max_depth":2,"packed_tokens":47693,"segment_count":9,"segments_below":[1,2,5,5,5,5,5,5]},"label":"minimum_effective_span_11735","median_ms":1181.8339233398438,"n":8,"spread_pct":0.5177045676463979},{"features":{"max_depth":2,"packed_tokens":59300,"segment_count":9,"segments_below":[1,4,4,4,4,4,4,4]},"label":"minimum_effective_span_11758","median_ms":1932.0086669921875,"n":8,"spread_pct":1.0736326464592345},{"features":{"max_depth":2,"packed_tokens":71013,"segment_count":9,"segments_below":[2,3,3,3,3,3,3,3]},"label":"minimum_effective_span_11808","median_ms":2016.448486328125,"n":8,"spread_pct":0.5932108890105823},{"features":{"max_depth":2,"packed_tokens":82862,"segment_count":9,"segments_below":[1,2,2,2,2,2,2,2]},"label":"minimum_effective_span_11855","median_ms":2090.757568359375,"n":8,"spread_pct":0.6305420421887953},{"features":{"max_depth":6,"packed_tokens":12273,"segment_count":13,"segments_below":[9,12,12,12,12,12,12,12]},"label":"minimum_effective_span_12","median_ms":1202.5869750976562,"n":8,"spread_pct":4.619415591946175},{"features":{"max_depth":3,"packed_tokens":12536,"segment_count":10,"segments_below":[2,6,9,9,9,9,9,9]},"label":"minimum_effective_span_134","median_ms":959.4686889648438,"n":8,"spread_pct":4.266480745626493},{"features":{"max_depth":5,"packed_tokens":12286,"segment_count":12,"segments_below":[8,11,11,11,11,11,11,11]},"label":"minimum_effective_span_14","median_ms":1112.2821655273438,"n":8,"spread_pct":3.9745975980308206},{"features":{"max_depth":7,"packed_tokens":12259,"segment_count":14,"segments_below":[11,13,13,13,13,13,13,13]},"label":"minimum_effective_span_3","median_ms":1310.5744018554688,"n":8,"spread_pct":4.643466179058926},{"features":{"max_depth":4,"packed_tokens":12322,"segment_count":11,"segments_below":[5,10,10,10,10,10,10,10]},"label":"minimum_effective_span_37","median_ms":1029.399658203125,"n":8,"spread_pct":3.7240114305473764},{"features":{"max_depth":4,"packed_tokens":12364,"segment_count":11,"segments_below":[4,10,10,10,10,10,10,10]},"label":"minimum_effective_span_48","median_ms":1021.7846984863281,"n":8,"spread_pct":3.943546592912857},{"features":{"max_depth":3,"packed_tokens":12416,"segment_count":10,"segments_below":[2,9,9,9,9,9,9,9]},"label":"minimum_effective_span_53","median_ms":961.0345153808594,"n":8,"spread_pct":2.0666295155224264},{"features":{"max_depth":6,"packed_tokens":12264,"segment_count":13,"segments_below":[10,12,12,12,12,12,12,12]},"label":"minimum_effective_span_6","median_ms":1203.8671264648438,"n":8,"spread_pct":5.885411895128203},{"features":{"max_depth":3,"packed_tokens":12408,"segment_count":10,"segments_below":[3,7,9,9,9,9,9,9]},"label":"minimum_effective_span_87","median_ms":958.3133239746094,"n":8,"spread_pct":3.102552890641303},{"features":{"max_depth":1,"packed_tokens":94721,"segment_count":8,"segments_below":[0,0,0,0,0,0,0,0]},"label":"no_sharing","median_ms":2158.0384521484375,"n":8,"spread_pct":1.1914144354113898},{"features":{"max_depth":3,"packed_tokens":12608,"segment_count":10,"segments_below":[4,6,9,9,9,9,9,9]},"label":"uniform_depth_2","median_ms":956.4935607910156,"n":8,"spread_pct":2.676516503325003},{"features":{"max_depth":4,"packed_tokens":12598,"segment_count":11,"segments_below":[5,7,10,10,10,10,10,10]},"label":"uniform_depth_3","median_ms":1025.6336975097656,"n":8,"spread_pct":7.1268819757459285},{"features":{"max_depth":5,"packed_tokens":12506,"segment_count":12,"segments_below":[6,8,11,11,11,11,11,11]},"label":"uniform_depth_4","median_ms":1117.3798217773438,"n":8,"spread_pct":4.959406299756854},{"features":{"max_depth":6,"packed_tokens":12356,"segment_count":13,"segments_below":[8,12,12,12,12,12,12,12]},"label":"uniform_depth_5","median_ms":1236.0386962890625,"n":8,"spread_pct":5.092533016961218},{"features":{"max_depth":7,"packed_tokens":12262,"segment_count":14,"segments_below":[11,13,13,13,13,13,13,13]},"label":"uniform_depth_6","median_ms":1316.658447265625,"n":8,"spread_pct":7.393622475516097},{"features":{"max_depth":8,"packed_tokens":12257,"segment_count":15,"segments_below":[12,14,14,14,14,14,14,14]},"label":"uniform_depth_7","median_ms":1416.7862548828125,"n":8,"spread_pct":10.48272608191766}],"cell":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g7","device":"NVIDIA H200","facts":{"cp":4.0,"gdn_layers":24.0,"layers":32.0,"tp":1.0,"uses_gdn":1.0},"hidden_size":2560,"param_dtype":"torch.bfloat16","requests_sha256":"d04b77db2ed7d695d20c6f183215ead28d0337ae88b9ddbc3674926310d364b9","source":"b6f0643a1534e30cb7369eca048cfaff34a7cd158047dd1c599068464a415e97","workload":{"group":7,"kind":"ellavox"}}, @@ -58,8 +60,8 @@ {"candidates":[{"features":{"max_depth":2,"packed_tokens":19380,"segment_count":15,"segments_below":[0,0,1,4,8,10,14,15]},"label":"depth_one","median_ms":1107.144287109375,"n":8,"spread_pct":4.335775965007102},{"features":{"max_depth":2,"packed_tokens":25012,"segment_count":13,"segments_below":[0,0,0,1,4,7,11,13]},"label":"minimum_effective_span_2049","median_ms":1305.4077758789062,"n":8,"spread_pct":2.64851912545848},{"features":{"max_depth":2,"packed_tokens":20916,"segment_count":14,"segments_below":[0,0,1,2,6,9,13,14]},"label":"minimum_effective_span_513","median_ms":1173.864013671875,"n":8,"spread_pct":3.677009043192731},{"features":{"max_depth":1,"packed_tokens":29108,"segment_count":12,"segments_below":[0,0,0,0,2,5,9,12]},"label":"no_sharing","median_ms":1321.0035400390625,"n":8,"spread_pct":1.2978506719070204}],"cell":"cal-hetero|Qwen/Qwen3.5-4B|L32|tp2|cp1","device":"NVIDIA H200","facts":{"cp":1.0,"gdn_layers":24.0,"layers":32.0,"tp":2.0,"uses_gdn":1.0},"hidden_size":2560,"param_dtype":"torch.bfloat16","requests_sha256":"ff3a1fcb7cc64401a9b7c23a99af2568e823fe915b381223e7fc8723b667a319","source":"797746b9f425d844419fe609c164c2e4940507f3bbb2f82b815a73740d5abc2d","workload":{"kind":"hetero","seed":7777}} ], "fit_arguments": { - "exclude_cells": ["cp4|g1","cp4|g4"], - "holdout": "", + "exclude_cells": [], + "holdout": "cp4|g4", "objective": "regret", "require_complete": 8, "terms": ["token_per_rank","token_cp_exchange","token_tp_collective","gdn_token_per_rank","attention_token_cp_exchange","tiny_segment_per_layer","level_cp_per_layer","level_tp_per_layer","gdn_level","gdn_level_tp"] @@ -67,14 +69,14 @@ "integer_table_milli_us": {"attention_token_cp_exchange":27,"gdn_level":3336555,"gdn_level_tp":1194872,"gdn_token_per_rank":594,"level_cp_per_layer":187255,"level_tp_per_layer":684798,"tiny_segment_per_layer":219288,"token_cp_exchange":39,"token_per_rank":2002,"token_tp_collective":221}, "integer_table_sha256": "3624ae7dd985d92fed47b511e058f0e792559ead8919e11585dcb9860f3394ad", "manifest": { - "excluded": ["cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g1","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g4"], + "excluded": [], "expected": ["cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp1|g0","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp1|g1","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp1|g2","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp1|g3","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp1|g4","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp1|g5","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp1|g6","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp1|g7","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp2|g0","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp2|g1","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp2|g2","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp2|g3","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp2|g4","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp2|g5","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp2|g6","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp2|g7","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g0","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g1","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g2","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g3","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g4","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g5","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g6","cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g7","cal-grpo-g16|Qwen/Qwen3.5-4B|L32|tp1|cp1","cal-grpo-g16|Qwen/Qwen3.5-4B|L32|tp1|cp2","cal-grpo-g16|Qwen/Qwen3.5-4B|L32|tp1|cp4","cal-grpo-g16|Qwen/Qwen3.5-4B|L32|tp2|cp1","cal-grpo-g4x4|Qwen/Qwen3.5-4B|L32|tp1|cp1","cal-grpo-g4x4|Qwen/Qwen3.5-4B|L32|tp1|cp2","cal-grpo-g4x4|Qwen/Qwen3.5-4B|L32|tp1|cp4","cal-grpo-g4x4|Qwen/Qwen3.5-4B|L32|tp2|cp1","cal-grpo-g8-long|Qwen/Qwen3-4B|L2|tp1|cp1","cal-grpo-g8-long|Qwen/Qwen3-4B|L2|tp1|cp4","cal-grpo-g8-long|Qwen/Qwen3.5-4B|L2|tp1|cp1","cal-grpo-g8-long|Qwen/Qwen3.5-4B|L2|tp1|cp4","cal-grpo-g8|Qwen/Qwen3-4B|L36|tp1|cp1","cal-grpo-g8|Qwen/Qwen3-4B|L36|tp1|cp2","cal-grpo-g8|Qwen/Qwen3-4B|L36|tp1|cp4","cal-grpo-g8|Qwen/Qwen3-4B|L36|tp2|cp1","cal-grpo-g8|Qwen/Qwen3.5-4B|L32|tp1|cp1","cal-grpo-g8|Qwen/Qwen3.5-4B|L32|tp1|cp2","cal-grpo-g8|Qwen/Qwen3.5-4B|L32|tp1|cp4","cal-grpo-g8|Qwen/Qwen3.5-4B|L32|tp2|cp1","cal-hetero2|Qwen/Qwen3-4B|L36|tp1|cp1","cal-hetero2|Qwen/Qwen3-4B|L36|tp1|cp4","cal-hetero2|Qwen/Qwen3.5-4B|L32|tp1|cp1","cal-hetero2|Qwen/Qwen3.5-4B|L32|tp1|cp2","cal-hetero2|Qwen/Qwen3.5-4B|L32|tp1|cp4","cal-hetero2|Qwen/Qwen3.5-4B|L32|tp2|cp1","cal-hetero3|Qwen/Qwen3.5-4B|L32|tp1|cp1","cal-hetero3|Qwen/Qwen3.5-4B|L32|tp1|cp2","cal-hetero3|Qwen/Qwen3.5-4B|L32|tp1|cp4","cal-hetero3|Qwen/Qwen3.5-4B|L32|tp2|cp1","cal-hetero|Qwen/Qwen3.5-4B|L32|tp1|cp1","cal-hetero|Qwen/Qwen3.5-4B|L32|tp1|cp2","cal-hetero|Qwen/Qwen3.5-4B|L32|tp1|cp4","cal-hetero|Qwen/Qwen3.5-4B|L32|tp2|cp1"], "path": "trainer_rank_cost_calibration_manifest.json" }, "measured_envelope": {"device_names":["NVIDIA H200"],"hidden_sizes":[2560],"param_dtypes":["torch.bfloat16"]}, "metrics": { - "integer_all": {"cells":56,"clear_misses":[],"max_regret_pct":4.215158034216835,"median_regret_pct":0.0,"ordered_pairs":3844,"p95_regret_pct":2.87166214323753,"pairwise_accuracy":0.9812695109261186}, - "test": {"cells":11,"clear_misses":[],"max_regret_pct":4.215158034216835,"median_regret_pct":0.0,"ordered_pairs":884,"p95_regret_pct":4.215158034216835,"pairwise_accuracy":0.9819004524886877}, + "integer_all": {"cells":58,"clear_misses":[],"max_regret_pct":4.215158034216835,"median_regret_pct":0.0,"ordered_pairs":3849,"p95_regret_pct":2.87166214323753,"pairwise_accuracy":0.9812938425565082}, + "test": {"cells":13,"clear_misses":[],"max_regret_pct":4.215158034216835,"median_regret_pct":0.0,"ordered_pairs":889,"p95_regret_pct":4.215158034216835,"pairwise_accuracy":0.9820022497187851}, "train": {"cells":45,"clear_misses":[],"max_regret_pct":4.028861954610052,"median_regret_pct":0.0,"ordered_pairs":2960,"p95_regret_pct":1.916476268936646,"pairwise_accuracy":0.981081081081081} }, "schema": "art.dev.trainer_rank_cost_calibration_certificate.v1" diff --git a/dev/trainer_rank_cost_calibration_manifest.json b/dev/trainer_rank_cost_calibration_manifest.json index cff79e3d0..0337ce0eb 100644 --- a/dev/trainer_rank_cost_calibration_manifest.json +++ b/dev/trainer_rank_cost_calibration_manifest.json @@ -17,10 +17,10 @@ {"campaign":"tr-cost-2gpu","cell":"cal-ellavox","cp":2,"group":6,"key":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp2|g6","layers":32,"model":"Qwen/Qwen3.5-4B","tp":1}, {"campaign":"tr-cost-2gpu","cell":"cal-ellavox","cp":2,"group":7,"key":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp2|g7","layers":32,"model":"Qwen/Qwen3.5-4B","tp":1}, {"campaign":"tr-cost-cp4-2","cell":"cal-ellavox","cp":4,"group":0,"key":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g0","layers":32,"model":"Qwen/Qwen3.5-4B","tp":1}, - {"campaign":"tr-cost-cp4-2","cell":"cal-ellavox","cp":4,"group":1,"key":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g1","layers":32,"model":"Qwen/Qwen3.5-4B","tp":1}, + {"campaign":"tr-cost-cp4-840","cell":"cal-ellavox","cp":4,"group":1,"key":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g1","layers":32,"model":"Qwen/Qwen3.5-4B","tp":1}, {"campaign":"tr-cost-cp4-2","cell":"cal-ellavox","cp":4,"group":2,"key":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g2","layers":32,"model":"Qwen/Qwen3.5-4B","tp":1}, {"campaign":"tr-cost-cp4-2","cell":"cal-ellavox","cp":4,"group":3,"key":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g3","layers":32,"model":"Qwen/Qwen3.5-4B","tp":1}, - {"campaign":"tr-cost-cp4-2","cell":"cal-ellavox","cp":4,"group":4,"key":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g4","layers":32,"model":"Qwen/Qwen3.5-4B","tp":1}, + {"campaign":"tr-cost-cp4-840","cell":"cal-ellavox","cp":4,"group":4,"key":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g4","layers":32,"model":"Qwen/Qwen3.5-4B","tp":1}, {"campaign":"tr-cost-cp4-2","cell":"cal-ellavox","cp":4,"group":5,"key":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g5","layers":32,"model":"Qwen/Qwen3.5-4B","tp":1}, {"campaign":"tr-cost-cp4-2","cell":"cal-ellavox","cp":4,"group":6,"key":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g6","layers":32,"model":"Qwen/Qwen3.5-4B","tp":1}, {"campaign":"tr-cost-cp4-2","cell":"cal-ellavox","cp":4,"group":7,"key":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g7","layers":32,"model":"Qwen/Qwen3.5-4B","tp":1}, @@ -59,10 +59,9 @@ {"campaign":"tr-cost-cp4","cell":"cal-hetero","cp":4,"group":null,"key":"cal-hetero|Qwen/Qwen3.5-4B|L32|tp1|cp4","layers":32,"model":"Qwen/Qwen3.5-4B","tp":1}, {"campaign":"tr-cost-2gpu","cell":"cal-hetero","cp":1,"group":null,"key":"cal-hetero|Qwen/Qwen3.5-4B|L32|tp2|cp1","layers":32,"model":"Qwen/Qwen3.5-4B","tp":2} ], - "description": "Expected calibration cells for the coefficient-version-2 campaign (2026-09-02): the exact union of the cells the checked-in recipes launch (dev/trainer_rank_cost_calibration_local.sh; the cp4 and 2gpu sky recipes in both CELL_SET modes), keyed exactly as the fitter keys cells. tests/unit/test_planner_cost_manifest_recipes.py asserts recipes and manifest agree. The fitter's --manifest validation requires every non-excluded cell to be present with a complete mandatory candidate set and rejects unexpected cells; exclusions must be listed here with a reason.", + "description": "Expected calibration cells for the coefficient-version-2 campaign (2026-09-02; Ellavox CP4 groups 1 and 4 re-measured 2026-09-03 after the harness desynchronization fix of issue #840): the exact union of the cells the checked-in recipes launch (dev/trainer_rank_cost_calibration_local.sh; the cp4 and 2gpu sky recipes in both CELL_SET modes), keyed exactly as the fitter keys cells. tests/unit/test_planner_cost_manifest_recipes.py asserts recipes and manifest agree. The fitter's --manifest validation requires every non-excluded cell to be present with a complete mandatory candidate set and rejects unexpected cells; exclusions must be listed here with a reason.", "excluded": [ - {"key":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g1","reason":"deterministic CP4 NCCL all-to-all hang in the context-parallel group; reproduced on a fresh cluster (issue #840)"}, - {"key":"cal-ellavox|Qwen/Qwen3.5-4B|L32|tp1|cp4|g4","reason":"deterministic CP4 NCCL all-to-all hang in the context-parallel group; reproduced on a fresh cluster (issue #840)"} + ], "schema": "art.dev.trainer_rank_cost_calibration_manifest.v1" } diff --git a/dev/trainer_rank_landing_acceptance_README.md b/dev/trainer_rank_landing_acceptance_README.md index 38e88d70d..a10e4cd5b 100644 --- a/dev/trainer_rank_landing_acceptance_README.md +++ b/dev/trainer_rank_landing_acceptance_README.md @@ -20,9 +20,9 @@ every gate below now passes on the landed implementation. | `tests/unit/test_trainer_rank_topology.py` | CPU | TP>1 runtimes construct; PP>1 and multi-chunk runtimes still refuse | pass | | `--phase tp2-public --tp 2` (`dev/trainer_rank_landing_acceptance_tp2.sky.yaml`) + `--tp 1` control (1x H200) + `--phase tp-compare` (CPU) | 2x H200 k8s | Qwen3.5-4B full model, DP1×TP2×CP1, public `dp_rank_forward`, active LoRA: TP peers plan identical physical layouts; automatic shares deeper than depth-one (group lengths 9,199 vs 37,871 for 53,216 logical tokens; physical 9,200 / 37,872 after per-group TP padding); odd group lengths exercise SP padding; measured rows compile-free and plan-cache-stable; numerics gated relative to the TP1 control's cross-layout divergence with source/workload fingerprints (same-layout TP2-vs-TP1 ratios 1.06/1.05, cross-layout ratios 1.05/1.06, losses within 0.06%, unstructured differences). Also runs the CI check script at TP=2 (0.0 divergence) | pass: automatic 720 ms vs depth-one 1,921 ms (62.5% paired gain at TP2; 71.9% at TP1) | | `--phase dp2-tp2-waves` (`dev/trainer_rank_landing_acceptance_dp2_tp2.sky.yaml`) | 4x H200 k8s | DP2×TP2 public `forward_micro_batches`: ≥2 waves, distinct DP payloads, identical wave shapes within each TP pair, every input returned once in order, forward+backward per wave, automatic vs depth-one parity, empty-DP-slot arm, no hang | see evidence log | -| `--phase cost-calibrate` (`dev/trainer_rank_cost_calibration_{cp4,2gpu}.sky.yaml`, `dev/trainer_rank_cost_calibration_local.sh`) | 1x/2x/4x H200 | every mandatory candidate layout of a cell timed through the public API (forward+backward, active LoRA, compile-free, max-rank) plus the production selection; layout features and topology/model facts to JSONL. Cells: GRPO g8/g16/g4x4 (Qwen3.5-4B GDN and Qwen3-4B attention, 2-layer and full height), three heterogeneous controls, Ellavox groups, at TP1/TP2 × CP1/CP2/CP4 | 56 cells, 3,844 within-cell pairs | -| `dev/trainer_rank_cost_fit.py` | CPU | paired within-cell deltas, non-negative least squares over the production term functions, regret-minimizing refinement; gates on whole held-out cells: pairwise ordering ≥90% on pairs separated >3%, median regret ≤2%, p95 ≤5%, none >10%, clear winners selected within 5%; `--selector-check` runs the shipped table through the real selector | final table (fit on 45 cells, evaluated on all 56; 3,844 pairs): 98.1% pairwise, p95 regret 2.9%, max 4.2%, no clear misses; pre-registered odd-Ellavox holdout passes; profile narrowed to the measured envelope (H200-class SM 9.0, bf16, hidden 2,560, dense) and bound to the certificate by test; expected-cell manifest validated; TP2, CP2 and attention-model ablations pass (all-heterogeneous ablation: one CP4 cell at 9.5%); campaign-1 table on the 18 later cells within 4.2%; timed production selection on the later CP2/TP2 cells: median −0.2%, max 0.4%; hand-set score: 78.6% pairwise, max regret 67% | -| `dev/trainer_rank_cost_calibration_certificate.json` (compact: one line per cell) + `tests/unit/test_planner_cost_certificate.py` | CPU | the shipped table is the certified table (hash) and the certified metrics hold on the recorded per-cell aggregates; exact 56 retained cell identities plus the two exclusions match the manifest; the profile envelope matches the certified evidence; runners fail loudly on any cell failure and the fitter refuses incomplete evidence unless exclusions are explicit (two Ellavox CP4 cells excluded: deterministic CP4 NCCL hang, issue #840) | pass | +| `--phase cost-calibrate` (`dev/trainer_rank_cost_calibration_{cp4,2gpu}.sky.yaml`, `dev/trainer_rank_cost_calibration_local.sh`) | 1x/2x/4x H200 | every mandatory candidate layout of a cell timed through the public API (forward+backward, active LoRA, compile-free, max-rank) plus the production selection; layout features and topology/model facts to JSONL. Cells: GRPO g8/g16/g4x4 (Qwen3.5-4B GDN and Qwen3-4B attention, 2-layer and full height), three heterogeneous controls, Ellavox groups, at TP1/TP2 × CP1/CP2/CP4 | 58 cells, 3,849 within-cell pairs | +| `dev/trainer_rank_cost_fit.py` | CPU | paired within-cell deltas, non-negative least squares over the production term functions, regret-minimizing refinement; gates on whole held-out cells: pairwise ordering ≥90% on pairs separated >3%, median regret ≤2%, p95 ≤5%, none >10%, clear winners selected within 5%; `--selector-check` runs the shipped table through the real selector | final table (fit on 45 cells, evaluated on all 58; 3,849 pairs; 13 held-out cells): 98.1% pairwise, p95 regret 2.9%, max 4.2%, no clear misses; pre-registered odd-Ellavox holdout passes; the two Ellavox CP4 cells re-measured after the issue #840 harness fix are held out too (shipped-table regret 0% and 2.8%); profile narrowed to the measured envelope (H200-class SM 9.0, bf16, hidden 2,560, dense) and bound to the certificate by test; expected-cell manifest validated; TP2, CP2 and attention-model ablations pass (all-heterogeneous ablation: one CP4 cell at 9.5%); campaign-1 table on the 18 later cells within 4.2%; timed production selection on the later CP2/TP2 cells: median −0.2%, max 0.4%; hand-set score: 78.6% pairwise, max regret 67% | +| `dev/trainer_rank_cost_calibration_certificate.json` (compact: one line per cell) + `tests/unit/test_planner_cost_certificate.py` | CPU | the shipped table is the certified table (hash) and the certified metrics hold on the recorded per-cell aggregates; exact 58 cell identities match the manifest with no exclusions; the profile envelope matches the certified evidence; runners fail loudly on any cell failure and the fitter refuses incomplete evidence unless exclusions are explicit (the two Ellavox CP4 cells that hung in the campaign were a harness desynchronization, issue #840, fixed and re-measured) | pass | | `tests/unit/test_planner_cost_profile.py` | CPU | the fitted table applies only inside the calibrated capability profile, narrowed to the measured envelope (SM 9.0 with H200-class device memory, bf16, hidden 2,560, non-MoE) and bound to the certificate by test; outside it the version-1 score is used verbatim | pass | | `--phase split-conversion --pressure ballast` | 1x H200 | same cell under real pressure (live ballast tensors, no test hooks). Training forward: ballast sized from the measured retained fraction f (budget midway inside the (1−f)·R/2 conversion window, width reported), unsplit refused before execution, split runs under the reduced headroom with parity, combined backward with ballast live, observed forward+backward peak within both the admitted budget and the predicted peak; deeper ballast refuses before execution. `no_grad` forward (retained ≈ outputs only; the demonstrated high-value case): budget 60% of the unsplit requirement converts with parity and the observed peak within budget and prediction | see evidence log | diff --git a/dev/trainer_rank_planner_design.md b/dev/trainer_rank_planner_design.md index 7f5aa244a..f5fffaae6 100644 --- a/dev/trainer_rank_planner_design.md +++ b/dev/trainer_rank_planner_design.md @@ -134,16 +134,18 @@ What the data showed: Gates (held-out cells, noise-qualified): pairwise ordering ≥ 90% on pairs separated by more than 3%, median regret ≤ 2%, p95 ≤ 5%, none above 10%, clear winners selected within 5%. The table is fitted on 45 cells and -evaluated on all 56 (3,844 within-cell pairs; the 11 odd Ellavox groups are -the pre-registered holdout): it ranks 98.1% of separated pairs correctly, p95 -regret 2.9%, max 4.2%, no clear misses; the holdout passes. Ablations withholding +evaluated on all 58 (3,849 within-cell pairs; the 11 odd Ellavox groups are +the pre-registered holdout, and the two Ellavox CP4 cells re-measured after +issue #840 are held out as well, 13 held-out cells): it ranks 98.1% of +separated pairs correctly, p95 regret 2.9%, max 4.2%, no clear misses; the +holdout passes. Ablations withholding every TP2 cell, every CP2 cell, or the whole attention model pass; withholding every heterogeneous cell misranks one CP4 heterogeneous cell by 9.5%, and withholding every CP4 cell does not extrapolate (25%), so those cells stay in the fit. Robustness: the table fitted on the first 38 cells, run through the real selector on the 18 later cells, was already within 4.2% everywhere, and the production selection timed in the later CP2/TP2 cells had median regret -−0.2%, max 0.4%. The hand-set version-1 score on the same 56 cells: 78.6% +−0.2%, max 0.4%. The hand-set version-1 score on the original 56 cells: 78.6% pairwise, max regret 67% (an Ellavox group at CP2). Calibrated domain: the table applies only inside `CalibrationProfile`, which @@ -155,7 +157,7 @@ one-time warning. `dev/trainer_rank_cost_calibration_manifest.json` lists the exact cells each recipe launches; the fitter's `--manifest` validation requires every non-excluded cell to be present and complete, rejects unexpected cells and duplicate cells with differing execution fingerprints, and the certificate -test asserts the 56 retained identities plus the two explicit exclusions. +test asserts all 58 identities (no exclusions). Landing gates re-derived: the sealed win-cell shape still selects deep sharing (prompt-level sharing at CP4, where it measures fastest; full sharing at CP1), @@ -182,11 +184,17 @@ metrics hold on the recorded aggregates; `--from-certificate` re-fits from it. The runners propagate every cell failure (no masked exit codes) and the fitter refuses to fit incomplete evidence unless the gaps are excluded explicitly (`--require-complete`, `--exclude-cells`). Two Ellavox CP4 cells -(groups 1 and 4) are excluded this way: their CP4 executions hang -deterministically in NCCL all-to-alls in the context-parallel group -(reproduced on a fresh cluster at identical collective sequence numbers) — a -pre-existing CP4 execution bug, tracked as issue #840; the version-1 score -selects the hanging layout for group 4, the fitted table happens not to. +(groups 1 and 4) were excluded this way at first because their calibration +runs deadlocked in NCCL all-to-alls in the context-parallel group (issue +#840). Tracing every collective per rank (`dev/trainer_rank_collective_trace.py`, +`dev/trainer_rank_collective_diff.py`) showed the harness, not the runtime, +at fault: the warm-up loop stopped when the *local* rank's forward was +compile-free, and one CP rank whose local shapes still recompiled ran an +extra warm-up of the previous layout while its peers moved on, so the ranks +exchanged different layouts. Warm-up completion is now decided from the +gathered world-wide compile statuses; both cells were re-measured cleanly +and folded into the certificate as held-out cells (shipped-table regret 0% +and 2.8%; the training cells and therefore the table are unchanged). ## Width feasibility is decided by the memory-minimal layout diff --git a/tests/unit/test_planner_cost_certificate.py b/tests/unit/test_planner_cost_certificate.py index c9ea6aca5..3add561e5 100644 --- a/tests/unit/test_planner_cost_certificate.py +++ b/tests/unit/test_planner_cost_certificate.py @@ -87,7 +87,8 @@ def held_out(cell: str) -> bool: def test_certificate_holds_exactly_the_manifest_cells() -> None: - """Exact cell identities: every expected cell minus the explicit exclusions.""" + """Exact cell identities: every expected cell minus the explicit exclusions + (none since the two Ellavox CP4 cells were re-measured after issue #840).""" payload = json.loads(_CERTIFICATE.read_text()) manifest = json.loads(_MANIFEST.read_text()) @@ -96,7 +97,7 @@ def test_certificate_holds_exactly_the_manifest_cells() -> None: assert excluded <= expected certified = {cell["cell"] for cell in payload["cells"]} assert certified == expected - excluded - assert len(certified) == 56 and len(excluded) == 2 + assert len(certified) == 58 and not excluded assert set(payload["manifest"]["excluded"]) == excluded assert payload["manifest"]["path"] == _MANIFEST.name # Every retained cell carries its execution fingerprints. diff --git a/tests/unit/test_planner_cost_manifest_recipes.py b/tests/unit/test_planner_cost_manifest_recipes.py index 706883e4d..6d5f70962 100644 --- a/tests/unit/test_planner_cost_manifest_recipes.py +++ b/tests/unit/test_planner_cost_manifest_recipes.py @@ -83,7 +83,7 @@ def test_recipes_launch_exactly_the_manifest_cells() -> None: } excluded = {cell["key"] for cell in manifest["excluded"]} assert excluded <= expected - assert len(expected - excluded) == 56 + assert len(expected - excluded) == 58 def test_manifest_campaign_labels_name_checked_in_recipes() -> None: @@ -94,4 +94,5 @@ def test_manifest_campaign_labels_name_checked_in_recipes() -> None: "tr-cost-cp4-2", "tr-cost-2gpu", "tr-cost-2gpu-2", + "tr-cost-cp4-840", }