From ad18184680026c8542261bd7441da9f7af9e12be Mon Sep 17 00:00:00 2001 From: Ye Yu Date: Tue, 25 Aug 2026 11:01:33 -0700 Subject: [PATCH 1/3] ar_validate: fail loudly when every sample fails validate_ar() swallows per-sample exceptions into WARNING lines and returns whatever succeeded. When that list came back empty, the reporting block was guarded by `if results and ...`, so the script printed nothing and exited 0 -- a run where 100% of samples failed was indistinguishable from a successful one. Observed on a Cosmos3-Nano EAGLE3 checkpoint sharded by device_map="auto": all 80/80 samples died with "Expected all tensors to be on the same device", the job still exited 0, and the wrapper stamped PASS with no AL number anywhere. Raise instead, so the caller sees a non-zero exit. Signed-off-by: Ye Yu --- examples/speculative_decoding/scripts/ar_validate.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/speculative_decoding/scripts/ar_validate.py b/examples/speculative_decoding/scripts/ar_validate.py index 5699c480b7b..709dd6da786 100644 --- a/examples/speculative_decoding/scripts/ar_validate.py +++ b/examples/speculative_decoding/scripts/ar_validate.py @@ -123,7 +123,14 @@ def main(): accelerator.device, ) - if results and accelerator.is_main_process: + if not results: + raise RuntimeError( + f"AR validation produced no results: all {args.num_samples} samples failed. " + "See the per-sample WARNING lines above for the underlying error. " + "Exiting non-zero so this is not mistaken for a successful validation." + ) + + if accelerator.is_main_process: all_ars = [ar for _, ar in results] avg_ar = sum(all_ars) / len(all_ars) print(f"\n==== AR Validation Results (osl={args.osl}, steps={args.steps}) ====") From db802b123f46c1e63354fefc37a96816f9689120 Mon Sep 17 00:00:00 2001 From: Ye Yu Date: Mon, 31 Aug 2026 12:16:09 -0700 Subject: [PATCH 2/3] address review: report the attempted sample count, not the requested one validate_ar() clamps to min(num_samples, len(ds)), so the failure message interpolated a count that could exceed what actually ran when --num_samples is larger than the dataset. Report the clamped value. Signed-off-by: Ye Yu --- examples/speculative_decoding/scripts/ar_validate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/speculative_decoding/scripts/ar_validate.py b/examples/speculative_decoding/scripts/ar_validate.py index 709dd6da786..f2e7a7ee37b 100644 --- a/examples/speculative_decoding/scripts/ar_validate.py +++ b/examples/speculative_decoding/scripts/ar_validate.py @@ -124,8 +124,11 @@ def main(): ) if not results: + # validate_ar() clamps to len(ds), so report what was actually attempted rather than + # the requested --num_samples, which can be larger than the dataset. + attempted = min(args.num_samples, len(ds)) raise RuntimeError( - f"AR validation produced no results: all {args.num_samples} samples failed. " + f"AR validation produced no results: all {attempted} samples failed. " "See the per-sample WARNING lines above for the underlying error. " "Exiting non-zero so this is not mistaken for a successful validation." ) From 638ad6a055a4248489c25fc0d7b11644e4683ae9 Mon Sep 17 00:00:00 2001 From: Ye Yu Date: Mon, 31 Aug 2026 12:24:57 -0700 Subject: [PATCH 3/3] address review: distinguish zero samples from all-samples-failed validate_ar() accepts a non-positive num_samples and returns [] without any sample having failed, so the new guard would have reported 'all 0 samples failed' (or a negative count). Check the effective count first and raise a distinct, accurate error for that case. Signed-off-by: Ye Yu --- .../speculative_decoding/scripts/ar_validate.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/speculative_decoding/scripts/ar_validate.py b/examples/speculative_decoding/scripts/ar_validate.py index f2e7a7ee37b..d8c28af48d7 100644 --- a/examples/speculative_decoding/scripts/ar_validate.py +++ b/examples/speculative_decoding/scripts/ar_validate.py @@ -123,10 +123,17 @@ def main(): accelerator.device, ) + # validate_ar() clamps to len(ds), so report what was actually attempted rather than the + # requested --num_samples, which can be larger than the dataset. A non-positive count means + # nothing ran at all -- distinct from "everything ran and failed", so say so separately. + attempted = min(args.num_samples, len(ds)) + if attempted <= 0: + raise ValueError( + f"No samples to validate: --num_samples={args.num_samples} with a dataset of " + f"{len(ds)} prompts. Pass a positive --num_samples." + ) + if not results: - # validate_ar() clamps to len(ds), so report what was actually attempted rather than - # the requested --num_samples, which can be larger than the dataset. - attempted = min(args.num_samples, len(ds)) raise RuntimeError( f"AR validation produced no results: all {attempted} samples failed. " "See the per-sample WARNING lines above for the underlying error. "