From 843aff86f6a1ea8b3aa2c6a381c56bd0448f4eac Mon Sep 17 00:00:00 2001 From: Jenny Liu Date: Thu, 17 Sep 2026 07:32:59 +0000 Subject: [PATCH 1/2] add nemotron_3.5_lightning_30b_nvfp4 and nemotron_3.5_lightning_30b_bf16 func and perf case Signed-off-by: Jenny Liu --- .../defs/accuracy/test_llm_api_pytorch.py | 32 ++- .../test_configs/Nemotron35_Lightning_30B.yml | 21 ++ .../defs/examples/serve/test_serve.py | 230 +++++++++++++----- tests/integration/defs/perf/_model_paths.py | 2 + .../defs/perf/pytorch_model_config.py | 48 ++++ tests/integration/defs/perf/test_perf.py | 9 + tests/integration/defs/test_e2e.py | 1 - .../test_lists/qa/llm_spark_func.yml | 3 + .../test_lists/qa/llm_spark_perf.yml | 2 + 9 files changed, 272 insertions(+), 76 deletions(-) create mode 100644 tests/integration/defs/examples/serve/test_configs/Nemotron35_Lightning_30B.yml diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index c2ce79a4e0a5..95769dc8bdeb 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -7157,20 +7157,22 @@ class TestNemotron35Lightning(LlmapiAccuracyTestHarness): EXTRA_EVALUATOR_KWARGS = dict(chat_template_kwargs=dict( enable_thinking=False)) - @skip_no_hopper - def test_nvfp4_marlin_mtp3_chunked_prefill(self): - """Single-GPU Hopper guard for the Marlin NVFP4 path. + def _run_mtp3_chunked_prefill(self, moe_backend, nvfp4_gemm_config=None): + """Evaluate the MTP=3 + chunked-prefill combination on one MoE backend. The checkpoint is MIXED_PRECISION: routed experts, shared experts and lm_head are W4A16_NVFP4, the Mamba projections are FP8 and the MTP - layers are left unquantized. ``moe_config.backend=MARLIN`` plus - ``nvfp4_gemm_config.allowed_backends=['marlin']`` pin both the MoE and - the dense NVFP4 GEMMs to Marlin, which is Ada/Hopper only. Chunked - prefill, CUDA graphs and the overlap scheduler are enabled together so - the combination with MTP drafting is covered end to end. + layers are left unquantized. Chunked prefill, CUDA graphs and the + overlap scheduler are enabled together so the combination with MTP + drafting is covered end to end. Only the backend pinning differs + between callers; everything else is held fixed so the two runs are + comparable against the same accuracy references. """ max_batch_size = 32 mtp_config = MTPDecodingConfig(max_draft_len=3) + extra_args = {} + if nvfp4_gemm_config is not None: + extra_args["nvfp4_gemm_config"] = nvfp4_gemm_config with LLM( self.MODEL_PATH, kv_cache_config=KvCacheConfig( @@ -7186,9 +7188,9 @@ def test_nvfp4_marlin_mtp3_chunked_prefill(self): cuda_graph_config=CudaGraphConfig(max_batch_size=max_batch_size, enable_padding=True), disable_overlap_scheduler=False, - moe_config=MoeConfig(backend="MARLIN"), - nvfp4_gemm_config={"allowed_backends": ["marlin"]}, + moe_config=MoeConfig(backend=moe_backend), speculative_config=mtp_config, + **extra_args, ) as llm: assert llm.args.quant_config.quant_algo == QuantAlgo.MIXED_PRECISION task = MMLU(self.MODEL_NAME) @@ -7198,6 +7200,16 @@ def test_nvfp4_marlin_mtp3_chunked_prefill(self): task.evaluate(llm, extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) + @skip_no_hopper + def test_nvfp4_marlin_mtp3_chunked_prefill(self): + self._run_mtp3_chunked_prefill( + moe_backend="MARLIN", + nvfp4_gemm_config={"allowed_backends": ["marlin"]}) + + @skip_pre_blackwell + def test_nvfp4_cutedsl_mtp3_chunked_prefill(self): + self._run_mtp3_chunked_prefill(moe_backend="CUTEDSL") + @skip_pre_blackwell class TestMiniMaxM3(LlmapiAccuracyTestHarness): diff --git a/tests/integration/defs/examples/serve/test_configs/Nemotron35_Lightning_30B.yml b/tests/integration/defs/examples/serve/test_configs/Nemotron35_Lightning_30B.yml new file mode 100644 index 000000000000..abbbaddcfdfa --- /dev/null +++ b/tests/integration/defs/examples/serve/test_configs/Nemotron35_Lightning_30B.yml @@ -0,0 +1,21 @@ +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + mamba_state_config: + periodic_snapshot_interval: 8192 + free_gpu_memory_fraction: 0.8 + mamba_ssm_cache_dtype: float16 + mamba_ssm_stochastic_rounding: true + mamba_ssm_philox_rounds: 5 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +speculative_config: + decoding_type: MTP + max_draft_len: 3 + allow_advanced_sampling: true +enable_chunked_prefill: true +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 10 +disable_overlap_scheduler: false diff --git a/tests/integration/defs/examples/serve/test_serve.py b/tests/integration/defs/examples/serve/test_serve.py index 7c046dff06b8..ea377816f31c 100644 --- a/tests/integration/defs/examples/serve/test_serve.py +++ b/tests/integration/defs/examples/serve/test_serve.py @@ -136,6 +136,81 @@ def check_openai_chat_completion(http_port, raise +_SHORT_PROMPT = "What is the capital of France?" +_LONG_PROMPT = ("Please summarize the following passage in one sentence.\n\n" + + ("The quick brown fox jumps over the lazy dog. " * 600)) + + +def check_mixed_prompt_batch(client, + model_name, + join_timeout=300, + require_reasoning=True): + """Send 1 long + 3 short chat requests at once and check every reply. + + Args: + require_reasoning: Require non-empty ``reasoning_content``; set False + where an empty thinking step is acceptable. + """ + results = {} + errors = {} + + def _complete(label, prompt): + try: + resp = client.chat.completions.create( + model=model_name, + messages=[{ + "role": "user", + "content": prompt + }], + max_completion_tokens=1024, + stream=False, + ) + assert len(resp.choices) == 1, \ + f"[{label}] expected 1 choice, got {len(resp.choices)}" + results[label] = resp.choices[0].message + except Exception as exc: + errors[label] = str(exc) + + threads = [ + threading.Thread(name="long_0", + target=_complete, + args=("long_0", _LONG_PROMPT), + daemon=True) + ] + for i in range(3): + threads.append( + threading.Thread(name=f"short_{i}", + target=_complete, + args=(f"short_{i}", _SHORT_PROMPT), + daemon=True)) + + for t in threads: + t.start() + for t in threads: + t.join(timeout=join_timeout) + + hung = [t.name for t in threads if t.is_alive()] + assert not hung, f"Timed out waiting for request threads: {hung}" + assert not errors, f"Requests failed: {errors}" + assert "long_0" in results, "Long prompt (chunked prefill) got no response" + assert all(f"short_{i}" in results + for i in range(3)), "One or more short prompts got no response" + + for label, msg in results.items(): + reasoning = msg.reasoning_content or "" + content = msg.content or "" + if require_reasoning: + # content may be empty if the token budget was consumed by + # thinking, which is expected model behaviour, not a server error. + assert len(reasoning) > 0, \ + f"[{label}] empty reasoning_content — request did not complete" + else: + assert len(reasoning) + len(content) > 0, \ + f"[{label}] empty response — request did not complete" + print_info(f"[{label}] reasoning: {reasoning!r}") + print_info(f"[{label}] content: {content!r}") + + @pytest.mark.parametrize("config_flag", ["--extra_llm_api_options", "--config"]) @skip_no_hopper def test_config_file_loading(serve_test_root, config_flag): @@ -298,77 +373,102 @@ def test_nemotron3_super_120b_nvfp4(serve_test_root): api_key="tensorrt_llm", ) - # Short prompt: exercises MTP decode path. - short_prompt = "What is the capital of France?" + with popen(cmd, env=env) as proc: + _wait_for_server_ready(proc, http_port=port, timeout=7200) + print_info("Server ready — sending mixed short+long prompt batch...") + check_mixed_prompt_batch(client, model_name) + + print_info("test_nemotron3_super_120b_nvfp4 PASSED") + + +_NEMOTRON35_LIGHTNING_COMMON_CONFIG = "Nemotron35_Lightning_30B.yml" +_NEMOTRON35_LIGHTNING_VARIANTS = { + "nvfp4": { + "model_dir": "NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", + "config_overrides": { + "moe_config": { + "backend": "CUTEDSL" + }, + "nvfp4_gemm_config": { + "allowed_backends": + ["marlin", "cutlass", "cublaslt", "cuda_core"] + }, + }, + }, + "bf16": { + "model_dir": "NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", + "config_overrides": { + "moe_config": { + "backend": "CUTLASS" + }, + }, + }, +} + + +@skip_pre_blackwell +@pytest.mark.parametrize("precision", list(_NEMOTRON35_LIGHTNING_VARIANTS)) +def test_nemotron35_lightning_30b(serve_test_root, tmp_path, precision): + """Nemotron 3.5 Lightning 30B-A3B with chunked prefill + MTP=3.""" + variant = _NEMOTRON35_LIGHTNING_VARIANTS[precision] + model_path = f"{llm_models_root()}/{variant['model_dir']}" + common_config = (f"{serve_test_root}/test_configs/" + f"{_NEMOTRON35_LIGHTNING_COMMON_CONFIG}") + + assert os.path.exists(model_path), f"Model not found: {model_path}" + assert os.path.exists(common_config), f"Config not found: {common_config}" + + with open(common_config) as f: + config = yaml.safe_load(f) + overlapping = set(config) & set(variant["config_overrides"]) + assert not overlapping, \ + f"[{precision}] overrides would replace common settings: {overlapping}" + config.update(variant["config_overrides"]) + + config_file = str(tmp_path / f"nemotron35_lightning_{precision}.yml") + with open(config_file, "w") as f: + yaml.dump(config, f) + print_info(f"[{precision}] merged serve config:\n{yaml.dump(config)}") - # Long prompt: 3000+ words to exceed max_num_tokens (8192 tokens) and - # force chunked prefill. The content is repeated to guarantee length. - long_prompt = ( - "Please summarize the following passage in one sentence.\n\n" + - ("The quick brown fox jumps over the lazy dog. " * 600)) + port = get_free_port_in_ci() + env = os.environ.copy() + env["TLLM_ALLOW_LONG_MAX_MODEL_LEN"] = "1" + + cmd = [ + "trtllm-serve", + model_path, + "--host", + "0.0.0.0", + "--port", + str(port), + "--max_batch_size", + "1", + "--max_num_tokens", + "8192", + "--trust_remote_code", + "--reasoning_parser", + "nemotron-v3", + "--tool_parser", + "qwen3_coder", + "--config", + config_file, + ] + + model_name = os.path.basename(model_path) + client = OpenAI( + base_url=f"http://localhost:{port}/v1", + api_key="tensorrt_llm", + ) with popen(cmd, env=env) as proc: _wait_for_server_ready(proc, http_port=port, timeout=7200) print_info("Server ready — sending mixed short+long prompt batch...") + check_mixed_prompt_batch(client, + model_name, + join_timeout=600, + require_reasoning=False) - # Send all requests in parallel threads so the server batches them. - results = {} - errors = {} - - def _complete(label, prompt): - try: - resp = client.chat.completions.create( - model=model_name, - messages=[{ - "role": "user", - "content": prompt - }], - max_completion_tokens=1024, - stream=False, - ) - assert len(resp.choices) == 1, \ - f"[{label}] expected 1 choice, got {len(resp.choices)}" - results[label] = resp.choices[0].message - except Exception as exc: - errors[label] = str(exc) - - threads = [] - # 1 long-prompt request + 3 short-prompt requests sent simultaneously. - threads.append( - threading.Thread(name="long_0", - target=_complete, - args=("long_0", long_prompt), - daemon=True)) - for i in range(3): - threads.append( - threading.Thread(name=f"short_{i}", - target=_complete, - args=(f"short_{i}", short_prompt), - daemon=True)) - - for t in threads: - t.start() - for t in threads: - t.join(timeout=300) - - hung = [t.name for t in threads if t.is_alive()] - assert not hung, f"Timed out waiting for request threads: {hung}" - assert not errors, f"Requests failed: {errors}" - assert "long_0" in results, "Long prompt (chunked prefill) got no response" - assert all( - f"short_{i}" in results - for i in range(3)), "One or more short prompts got no response" - - for label, msg in results.items(): - # A valid reasoning-model response must have reasoning_content. - # content may be empty if the token budget was consumed by thinking, - # which is expected model behaviour, not a server error. - assert len(msg.reasoning_content) > 0, \ - f"[{label}] empty reasoning_content — request did not complete" - print_info(f"[{label}] reasoning: {msg.reasoning_content!r}") - print_info(f"[{label}] content: {msg.content!r}") - - print_info("test_nemotron3_super_120b_nvfp4 PASSED") + print_info(f"test_nemotron35_lightning_30b[{precision}] PASSED") _NEMOTRON3_NANO_OMNI_MODEL_DIR = "NVIDIA-Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4" diff --git a/tests/integration/defs/perf/_model_paths.py b/tests/integration/defs/perf/_model_paths.py index 6946797cfaf2..12a392065cff 100644 --- a/tests/integration/defs/perf/_model_paths.py +++ b/tests/integration/defs/perf/_model_paths.py @@ -52,6 +52,8 @@ # Nemotron-3-Nano-Omni-30B (text + image multimodal) "nemotron_3_nano_omni_nvfp4": "NVIDIA-Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", "nemotron_3_nano_omni_nvfp4_image": "NVIDIA-Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4", + "nemotron_3.5_lightning_30b_nvfp4_mtp": "NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", + "nemotron_3.5_lightning_30b_bf16_mtp": "NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", # MiniMax M3 (block-sparse MoE, MXFP8 weights, BF16 activations + KV cache) "minimax_m3_mxfp8": "MiniMax-M3-MXFP8", # Qwen3.5 dense + MoE diff --git a/tests/integration/defs/perf/pytorch_model_config.py b/tests/integration/defs/perf/pytorch_model_config.py index 5e05e983517a..20bb10cfd5cc 100644 --- a/tests/integration/defs/perf/pytorch_model_config.py +++ b/tests/integration/defs/perf/pytorch_model_config.py @@ -673,6 +673,54 @@ def get_model_yaml_config(model_label: str, }, } }, + # Nemotron-3.5-Lightning-30B with MTP=3, NVFP4 and BF16. + { + 'patterns': [ + 'nemotron_3.5_lightning_30b_nvfp4_mtp-serve-pytorch-streaming-', + 'nemotron_3.5_lightning_30b_bf16_mtp-serve-pytorch-streaming-', + ], + 'config': { + 'enable_chunked_prefill': True, + 'stream_interval': 10, + 'num_postprocess_workers': 4, + 'cuda_graph_config': { + 'enable_padding': True, + 'max_batch_size': 16, + }, + 'kv_cache_config': { + 'enable_block_reuse': False, + 'free_gpu_memory_fraction': 0.8, + 'mamba_ssm_cache_dtype': 'float16', + 'mamba_ssm_stochastic_rounding': True, + 'mamba_ssm_philox_rounds': 5, + 'mamba_state_config': { + 'periodic_snapshot_interval': 8192, + }, + }, + 'speculative_config': { + 'decoding_type': 'MTP', + 'max_draft_len': 3, + }, + } + }, + { + 'patterns': + ['nemotron_3.5_lightning_30b_nvfp4_mtp-serve-pytorch-streaming-'], + 'config': { + 'moe_config': { + 'backend': 'CUTEDSL', + }, + } + }, + { + 'patterns': + ['nemotron_3.5_lightning_30b_bf16_mtp-serve-pytorch-streaming-'], + 'config': { + 'moe_config': { + 'backend': 'CUTLASS', + }, + } + }, # Nemotron-3-Super-120B-NVFP4 (streaming/low-latency variant for spark perf) # Streaming serve cases use small cuda_graph batch and no attention DP for latency. { diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index 9fd4708e3928..b2aaa327e3a6 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -49,12 +49,17 @@ TIMING_CACHE_DIR = os.environ.get("TIMING_CACHE_DIR", "") +# Models served with the Nemotron reasoning/tool parsers, a long max model len +# and a longer server-start timeout. "nano-v3" and "nemotron-v3" are two +# registered names for the same reasoning parser, so Lightning belongs here too. NEMOTRON_SUPER_MODELS = { "nemotron_3_super_120b_nvfp4", "nemotron_3_super_120b_nvfp4_mtp", "nemotron_3_ultra_550b_nvfp4", "nemotron_3_nano_omni_nvfp4", "nemotron_3_nano_omni_nvfp4_image", + "nemotron_3.5_lightning_30b_nvfp4_mtp", + "nemotron_3.5_lightning_30b_bf16_mtp", } KIMI_K3_MODELS = {"kimi_k3"} @@ -83,6 +88,8 @@ "qwen3.6_35b_a3b_fp4_mtp", "nemotron_3_nano_omni_nvfp4", "nemotron_3_nano_omni_nvfp4_image", + "nemotron_3.5_lightning_30b_nvfp4_mtp", + "nemotron_3.5_lightning_30b_bf16_mtp", "nemotron_nano_12b_v2", } @@ -104,6 +111,8 @@ # Spec-dec models real dataset in serve perf tests. SPEC_DEC_REAL_DATASET_MODELS = { "nemotron_3_super_120b_nvfp4_mtp": "cnn_dailymail", + "nemotron_3.5_lightning_30b_nvfp4_mtp": "cnn_dailymail", + "nemotron_3.5_lightning_30b_bf16_mtp": "cnn_dailymail", } # All spec-decoding models (MTP, Eagle3, etc.). Used to skip --ignore-eos in diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index de9e62ad6e46..be0cc40dcafd 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -952,7 +952,6 @@ def test_ptp_quickstart_advanced(llm_root, llm_venv, model_name, model_path): "--disable_kv_cache_reuse", "--trust_remote_code", "--max_batch_size=8", - "--use_kv_cache_manager_v2=false", "--model_dir", f"{llm_models_root()}/{model_path}", ]) diff --git a/tests/integration/test_lists/qa/llm_spark_func.yml b/tests/integration/test_lists/qa/llm_spark_func.yml index 739455999a46..acecff77804d 100644 --- a/tests/integration/test_lists/qa/llm_spark_func.yml +++ b/tests/integration/test_lists/qa/llm_spark_func.yml @@ -35,6 +35,9 @@ llm_spark_func: - test_e2e.py::test_trtllm_benchmark_serving[gpt_oss/gpt-oss-20b] - test_e2e.py::test_openai_chat_guided_decoding[meta-llama/Llama-3.1-8B-Instruct] - examples/serve/test_serve.py::test_nemotron3_super_120b_nvfp4 + - accuracy/test_llm_api_pytorch.py::TestNemotron35Lightning::test_nvfp4_cutedsl_mtp3_chunked_prefill + - examples/serve/test_serve.py::test_nemotron35_lightning_30b[nvfp4] + - examples/serve/test_serve.py::test_nemotron35_lightning_30b[bf16] - examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[text_reasoning_on] - examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[text_reasoning_off] - examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[text_streaming] diff --git a/tests/integration/test_lists/qa/llm_spark_perf.yml b/tests/integration/test_lists/qa/llm_spark_perf.yml index 7cbb2ef70e70..8c4e816e229c 100644 --- a/tests/integration/test_lists/qa/llm_spark_perf.yml +++ b/tests/integration/test_lists/qa/llm_spark_perf.yml @@ -21,6 +21,8 @@ llm_spark_perf: - perf/test_perf.py::test_perf[nemotron_3_nano_omni_nvfp4-serve-pytorch-streaming-float4-maxbs:1-input_output_len:32768,1024-reqs:1-con:1] # Nemotron-3-Nano-Omni-30B NVFP4: image benchmark (1526x1024 image + ISL 2048) - perf/test_perf.py::test_perf[nemotron_3_nano_omni_nvfp4_image-serve-pytorch-streaming-float4-maxbs:1-input_output_len:2048,1024-reqs:1-con:1] + - perf/test_perf.py::test_perf[nemotron_3.5_lightning_30b_nvfp4_mtp-serve-pytorch-streaming-float4-maxbs:1-maxnt:8192-input_output_len:2048,256-kv_cache_dtype:fp8-reqs:1-con:1] + - perf/test_perf.py::test_perf[nemotron_3.5_lightning_30b_bf16_mtp-serve-pytorch-streaming-bfloat16-maxbs:1-maxnt:8192-input_output_len:2048,256-kv_cache_dtype:fp8-reqs:1-con:1] - perf/test_perf.py::test_perf[nemotron_3_super_120b_nvfp4-serve-pytorch-streaming-float4-maxbs:8-maxnt:4096-input_output_len:2048,256-kv_cache_dtype:fp8-reqs:1-con:1] - perf/test_perf.py::test_perf[nemotron_3_super_120b_nvfp4_mtp-serve-pytorch-streaming-float4-maxbs:8-maxnt:4096-input_output_len:2048,256-kv_cache_dtype:fp8-reqs:1-con:1] - perf/test_perf.py::test_perf[nemotron_3_super_120b_nvfp4-serve-pytorch-streaming-float4-maxbs:8-maxnt:4096-input_output_len:8192,512-kv_cache_dtype:fp8-reqs:1-con:1] From 0faffc22c5beb5379b2e25195baeab77b2f781db Mon Sep 17 00:00:00 2001 From: Jenny Liu Date: Fri, 18 Sep 2026 01:34:55 +0000 Subject: [PATCH 2/2] change max-batch-size to 8 Signed-off-by: Jenny Liu --- tests/integration/defs/examples/serve/test_serve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/defs/examples/serve/test_serve.py b/tests/integration/defs/examples/serve/test_serve.py index ea377816f31c..fb316b915a00 100644 --- a/tests/integration/defs/examples/serve/test_serve.py +++ b/tests/integration/defs/examples/serve/test_serve.py @@ -442,7 +442,7 @@ def test_nemotron35_lightning_30b(serve_test_root, tmp_path, precision): "--port", str(port), "--max_batch_size", - "1", + "8", "--max_num_tokens", "8192", "--trust_remote_code",