From fb4fca6625e9209c54f518607292f877e1f2a88c Mon Sep 17 00:00:00 2001 From: rounakbende10 Date: Thu, 3 Sep 2026 11:08:33 -0400 Subject: [PATCH 1/3] Add model configs: GLM, DeepSeek V4, Inkling, Laguna, Laguna NVFP4 - GLM 5.2 FP8: 262K context, v0.24.0 - DeepSeek V4 Flash: 1M context - DeepSeek V4 Flash NVFP4: 1M context, native FP4 on B200 - Inkling Small BF16: 1M context - Laguna S 2.1 BF16: 1M context, 30.61x concurrency on 8x H200 - Laguna S 2.1 NVFP4: 1M context, 2.33x concurrency on 1x B200 - Qwen 3.6 27B: updated max-model-len 131K to 262K --- src/coding_agent_bench/models/__init__.py | 12 +++ src/coding_agent_bench/models/configs.py | 115 +++++++++++++++++++++- 2 files changed, 126 insertions(+), 1 deletion(-) diff --git a/src/coding_agent_bench/models/__init__.py b/src/coding_agent_bench/models/__init__.py index c5a4a8c..a66588d 100644 --- a/src/coding_agent_bench/models/__init__.py +++ b/src/coding_agent_bench/models/__init__.py @@ -6,6 +6,12 @@ RedHatAI_Mistral_Small_4_119B_2603_NVFP4, RedHatAI_NVIDIA_Nemotron_3_Super_120B_A12B_NVFP4, RedHatAI_Qwen3_6_27B_FP8, + RedHatAI_GLM_5_2_FP8, + RedHatAI_DeepSeek_V4_Flash, + RedHatAI_DeepSeek_V4_Flash_NVFP4_FP8, + RedHatAI_Inkling_Small, + RedHatAI_Laguna_S_2_1, + poolside_Laguna_S_2_1_NVFP4, ) MODEL_CONFIGS: list[type[ModelConfig]] = [ @@ -15,6 +21,12 @@ RedHatAI_Mistral_Small_4_119B_2603_NVFP4, RedHatAI_NVIDIA_Nemotron_3_Super_120B_A12B_NVFP4, RedHatAI_Qwen3_6_27B_FP8, + RedHatAI_GLM_5_2_FP8, + RedHatAI_DeepSeek_V4_Flash, + RedHatAI_DeepSeek_V4_Flash_NVFP4_FP8, + RedHatAI_Inkling_Small, + RedHatAI_Laguna_S_2_1, + poolside_Laguna_S_2_1_NVFP4, ] MODEL_REGISTRY: dict[str, ModelConfig] = {cls.name: cls() for cls in MODEL_CONFIGS} diff --git a/src/coding_agent_bench/models/configs.py b/src/coding_agent_bench/models/configs.py index 7e4a5bd..255254b 100644 --- a/src/coding_agent_bench/models/configs.py +++ b/src/coding_agent_bench/models/configs.py @@ -76,7 +76,7 @@ class RedHatAI_NVIDIA_Nemotron_3_Super_120B_A12B_NVFP4(ModelConfig): ] class RedHatAI_Qwen3_6_27B_FP8(ModelConfig): - + name = "RedHatAI/Qwen3.6-27B-FP8" model_max_len = 131072 args = [ @@ -90,3 +90,116 @@ class RedHatAI_Qwen3_6_27B_FP8(ModelConfig): "--tool-call-parser", "qwen3_coder", "--default-chat-template-kwargs", '{"enable_thinking": true}', ] + + +class RedHatAI_GLM_5_2_FP8(ModelConfig): + # Verified: 8x H200 141GB, concurrency 2.23x at 262K context + # Note: cannot fit 1M context on 8x H200 (needs 52.68 GiB KV, only 23.78 GiB available) + + name = "RedHatAI/GLM-5.2-FP8" + model_max_len = 262144 + args = [ + "--model", "RedHatAI/GLM-5.2-FP8", + "--dtype", "auto", + "--max-model-len", "262144", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--enable-expert-parallel", + "--enable-auto-tool-choice", + "--reasoning-parser", "glm45", + "--tool-call-parser", "glm47", + ] + + +class RedHatAI_DeepSeek_V4_Flash(ModelConfig): + # Verified: 8x H200 141GB, concurrency 9.91x at 1M context + # Note: --moe-backend deep_gemm_mega_moe is B200-only (SM100) + + name = "RedHatAI/DeepSeek-V4-Flash" + image = "vllm/vllm-openai:v0.27.1" + model_max_len = 1048576 + args = [ + "--model", "RedHatAI/DeepSeek-V4-Flash", + "--dtype", "auto", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--block-size", "256", + "--enable-expert-parallel", + "--enable-auto-tool-choice", + "--tokenizer-mode", "deepseek_v4", + "--tool-call-parser", "deepseek_v4", + "--reasoning-parser", "deepseek_v4", + ] + + +class RedHatAI_DeepSeek_V4_Flash_NVFP4_FP8(ModelConfig): + # Verified: 8x H200 141GB, concurrency 9.81x at 1M context + # Note: Marlin FP4 fallback on H200 (no native SM100 FP4), similar concurrency to base + + name = "RedHatAI/DeepSeek-V4-Flash-NVFP4-FP8" + image = "vllm/vllm-openai:v0.27.1" + model_max_len = 1048576 + args = [ + "--model", "RedHatAI/DeepSeek-V4-Flash-NVFP4-FP8", + "--dtype", "auto", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--block-size", "256", + "--enable-expert-parallel", + "--enable-auto-tool-choice", + "--tokenizer-mode", "deepseek_v4", + "--tool-call-parser", "deepseek_v4", + "--reasoning-parser", "deepseek_v4", + ] + + +class RedHatAI_Inkling_Small(ModelConfig): + # Verified: 8x H200 141GB, BF16, concurrency 13.22x at 1M context + + name = "RedHatAI/Inkling-Small" + image = "vllm/vllm-openai:v0.27.1" + model_max_len = 1048576 + args = [ + "--model", "RedHatAI/Inkling-Small", + "--dtype", "auto", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--enable-auto-tool-choice", + "--tool-call-parser", "inkling", + "--reasoning-parser", "inkling", + ] + + +class RedHatAI_Laguna_S_2_1(ModelConfig): + # Verified: 8x H200 141GB, BF16, max-model-len 1048576, concurrency 30.61x + + name = "RedHatAI/Laguna-S-2.1" + model_max_len = 1048576 + args = [ + "--model", "RedHatAI/Laguna-S-2.1", + "--dtype", "auto", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--enable-auto-tool-choice", + "--reasoning-parser", "poolside_v1", + "--tool-call-parser", "poolside_v1", + "--default-chat-template-kwargs", '{"enable_thinking": true}', + ] + +class poolside_Laguna_S_2_1_NVFP4(ModelConfig): + # Verified: 1x B200 183GB, NVFP4, max-model-len 1048576, concurrency 2.33x + + name = "poolside/Laguna-S-2.1-NVFP4" + model_max_len = 1048576 + args = [ + "--model", "poolside/Laguna-S-2.1-NVFP4", + "--dtype", "auto", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--enable-auto-tool-choice", + "--reasoning-parser", "poolside_v1", + "--tool-call-parser", "poolside_v1", + "--default-chat-template-kwargs", '{"enable_thinking": true}', + ] + + From 1cdad6b60daccd2a245016fd2be4e8905ed1d0a5 Mon Sep 17 00:00:00 2001 From: rounakbende10 Date: Wed, 9 Sep 2026 15:05:51 -0400 Subject: [PATCH 2/3] Add hardware-specific args and verify B200 DeepSeek/Inkling configs Adds a hardware_extra_args field to ModelConfig so per-platform vLLM flags (e.g. B200-only kernels) can be layered on top of the base args without duplicating whole configs. NebiusInstanceManager.start_model now reads the instance platform and injects the matching extra args. Applies this to DeepSeek V4 Flash (FP8 and NVFP4) for B200-specific MoE backend and indexer cache flags, with concurrency numbers verified on real 8x B200 hardware: - DeepSeek V4 Flash (FP8): 14.02x at 1M (mega_moe + fp4_indexer_cache) - DeepSeek V4 Flash NVFP4: 13.60x at 1M (native FP4 + fp4_indexer_cache) Also documents two Inkling-Small findings from testing: --tokenizer-mode inkling is a no-op, and --kernel-config.enable_flashinfer_autotune=False breaks fp8 KV cache. --- src/coding_agent_bench/models/base.py | 3 ++- src/coding_agent_bench/models/configs.py | 12 ++++++++++-- src/coding_agent_bench/nebius_utils.py | 7 +++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/coding_agent_bench/models/base.py b/src/coding_agent_bench/models/base.py index acc064d..d2c885e 100644 --- a/src/coding_agent_bench/models/base.py +++ b/src/coding_agent_bench/models/base.py @@ -1,11 +1,12 @@ from abc import ABC class ModelConfig(ABC): - + name: str image: str = "vllm/vllm-openai:v0.24.0" args: list[str] model_max_len: int + hardware_extra_args: dict[str, list[str]] = {} default_args: list[str] = [ "--gpu-memory-utilization", "0.9", "--async-scheduling", diff --git a/src/coding_agent_bench/models/configs.py b/src/coding_agent_bench/models/configs.py index 255254b..0578d51 100644 --- a/src/coding_agent_bench/models/configs.py +++ b/src/coding_agent_bench/models/configs.py @@ -113,7 +113,7 @@ class RedHatAI_GLM_5_2_FP8(ModelConfig): class RedHatAI_DeepSeek_V4_Flash(ModelConfig): # Verified: 8x H200 141GB, concurrency 9.91x at 1M context - # Note: --moe-backend deep_gemm_mega_moe is B200-only (SM100) + # B200: 14.02x at 1M with mega_moe + attention_config.use_fp4_indexer_cache name = "RedHatAI/DeepSeek-V4-Flash" image = "vllm/vllm-openai:v0.27.1" @@ -130,11 +130,14 @@ class RedHatAI_DeepSeek_V4_Flash(ModelConfig): "--tool-call-parser", "deepseek_v4", "--reasoning-parser", "deepseek_v4", ] + hardware_extra_args = { + "gpu-b200-sxm": ["--moe-backend", "deep_gemm_mega_moe", "--attention_config.use_fp4_indexer_cache", "True"], + } class RedHatAI_DeepSeek_V4_Flash_NVFP4_FP8(ModelConfig): # Verified: 8x H200 141GB, concurrency 9.81x at 1M context - # Note: Marlin FP4 fallback on H200 (no native SM100 FP4), similar concurrency to base + # B200: 13.60x at 1M with native FP4 + attention_config.use_fp4_indexer_cache. Marlin fallback on H200. name = "RedHatAI/DeepSeek-V4-Flash-NVFP4-FP8" image = "vllm/vllm-openai:v0.27.1" @@ -151,10 +154,15 @@ class RedHatAI_DeepSeek_V4_Flash_NVFP4_FP8(ModelConfig): "--tool-call-parser", "deepseek_v4", "--reasoning-parser", "deepseek_v4", ] + hardware_extra_args = { + "gpu-b200-sxm": ["--attention_config.use_fp4_indexer_cache", "True"], + } class RedHatAI_Inkling_Small(ModelConfig): # Verified: 8x H200 141GB, BF16, concurrency 13.22x at 1M context + # --tokenizer-mode inkling is a no-op (vLLM already defaults to it for this architecture) + # --kernel-config.enable_flashinfer_autotune=False breaks fp8 KV cache (AssertionError in FA4 kernel) — do not add name = "RedHatAI/Inkling-Small" image = "vllm/vllm-openai:v0.27.1" diff --git a/src/coding_agent_bench/nebius_utils.py b/src/coding_agent_bench/nebius_utils.py index 644ce9d..97dbab1 100644 --- a/src/coding_agent_bench/nebius_utils.py +++ b/src/coding_agent_bench/nebius_utils.py @@ -429,8 +429,10 @@ async def start_model(self, instance_name: str, model_name: str): else: raise e - # Get the number of gpus from resources (e.g. 1gpu-16vcpu-200gb) - resources = instance_details.get("spec", {}).get("resources", {}).get("preset") + # Get the number of gpus and platform from resources (e.g. 1gpu-16vcpu-200gb) + spec_resources = instance_details.get("spec", {}).get("resources", {}) + resources = spec_resources.get("preset") + platform = spec_resources.get("platform", "") if resources is None: raise ValueError("Unable to get resources from instance details") tensor_parallel_size = int(resources.split("-")[0].replace("gpu", "")) @@ -449,6 +451,7 @@ async def start_model(self, instance_name: str, model_name: str): ] docker_cmd += [model_config.image] docker_cmd += model_config.args + model_config.default_args + docker_cmd += model_config.hardware_extra_args.get(platform, []) docker_cmd += ["--tensor-parallel-size", str(tensor_parallel_size)] hf_token = os.environ.get("HF_TOKEN", "") From ddd797bb0cbfafde711c28adee2fb497198068c5 Mon Sep 17 00:00:00 2001 From: rounakbende10 Date: Wed, 9 Sep 2026 15:12:03 -0400 Subject: [PATCH 3/3] Also match gpu-b200-sxm-a platform variant for DeepSeek hardware args Some Nebius regions/tenants expose B200 under a different platform name suffix; match both so the extra args apply regardless of which variant the instance reports. --- src/coding_agent_bench/models/configs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/coding_agent_bench/models/configs.py b/src/coding_agent_bench/models/configs.py index 0578d51..e5ca1d7 100644 --- a/src/coding_agent_bench/models/configs.py +++ b/src/coding_agent_bench/models/configs.py @@ -132,6 +132,7 @@ class RedHatAI_DeepSeek_V4_Flash(ModelConfig): ] hardware_extra_args = { "gpu-b200-sxm": ["--moe-backend", "deep_gemm_mega_moe", "--attention_config.use_fp4_indexer_cache", "True"], + "gpu-b200-sxm-a": ["--moe-backend", "deep_gemm_mega_moe", "--attention_config.use_fp4_indexer_cache", "True"], } @@ -156,6 +157,7 @@ class RedHatAI_DeepSeek_V4_Flash_NVFP4_FP8(ModelConfig): ] hardware_extra_args = { "gpu-b200-sxm": ["--attention_config.use_fp4_indexer_cache", "True"], + "gpu-b200-sxm-a": ["--attention_config.use_fp4_indexer_cache", "True"], }