diff --git a/src/coding_agent_bench/models/__init__.py b/src/coding_agent_bench/models/__init__.py index c5a4a8c..a66588d 100644 --- a/src/coding_agent_bench/models/__init__.py +++ b/src/coding_agent_bench/models/__init__.py @@ -6,6 +6,12 @@ RedHatAI_Mistral_Small_4_119B_2603_NVFP4, RedHatAI_NVIDIA_Nemotron_3_Super_120B_A12B_NVFP4, RedHatAI_Qwen3_6_27B_FP8, + RedHatAI_GLM_5_2_FP8, + RedHatAI_DeepSeek_V4_Flash, + RedHatAI_DeepSeek_V4_Flash_NVFP4_FP8, + RedHatAI_Inkling_Small, + RedHatAI_Laguna_S_2_1, + poolside_Laguna_S_2_1_NVFP4, ) MODEL_CONFIGS: list[type[ModelConfig]] = [ @@ -15,6 +21,12 @@ RedHatAI_Mistral_Small_4_119B_2603_NVFP4, RedHatAI_NVIDIA_Nemotron_3_Super_120B_A12B_NVFP4, RedHatAI_Qwen3_6_27B_FP8, + RedHatAI_GLM_5_2_FP8, + RedHatAI_DeepSeek_V4_Flash, + RedHatAI_DeepSeek_V4_Flash_NVFP4_FP8, + RedHatAI_Inkling_Small, + RedHatAI_Laguna_S_2_1, + poolside_Laguna_S_2_1_NVFP4, ] MODEL_REGISTRY: dict[str, ModelConfig] = {cls.name: cls() for cls in MODEL_CONFIGS} diff --git a/src/coding_agent_bench/models/base.py b/src/coding_agent_bench/models/base.py index acc064d..d2c885e 100644 --- a/src/coding_agent_bench/models/base.py +++ b/src/coding_agent_bench/models/base.py @@ -1,11 +1,12 @@ from abc import ABC class ModelConfig(ABC): - + name: str image: str = "vllm/vllm-openai:v0.24.0" args: list[str] model_max_len: int + hardware_extra_args: dict[str, list[str]] = {} default_args: list[str] = [ "--gpu-memory-utilization", "0.9", "--async-scheduling", diff --git a/src/coding_agent_bench/models/configs.py b/src/coding_agent_bench/models/configs.py index 7e4a5bd..e5ca1d7 100644 --- a/src/coding_agent_bench/models/configs.py +++ b/src/coding_agent_bench/models/configs.py @@ -76,7 +76,7 @@ class RedHatAI_NVIDIA_Nemotron_3_Super_120B_A12B_NVFP4(ModelConfig): ] class RedHatAI_Qwen3_6_27B_FP8(ModelConfig): - + name = "RedHatAI/Qwen3.6-27B-FP8" model_max_len = 131072 args = [ @@ -90,3 +90,126 @@ class RedHatAI_Qwen3_6_27B_FP8(ModelConfig): "--tool-call-parser", "qwen3_coder", "--default-chat-template-kwargs", '{"enable_thinking": true}', ] + + +class RedHatAI_GLM_5_2_FP8(ModelConfig): + # Verified: 8x H200 141GB, concurrency 2.23x at 262K context + # Note: cannot fit 1M context on 8x H200 (needs 52.68 GiB KV, only 23.78 GiB available) + + name = "RedHatAI/GLM-5.2-FP8" + model_max_len = 262144 + args = [ + "--model", "RedHatAI/GLM-5.2-FP8", + "--dtype", "auto", + "--max-model-len", "262144", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--enable-expert-parallel", + "--enable-auto-tool-choice", + "--reasoning-parser", "glm45", + "--tool-call-parser", "glm47", + ] + + +class RedHatAI_DeepSeek_V4_Flash(ModelConfig): + # Verified: 8x H200 141GB, concurrency 9.91x at 1M context + # B200: 14.02x at 1M with mega_moe + attention_config.use_fp4_indexer_cache + + name = "RedHatAI/DeepSeek-V4-Flash" + image = "vllm/vllm-openai:v0.27.1" + model_max_len = 1048576 + args = [ + "--model", "RedHatAI/DeepSeek-V4-Flash", + "--dtype", "auto", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--block-size", "256", + "--enable-expert-parallel", + "--enable-auto-tool-choice", + "--tokenizer-mode", "deepseek_v4", + "--tool-call-parser", "deepseek_v4", + "--reasoning-parser", "deepseek_v4", + ] + hardware_extra_args = { + "gpu-b200-sxm": ["--moe-backend", "deep_gemm_mega_moe", "--attention_config.use_fp4_indexer_cache", "True"], + "gpu-b200-sxm-a": ["--moe-backend", "deep_gemm_mega_moe", "--attention_config.use_fp4_indexer_cache", "True"], + } + + +class RedHatAI_DeepSeek_V4_Flash_NVFP4_FP8(ModelConfig): + # Verified: 8x H200 141GB, concurrency 9.81x at 1M context + # B200: 13.60x at 1M with native FP4 + attention_config.use_fp4_indexer_cache. Marlin fallback on H200. + + name = "RedHatAI/DeepSeek-V4-Flash-NVFP4-FP8" + image = "vllm/vllm-openai:v0.27.1" + model_max_len = 1048576 + args = [ + "--model", "RedHatAI/DeepSeek-V4-Flash-NVFP4-FP8", + "--dtype", "auto", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--block-size", "256", + "--enable-expert-parallel", + "--enable-auto-tool-choice", + "--tokenizer-mode", "deepseek_v4", + "--tool-call-parser", "deepseek_v4", + "--reasoning-parser", "deepseek_v4", + ] + hardware_extra_args = { + "gpu-b200-sxm": ["--attention_config.use_fp4_indexer_cache", "True"], + "gpu-b200-sxm-a": ["--attention_config.use_fp4_indexer_cache", "True"], + } + + +class RedHatAI_Inkling_Small(ModelConfig): + # Verified: 8x H200 141GB, BF16, concurrency 13.22x at 1M context + # --tokenizer-mode inkling is a no-op (vLLM already defaults to it for this architecture) + # --kernel-config.enable_flashinfer_autotune=False breaks fp8 KV cache (AssertionError in FA4 kernel) — do not add + + name = "RedHatAI/Inkling-Small" + image = "vllm/vllm-openai:v0.27.1" + model_max_len = 1048576 + args = [ + "--model", "RedHatAI/Inkling-Small", + "--dtype", "auto", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--enable-auto-tool-choice", + "--tool-call-parser", "inkling", + "--reasoning-parser", "inkling", + ] + + +class RedHatAI_Laguna_S_2_1(ModelConfig): + # Verified: 8x H200 141GB, BF16, max-model-len 1048576, concurrency 30.61x + + name = "RedHatAI/Laguna-S-2.1" + model_max_len = 1048576 + args = [ + "--model", "RedHatAI/Laguna-S-2.1", + "--dtype", "auto", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--enable-auto-tool-choice", + "--reasoning-parser", "poolside_v1", + "--tool-call-parser", "poolside_v1", + "--default-chat-template-kwargs", '{"enable_thinking": true}', + ] + +class poolside_Laguna_S_2_1_NVFP4(ModelConfig): + # Verified: 1x B200 183GB, NVFP4, max-model-len 1048576, concurrency 2.33x + + name = "poolside/Laguna-S-2.1-NVFP4" + model_max_len = 1048576 + args = [ + "--model", "poolside/Laguna-S-2.1-NVFP4", + "--dtype", "auto", + "--trust-remote-code", + "--kv-cache-dtype", "fp8", + "--enable-auto-tool-choice", + "--reasoning-parser", "poolside_v1", + "--tool-call-parser", "poolside_v1", + "--default-chat-template-kwargs", '{"enable_thinking": true}', + ] + + diff --git a/src/coding_agent_bench/nebius_utils.py b/src/coding_agent_bench/nebius_utils.py index 644ce9d..97dbab1 100644 --- a/src/coding_agent_bench/nebius_utils.py +++ b/src/coding_agent_bench/nebius_utils.py @@ -429,8 +429,10 @@ async def start_model(self, instance_name: str, model_name: str): else: raise e - # Get the number of gpus from resources (e.g. 1gpu-16vcpu-200gb) - resources = instance_details.get("spec", {}).get("resources", {}).get("preset") + # Get the number of gpus and platform from resources (e.g. 1gpu-16vcpu-200gb) + spec_resources = instance_details.get("spec", {}).get("resources", {}) + resources = spec_resources.get("preset") + platform = spec_resources.get("platform", "") if resources is None: raise ValueError("Unable to get resources from instance details") tensor_parallel_size = int(resources.split("-")[0].replace("gpu", "")) @@ -449,6 +451,7 @@ async def start_model(self, instance_name: str, model_name: str): ] docker_cmd += [model_config.image] docker_cmd += model_config.args + model_config.default_args + docker_cmd += model_config.hardware_extra_args.get(platform, []) docker_cmd += ["--tensor-parallel-size", str(tensor_parallel_size)] hf_token = os.environ.get("HF_TOKEN", "")