perf(agentx): Refresh Qwen3.5 GB300 Pareto recipes

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml

@@ -0,0 +1,89 @@
+name: agg-gb300-tp2-c1-mtp-hicache-jid2530006
+model:
+  path: qwen3.5-fp4
+  container: dynamo-sglang
+  precision: fp4
+slurm:
+  time_limit: '8:00:00'
+health_check:
+  max_attempts: 1440
+  interval_seconds: 10
+resources:
+  gpu_type: gb300
+  gpus_per_node: 2
+  agg_nodes: 1
+  agg_workers: 1
+  gpus_per_agg: 2
+infra:
+  nats_max_payload_mb: 8
+frontend:
+  type: sglang
+backend:
+  aggregated_environment:
+    NCCL_NVLS_ENABLE: '1'
+    PYTHONNOUSERSITE: '1'
+    SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
+    SGLANG_SIMULATE_ACC_LEN: '3.39'
+    SGLANG_SIMULATE_ACC_METHOD: match-expected
+    SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token
+    SGL_ENABLE_JIT_DEEPGEMM: 'false'
+    TORCH_CUDA_ARCH_LIST: '10.0'
+  sglang_config:
+    aggregated:
+      served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4
+      enable-cache-report: true
+      model-path: /model/
+      trust-remote-code: true
+      tensor-parallel-size: 2
+      data-parallel-size: 1
+      expert-parallel-size: 1
+      enable-symm-mem: true
+      quantization: modelopt_fp4
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      mamba-scheduler-strategy: extra_buffer
+      mamba-track-interval: 8192
+      attention-backend: trtllm_mha
+      moe-runner-backend: flashinfer_trtllm
+      speculative-algorithm: NEXTN
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      cuda-graph-max-bs: 80
+      max-running-requests: 72
+      max-prefill-tokens: 16384
+      chunked-prefill-size: 16384
+      mem-fraction-static: 0.75
+      max-mamba-cache-size: 360
+      allow-auto-truncate: true
+      stream-interval: 50
+      scheduler-recv-interval: 10
+      tokenizer-worker-num: 6
+      page-size: 64
+      enable-hierarchical-cache: true
+      hicache-ratio: 0.9
+      hicache-io-backend: kernel
+      hicache-mem-layout: page_first_direct
+      hicache-write-policy: write_back
+      mamba-max-states-per-path: 1
+  type: sglang
+sbatch_directives:
+  mem: '0'
+  cpus-per-task: '144'
+srun_options:
+  mem: '0'
+  container-remap-root: ''
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: '8000'
+    IS_MULTINODE: 'false'
+    TP: '2'
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml

@@ -0,0 +1,89 @@
+name: agg-gb300-tp2-c24-mtp-hicache-jid2530012
+model:
+  path: qwen3.5-fp4
+  container: dynamo-sglang
+  precision: fp4
+slurm:
+  time_limit: '8:00:00'
+health_check:
+  max_attempts: 1440
+  interval_seconds: 10
+resources:
+  gpu_type: gb300
+  gpus_per_node: 2
+  agg_nodes: 1
+  agg_workers: 1
+  gpus_per_agg: 2
+infra:
+  nats_max_payload_mb: 8
+frontend:
+  type: sglang
+backend:
+  aggregated_environment:
+    NCCL_NVLS_ENABLE: '1'
+    PYTHONNOUSERSITE: '1'
+    SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
+    SGLANG_SIMULATE_ACC_LEN: '3.39'
+    SGLANG_SIMULATE_ACC_METHOD: match-expected
+    SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token
+    SGL_ENABLE_JIT_DEEPGEMM: 'false'
+    TORCH_CUDA_ARCH_LIST: '10.0'
+  sglang_config:
+    aggregated:
+      served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4
+      enable-cache-report: true
+      model-path: /model/
+      trust-remote-code: true
+      tensor-parallel-size: 2
+      data-parallel-size: 1
+      expert-parallel-size: 1
+      enable-symm-mem: true
+      quantization: modelopt_fp4
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      mamba-scheduler-strategy: extra_buffer
+      mamba-track-interval: 8192
+      attention-backend: trtllm_mha
+      moe-runner-backend: flashinfer_trtllm
+      speculative-algorithm: NEXTN
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      cuda-graph-max-bs: 80
+      max-running-requests: 72
+      max-prefill-tokens: 16384
+      chunked-prefill-size: 16384
+      mem-fraction-static: 0.75
+      max-mamba-cache-size: 360
+      allow-auto-truncate: true
+      stream-interval: 50
+      scheduler-recv-interval: 10
+      tokenizer-worker-num: 6
+      page-size: 64
+      enable-hierarchical-cache: true
+      hicache-ratio: 0.9
+      hicache-io-backend: kernel
+      hicache-mem-layout: page_first_direct
+      hicache-write-policy: write_back
+      mamba-max-states-per-path: 1
+  type: sglang
+sbatch_directives:
+  mem: '0'
+  cpus-per-task: '144'
+srun_options:
+  mem: '0'
+  container-remap-root: ''
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: '8000'
+    IS_MULTINODE: 'false'
+    TP: '2'
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml

@@ -0,0 +1,89 @@
+name: agg-gb300-tp2-c32-mtp-hicache-jid2530013
+model:
+  path: qwen3.5-fp4
+  container: dynamo-sglang
+  precision: fp4
+slurm:
+  time_limit: '8:00:00'
+health_check:
+  max_attempts: 1440
+  interval_seconds: 10
+resources:
+  gpu_type: gb300
+  gpus_per_node: 2
+  agg_nodes: 1
+  agg_workers: 1
+  gpus_per_agg: 2
+infra:
+  nats_max_payload_mb: 8
+frontend:
+  type: sglang
+backend:
+  aggregated_environment:
+    NCCL_NVLS_ENABLE: '1'
+    PYTHONNOUSERSITE: '1'
+    SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
+    SGLANG_SIMULATE_ACC_LEN: '3.39'
+    SGLANG_SIMULATE_ACC_METHOD: match-expected
+    SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token
+    SGL_ENABLE_JIT_DEEPGEMM: 'false'
+    TORCH_CUDA_ARCH_LIST: '10.0'
+  sglang_config:
+    aggregated:
+      served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4
+      enable-cache-report: true
+      model-path: /model/
+      trust-remote-code: true
+      tensor-parallel-size: 2
+      data-parallel-size: 1
+      expert-parallel-size: 1
+      enable-symm-mem: true
+      quantization: modelopt_fp4
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      mamba-scheduler-strategy: extra_buffer
+      mamba-track-interval: 8192
+      attention-backend: trtllm_mha
+      moe-runner-backend: flashinfer_trtllm
+      speculative-algorithm: NEXTN
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      cuda-graph-max-bs: 80
+      max-running-requests: 72
+      max-prefill-tokens: 16384
+      chunked-prefill-size: 16384
+      mem-fraction-static: 0.75
+      max-mamba-cache-size: 360
+      allow-auto-truncate: true
+      stream-interval: 50
+      scheduler-recv-interval: 10
+      tokenizer-worker-num: 6
+      page-size: 64
+      enable-hierarchical-cache: true
+      hicache-ratio: 0.9
+      hicache-io-backend: kernel
+      hicache-mem-layout: page_first_direct
+      hicache-write-policy: write_back
+      mamba-max-states-per-path: 1
+  type: sglang
+sbatch_directives:
+  mem: '0'
+  cpus-per-task: '144'
+srun_options:
+  mem: '0'
+  container-remap-root: ''
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: '8000'
+    IS_MULTINODE: 'false'
+    TP: '2'
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml

@@ -0,0 +1,89 @@
+name: agg-gb300-tp2-c40-mtp-hicache-jid2530015
+model:
+  path: qwen3.5-fp4
+  container: dynamo-sglang
+  precision: fp4
+slurm:
+  time_limit: '8:00:00'
+health_check:
+  max_attempts: 1440
+  interval_seconds: 10
+resources:
+  gpu_type: gb300
+  gpus_per_node: 2
+  agg_nodes: 1
+  agg_workers: 1
+  gpus_per_agg: 2
+infra:
+  nats_max_payload_mb: 8
+frontend:
+  type: sglang
+backend:
+  aggregated_environment:
+    NCCL_NVLS_ENABLE: '1'
+    PYTHONNOUSERSITE: '1'
+    SGLANG_ENABLE_FLASHINFER_GEMM: 'true'
+    SGLANG_SIMULATE_ACC_LEN: '3.39'
+    SGLANG_SIMULATE_ACC_METHOD: match-expected
+    SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token
+    SGL_ENABLE_JIT_DEEPGEMM: 'false'
+    TORCH_CUDA_ARCH_LIST: '10.0'
+  sglang_config:
+    aggregated:
+      served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4
+      enable-cache-report: true
+      model-path: /model/
+      trust-remote-code: true
+      tensor-parallel-size: 2
+      data-parallel-size: 1
+      expert-parallel-size: 1
+      enable-symm-mem: true
+      quantization: modelopt_fp4
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      mamba-scheduler-strategy: extra_buffer
+      mamba-track-interval: 8192
+      attention-backend: trtllm_mha
+      moe-runner-backend: flashinfer_trtllm
+      speculative-algorithm: NEXTN
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      cuda-graph-max-bs: 80
+      max-running-requests: 72
+      max-prefill-tokens: 16384
+      chunked-prefill-size: 16384
+      mem-fraction-static: 0.75
+      max-mamba-cache-size: 360
+      allow-auto-truncate: true
+      stream-interval: 50
+      scheduler-recv-interval: 10
+      tokenizer-worker-num: 6
+      page-size: 64
+      enable-hierarchical-cache: true
+      hicache-ratio: 0.9
+      hicache-io-backend: kernel
+      hicache-mem-layout: page_first_direct
+      hicache-write-policy: write_back
+      mamba-max-states-per-path: 1
+  type: sglang
+sbatch_directives:
+  mem: '0'
+  cpus-per-task: '144'
+srun_options:
+  mem: '0'
+  container-remap-root: ''
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: '8000'
+    IS_MULTINODE: 'false'
+    TP: '2'
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'