[Core] Account for num_gpu_blocks_override in max_model_len checks (#41069)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-04-28 15:44:09 -07:00
committed by GitHub
parent f05f3664c3
commit e68fa1b90a
5 changed files with 128 additions and 60 deletions
+5 -2
View File
@@ -34,7 +34,10 @@ def _run_vllm(vllm_runner):
mode=CompilationMode.VLLM_COMPILE,
cudagraph_mode=CUDAGraphMode.NONE,
),
num_gpu_blocks_override=8,
# Phi-tiny-MoE uses SWA, whose admission cap is `cdiv(L, block_size) + 1`
# at default block_size=16 — i.e. 17 blocks for max_model_len=256. Use
# 32 for headroom.
num_gpu_blocks_override=32,
):
pass
@@ -190,7 +193,7 @@ def _run_model(vllm_runner, spec: ModelStartupSpec):
cudagraph_mode=CUDAGraphMode.NONE,
pass_config=PassConfig(fuse_allreduce_rms=False),
),
num_gpu_blocks_override=8,
num_gpu_blocks_override=16,
):
pass