[Core] Account for num_gpu_blocks_override in max_model_len checks (#41069)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -34,7 +34,10 @@ def _run_vllm(vllm_runner):
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
cudagraph_mode=CUDAGraphMode.NONE,
|
||||
),
|
||||
num_gpu_blocks_override=8,
|
||||
# Phi-tiny-MoE uses SWA, whose admission cap is `cdiv(L, block_size) + 1`
|
||||
# at default block_size=16 — i.e. 17 blocks for max_model_len=256. Use
|
||||
# 32 for headroom.
|
||||
num_gpu_blocks_override=32,
|
||||
):
|
||||
pass
|
||||
|
||||
@@ -190,7 +193,7 @@ def _run_model(vllm_runner, spec: ModelStartupSpec):
|
||||
cudagraph_mode=CUDAGraphMode.NONE,
|
||||
pass_config=PassConfig(fuse_allreduce_rms=False),
|
||||
),
|
||||
num_gpu_blocks_override=8,
|
||||
num_gpu_blocks_override=16,
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
Reference in New Issue
Block a user