[Core] Account for num_gpu_blocks_override in max_model_len checks (#41069)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-04-28 15:44:09 -07:00
committed by GitHub
parent f05f3664c3
commit e68fa1b90a
5 changed files with 128 additions and 60 deletions
@@ -324,10 +324,13 @@ def run_test(
):
spec_decoding = spec_config is not None
cache_arg: dict[str, Any] = (
# Force preemptions
dict(num_gpu_blocks_override=32)
# Force preemptions: with 32 blocks the cache holds at most a single
# max-length request, so the ~34 concurrent prompts contend and trigger
# preemption. (Prompts here are << max_model_len, so dropping
# max_model_len from 4096 to 512 doesn't change generation behavior.)
dict(num_gpu_blocks_override=32, max_model_len=512)
if test_preemption
else dict(gpu_memory_utilization=0.9)
else dict(gpu_memory_utilization=0.9, max_model_len=4096)
)
spec_mml = (spec_config or {}).get("max_model_len")
spec_method = (spec_config or {}).get("method", "none")
@@ -343,7 +346,6 @@ def run_test(
with VllmRunner(
model,
max_model_len=4096,
enable_chunked_prefill=test_prefill_chunking,
# Force prefill chunking
max_num_batched_tokens=48 if test_prefill_chunking else None,