[Core] Account for num_gpu_blocks_override in max_model_len checks (#41069)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-04-28 15:44:09 -07:00
committed by GitHub
parent f05f3664c3
commit e68fa1b90a
5 changed files with 128 additions and 60 deletions
+5 -2
View File
@@ -34,7 +34,10 @@ def _run_vllm(vllm_runner):
mode=CompilationMode.VLLM_COMPILE,
cudagraph_mode=CUDAGraphMode.NONE,
),
num_gpu_blocks_override=8,
# Phi-tiny-MoE uses SWA, whose admission cap is `cdiv(L, block_size) + 1`
# at default block_size=16 — i.e. 17 blocks for max_model_len=256. Use
# 32 for headroom.
num_gpu_blocks_override=32,
):
pass
@@ -190,7 +193,7 @@ def _run_model(vllm_runner, spec: ModelStartupSpec):
cudagraph_mode=CUDAGraphMode.NONE,
pass_config=PassConfig(fuse_allreduce_rms=False),
),
num_gpu_blocks_override=8,
num_gpu_blocks_override=16,
):
pass
+48
View File
@@ -2074,6 +2074,54 @@ def test_auto_fit_max_model_len_not_triggered():
assert vllm_config.model_config.max_model_len == 16
def test_auto_fit_max_model_len_respects_num_gpu_blocks_override():
"""Auto-fit must size max_model_len against the override-clamped pool, not
the raw `available_memory`. Without this, auto-fit could pick a
max_model_len that no longer fits once `num_gpu_blocks_override` is applied.
"""
model_config = ModelConfig(max_model_len=16384)
model_config.original_max_model_len = -1 # request auto-fit
vllm_config = VllmConfig(model_config=model_config)
# Cap the cache to 32 blocks regardless of available memory.
vllm_config.cache_config.num_gpu_blocks_override = 32
mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2
kv_cache_specs = {
"layer_1": new_kv_cache_spec(), # block_size=16
"layer_2": new_kv_cache_spec(),
}
# Plenty of raw memory (1024 blocks per layer would fit max_model_len=16384).
large_available_memory = mem_per_block_per_layer * 2 * 1024
get_kv_cache_configs(vllm_config, [kv_cache_specs], [large_available_memory])
# 32 blocks * block_size 16 = 512 token slots, so max_model_len must
# auto-fit at or below that.
assert 0 < vllm_config.model_config.max_model_len <= 32 * 16
def test_check_enough_kv_cache_memory_respects_num_gpu_blocks_override():
"""Admission check must use the override-clamped pool size, not raw
`available_memory`. Without this, startup could accept a max_model_len
that does not actually fit in `num_gpu_blocks_override` blocks.
"""
model_config = ModelConfig(max_model_len=16384)
vllm_config = VllmConfig(model_config=model_config)
# 32 blocks is far too small for max_model_len=16384 (would need 1024).
vllm_config.cache_config.num_gpu_blocks_override = 32
mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2
kv_cache_specs = {
"layer_1": new_kv_cache_spec(),
"layer_2": new_kv_cache_spec(),
}
# Plenty of raw memory: a bytes-only check against this would pass.
large_available_memory = mem_per_block_per_layer * 2 * 1024
with pytest.raises(ValueError, match="max seq len"):
get_kv_cache_configs(vllm_config, [kv_cache_specs], [large_available_memory])
def test_unify_hybrid_kv_cache_specs():
# 1. has_full_attention and has_sliding_window
before_spec_1 = new_kv_cache_spec()
@@ -324,10 +324,13 @@ def run_test(
):
spec_decoding = spec_config is not None
cache_arg: dict[str, Any] = (
# Force preemptions
dict(num_gpu_blocks_override=32)
# Force preemptions: with 32 blocks the cache holds at most a single
# max-length request, so the ~34 concurrent prompts contend and trigger
# preemption. (Prompts here are << max_model_len, so dropping
# max_model_len from 4096 to 512 doesn't change generation behavior.)
dict(num_gpu_blocks_override=32, max_model_len=512)
if test_preemption
else dict(gpu_memory_utilization=0.9)
else dict(gpu_memory_utilization=0.9, max_model_len=4096)
)
spec_mml = (spec_config or {}).get("max_model_len")
spec_method = (spec_config or {}).get("method", "none")
@@ -343,7 +346,6 @@ def run_test(
with VllmRunner(
model,
max_model_len=4096,
enable_chunked_prefill=test_prefill_chunking,
# Force prefill chunking
max_num_batched_tokens=48 if test_prefill_chunking else None,