[Core] Account for num_gpu_blocks_override in max_model_len checks (#41069)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -34,7 +34,10 @@ def _run_vllm(vllm_runner):
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
cudagraph_mode=CUDAGraphMode.NONE,
|
||||
),
|
||||
num_gpu_blocks_override=8,
|
||||
# Phi-tiny-MoE uses SWA, whose admission cap is `cdiv(L, block_size) + 1`
|
||||
# at default block_size=16 — i.e. 17 blocks for max_model_len=256. Use
|
||||
# 32 for headroom.
|
||||
num_gpu_blocks_override=32,
|
||||
):
|
||||
pass
|
||||
|
||||
@@ -190,7 +193,7 @@ def _run_model(vllm_runner, spec: ModelStartupSpec):
|
||||
cudagraph_mode=CUDAGraphMode.NONE,
|
||||
pass_config=PassConfig(fuse_allreduce_rms=False),
|
||||
),
|
||||
num_gpu_blocks_override=8,
|
||||
num_gpu_blocks_override=16,
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
@@ -2074,6 +2074,54 @@ def test_auto_fit_max_model_len_not_triggered():
|
||||
assert vllm_config.model_config.max_model_len == 16
|
||||
|
||||
|
||||
def test_auto_fit_max_model_len_respects_num_gpu_blocks_override():
|
||||
"""Auto-fit must size max_model_len against the override-clamped pool, not
|
||||
the raw `available_memory`. Without this, auto-fit could pick a
|
||||
max_model_len that no longer fits once `num_gpu_blocks_override` is applied.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=16384)
|
||||
model_config.original_max_model_len = -1 # request auto-fit
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
# Cap the cache to 32 blocks regardless of available memory.
|
||||
vllm_config.cache_config.num_gpu_blocks_override = 32
|
||||
|
||||
mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2
|
||||
kv_cache_specs = {
|
||||
"layer_1": new_kv_cache_spec(), # block_size=16
|
||||
"layer_2": new_kv_cache_spec(),
|
||||
}
|
||||
# Plenty of raw memory (1024 blocks per layer would fit max_model_len=16384).
|
||||
large_available_memory = mem_per_block_per_layer * 2 * 1024
|
||||
|
||||
get_kv_cache_configs(vllm_config, [kv_cache_specs], [large_available_memory])
|
||||
|
||||
# 32 blocks * block_size 16 = 512 token slots, so max_model_len must
|
||||
# auto-fit at or below that.
|
||||
assert 0 < vllm_config.model_config.max_model_len <= 32 * 16
|
||||
|
||||
|
||||
def test_check_enough_kv_cache_memory_respects_num_gpu_blocks_override():
|
||||
"""Admission check must use the override-clamped pool size, not raw
|
||||
`available_memory`. Without this, startup could accept a max_model_len
|
||||
that does not actually fit in `num_gpu_blocks_override` blocks.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=16384)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
# 32 blocks is far too small for max_model_len=16384 (would need 1024).
|
||||
vllm_config.cache_config.num_gpu_blocks_override = 32
|
||||
|
||||
mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2
|
||||
kv_cache_specs = {
|
||||
"layer_1": new_kv_cache_spec(),
|
||||
"layer_2": new_kv_cache_spec(),
|
||||
}
|
||||
# Plenty of raw memory: a bytes-only check against this would pass.
|
||||
large_available_memory = mem_per_block_per_layer * 2 * 1024
|
||||
|
||||
with pytest.raises(ValueError, match="max seq len"):
|
||||
get_kv_cache_configs(vllm_config, [kv_cache_specs], [large_available_memory])
|
||||
|
||||
|
||||
def test_unify_hybrid_kv_cache_specs():
|
||||
# 1. has_full_attention and has_sliding_window
|
||||
before_spec_1 = new_kv_cache_spec()
|
||||
|
||||
@@ -324,10 +324,13 @@ def run_test(
|
||||
):
|
||||
spec_decoding = spec_config is not None
|
||||
cache_arg: dict[str, Any] = (
|
||||
# Force preemptions
|
||||
dict(num_gpu_blocks_override=32)
|
||||
# Force preemptions: with 32 blocks the cache holds at most a single
|
||||
# max-length request, so the ~34 concurrent prompts contend and trigger
|
||||
# preemption. (Prompts here are << max_model_len, so dropping
|
||||
# max_model_len from 4096 to 512 doesn't change generation behavior.)
|
||||
dict(num_gpu_blocks_override=32, max_model_len=512)
|
||||
if test_preemption
|
||||
else dict(gpu_memory_utilization=0.9)
|
||||
else dict(gpu_memory_utilization=0.9, max_model_len=4096)
|
||||
)
|
||||
spec_mml = (spec_config or {}).get("max_model_len")
|
||||
spec_method = (spec_config or {}).get("method", "none")
|
||||
@@ -343,7 +346,6 @@ def run_test(
|
||||
|
||||
with VllmRunner(
|
||||
model,
|
||||
max_model_len=4096,
|
||||
enable_chunked_prefill=test_prefill_chunking,
|
||||
# Force prefill chunking
|
||||
max_num_batched_tokens=48 if test_prefill_chunking else None,
|
||||
|
||||
Reference in New Issue
Block a user