[Bugfix][Core] Reserve the KV null block when validating max_model_len (#47272)
Signed-off-by: hyungjun oh <92hyungjun@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Claude
Nick Hill
parent
c0233fcf01
commit
76fb6d210a
@@ -27,6 +27,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
FreeKVCacheBlockQueue,
|
||||
KVCacheBlock,
|
||||
check_enough_kv_cache_memory,
|
||||
estimate_max_model_len,
|
||||
generate_block_hash_extra_keys,
|
||||
generate_scheduler_kv_cache_config,
|
||||
@@ -2685,7 +2686,9 @@ def test_auto_fit_max_model_len_with_hybrid():
|
||||
"layer_2": new_kv_cache_spec(),
|
||||
}
|
||||
|
||||
available_memory = mem_per_block_per_layer * (1024 // 16 + 1 + gamma)
|
||||
# One extra block on top of what a 1024-token request needs: the pool
|
||||
# reserves one block as the null block.
|
||||
available_memory = mem_per_block_per_layer * (1024 // 16 + 1 + gamma + 1)
|
||||
_kv_cache_configs = get_kv_cache_configs(
|
||||
vllm_config, [kv_cache_specs], [available_memory]
|
||||
)
|
||||
@@ -3095,3 +3098,72 @@ def test_resolve_block_hashes_rejects_mismatched_view():
|
||||
mismatched = BlockHashListWithBlockSize(raw, 2, 8)
|
||||
with pytest.raises(AssertionError):
|
||||
resolve_block_hashes(mismatched, 2, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_override", [True, False])
|
||||
def test_kv_cache_reserves_null_block_for_max_model_len(use_override):
|
||||
"""A KV cache sized to exactly the blocks a max_model_len request needs must
|
||||
be rejected. BlockPool keeps one block as the null block, so only
|
||||
num_blocks - 1 are usable; accepting num_blocks == needed would leave the
|
||||
request unschedulable and hang the engine. Covers both the
|
||||
num_gpu_blocks_override path and the memory-derived block count.
|
||||
"""
|
||||
block_size = 16
|
||||
max_model_len = 512 # needs 512 / 16 = 32 blocks
|
||||
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
|
||||
spec = new_kv_cache_spec(block_size=block_size)
|
||||
|
||||
# 32 blocks -> only 31 usable after the null block: one short -> reject.
|
||||
if use_override:
|
||||
# Ample raw memory; the override alone constrains the block count.
|
||||
vllm_config.cache_config.num_gpu_blocks_override = 32
|
||||
available_memory = [spec.page_size_bytes * 1024]
|
||||
else:
|
||||
available_memory = [spec.page_size_bytes * 32]
|
||||
|
||||
with pytest.raises(ValueError, match="max seq len"):
|
||||
get_kv_cache_configs(vllm_config, [{"layer1": spec}], available_memory)
|
||||
|
||||
|
||||
def test_auto_fit_max_model_len_reserves_null_block():
|
||||
"""Auto-fit (max_model_len=-1) must size max_model_len against usable
|
||||
blocks, not the total pool. With memory for exactly 64 blocks, one is the
|
||||
null block, so auto-fit must settle on 63 * block_size; picking the full
|
||||
pool would leave a max-length request unschedulable and hang the engine.
|
||||
"""
|
||||
block_size = 16
|
||||
model_config = ModelConfig(max_model_len=1024)
|
||||
model_config.original_max_model_len = -1
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
spec = new_kv_cache_spec(block_size=block_size)
|
||||
|
||||
# Exactly the 1024 / 16 = 64 blocks a full-length request would need.
|
||||
available_memory = [spec.page_size_bytes * 64]
|
||||
|
||||
get_kv_cache_configs(vllm_config, [{"layer1": spec}], available_memory)
|
||||
|
||||
assert vllm_config.model_config.max_model_len == 63 * block_size
|
||||
|
||||
|
||||
def test_check_enough_kv_cache_memory_reserves_null_block():
|
||||
"""The public admission check must reject a KV cache sized to exactly the
|
||||
blocks a max_model_len request needs. BlockPool keeps one block as the
|
||||
null block, so only num_blocks - 1 are usable; accepting
|
||||
num_blocks == needed would leave the request unschedulable and hang the
|
||||
engine.
|
||||
"""
|
||||
block_size = 16
|
||||
max_model_len = 512 # needs 512 / 16 = 32 blocks
|
||||
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
|
||||
spec = new_kv_cache_spec(block_size=block_size)
|
||||
|
||||
# 32 blocks -> only 31 usable after the null block: one short -> reject.
|
||||
with pytest.raises(ValueError, match="max seq len"):
|
||||
check_enough_kv_cache_memory(
|
||||
vllm_config, {"layer1": spec}, spec.page_size_bytes * 32
|
||||
)
|
||||
|
||||
# 33 blocks -> 32 usable after the null block -> accept.
|
||||
check_enough_kv_cache_memory(
|
||||
vllm_config, {"layer1": spec}, spec.page_size_bytes * 33
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user