[Bugfix][Core] Reserve the KV null block when validating max_model_len (#47272)

Signed-off-by: hyungjun oh <92hyungjun@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
92hyungjun
2026-08-20 02:49:23 +00:00
committed by GitHub
co-authored by Claude Nick Hill
parent c0233fcf01
commit 76fb6d210a
6 changed files with 109 additions and 16 deletions
+73 -1
View File
@@ -27,6 +27,7 @@ from vllm.v1.core.kv_cache_utils import (
BlockHash,
FreeKVCacheBlockQueue,
KVCacheBlock,
check_enough_kv_cache_memory,
estimate_max_model_len,
generate_block_hash_extra_keys,
generate_scheduler_kv_cache_config,
@@ -2685,7 +2686,9 @@ def test_auto_fit_max_model_len_with_hybrid():
"layer_2": new_kv_cache_spec(),
}
available_memory = mem_per_block_per_layer * (1024 // 16 + 1 + gamma)
# One extra block on top of what a 1024-token request needs: the pool
# reserves one block as the null block.
available_memory = mem_per_block_per_layer * (1024 // 16 + 1 + gamma + 1)
_kv_cache_configs = get_kv_cache_configs(
vllm_config, [kv_cache_specs], [available_memory]
)
@@ -3095,3 +3098,72 @@ def test_resolve_block_hashes_rejects_mismatched_view():
mismatched = BlockHashListWithBlockSize(raw, 2, 8)
with pytest.raises(AssertionError):
resolve_block_hashes(mismatched, 2, 4)
@pytest.mark.parametrize("use_override", [True, False])
def test_kv_cache_reserves_null_block_for_max_model_len(use_override):
"""A KV cache sized to exactly the blocks a max_model_len request needs must
be rejected. BlockPool keeps one block as the null block, so only
num_blocks - 1 are usable; accepting num_blocks == needed would leave the
request unschedulable and hang the engine. Covers both the
num_gpu_blocks_override path and the memory-derived block count.
"""
block_size = 16
max_model_len = 512 # needs 512 / 16 = 32 blocks
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
spec = new_kv_cache_spec(block_size=block_size)
# 32 blocks -> only 31 usable after the null block: one short -> reject.
if use_override:
# Ample raw memory; the override alone constrains the block count.
vllm_config.cache_config.num_gpu_blocks_override = 32
available_memory = [spec.page_size_bytes * 1024]
else:
available_memory = [spec.page_size_bytes * 32]
with pytest.raises(ValueError, match="max seq len"):
get_kv_cache_configs(vllm_config, [{"layer1": spec}], available_memory)
def test_auto_fit_max_model_len_reserves_null_block():
"""Auto-fit (max_model_len=-1) must size max_model_len against usable
blocks, not the total pool. With memory for exactly 64 blocks, one is the
null block, so auto-fit must settle on 63 * block_size; picking the full
pool would leave a max-length request unschedulable and hang the engine.
"""
block_size = 16
model_config = ModelConfig(max_model_len=1024)
model_config.original_max_model_len = -1
vllm_config = VllmConfig(model_config=model_config)
spec = new_kv_cache_spec(block_size=block_size)
# Exactly the 1024 / 16 = 64 blocks a full-length request would need.
available_memory = [spec.page_size_bytes * 64]
get_kv_cache_configs(vllm_config, [{"layer1": spec}], available_memory)
assert vllm_config.model_config.max_model_len == 63 * block_size
def test_check_enough_kv_cache_memory_reserves_null_block():
"""The public admission check must reject a KV cache sized to exactly the
blocks a max_model_len request needs. BlockPool keeps one block as the
null block, so only num_blocks - 1 are usable; accepting
num_blocks == needed would leave the request unschedulable and hang the
engine.
"""
block_size = 16
max_model_len = 512 # needs 512 / 16 = 32 blocks
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
spec = new_kv_cache_spec(block_size=block_size)
# 32 blocks -> only 31 usable after the null block: one short -> reject.
with pytest.raises(ValueError, match="max seq len"):
check_enough_kv_cache_memory(
vllm_config, {"layer1": spec}, spec.page_size_bytes * 32
)
# 33 blocks -> 32 usable after the null block -> accept.
check_enough_kv_cache_memory(
vllm_config, {"layer1": spec}, spec.page_size_bytes * 33
)