[Bugfix] Max concurrency estimation and check_enough_kv_cache_memory for models with sliding window layers (#19029)

Signed-off-by: Chen Zhang <zhangch99@outlook.com>
This commit is contained in:
Chen Zhang
2025-06-04 00:14:06 +00:00
committed by GitHub
parent 5d96533e22
commit a8da78eac9
2 changed files with 125 additions and 26 deletions
+83 -7
View File
@@ -12,13 +12,11 @@ from vllm.utils import GiB_bytes, sha256
from vllm.v1.core.kv_cache_manager import KVCacheManager
# disable yapf here as it formats differently than isort such that both fail
# yapf: disable
from vllm.v1.core.kv_cache_utils import (FreeKVCacheBlockQueue, KVCacheBlock,
PrefixCachingMetrics,
estimate_max_model_len,
generate_block_hash_extra_keys,
hash_block_tokens,
hash_request_tokens,
unify_kv_cache_configs)
from vllm.v1.core.kv_cache_utils import (
FreeKVCacheBlockQueue, KVCacheBlock, PrefixCachingMetrics,
estimate_max_model_len, generate_block_hash_extra_keys,
get_max_concurrency_for_kv_cache_config, hash_block_tokens,
hash_request_tokens, unify_kv_cache_configs)
from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
KVCacheGroupSpec, KVCacheTensor,
SlidingWindowSpec)
@@ -597,6 +595,84 @@ def test_estimate_max_model_len(model_id, max_model_len,
assert estimated_max_len == want_estimated_max_len
def test_get_max_concurrency_for_kv_cache_config():
# Create a VllmConfig
model_id = "Qwen/Qwen1.5-7B"
max_model_len = 16384
model_config = ModelConfig(
model_id,
task="generate",
tokenizer=model_id,
tokenizer_mode="auto",
trust_remote_code=False,
seed=0,
dtype="float16",
max_model_len=max_model_len,
)
scheduler_config = SchedulerConfig(max_num_batched_tokens=1024,
enable_chunked_prefill=True)
vllm_config = VllmConfig(
model_config=model_config,
scheduler_config=scheduler_config,
)
full_attention_spec = FullAttentionSpec(
block_size=16,
num_kv_heads=32,
head_size=128,
dtype=torch.float16,
use_mla=False,
)
sliding_window_spec = SlidingWindowSpec(
block_size=16,
num_kv_heads=32,
head_size=128,
dtype=torch.float16,
use_mla=False,
sliding_window=1024,
)
kv_cache_config_full_attention = KVCacheConfig(
num_blocks=int(1024 * 1.5),
tensors={},
kv_cache_groups=[
KVCacheGroupSpec([f"layer_{i}" for i in range(32)],
full_attention_spec),
],
)
max_concurrency_full_attention = get_max_concurrency_for_kv_cache_config(
vllm_config, kv_cache_config_full_attention)
assert max_concurrency_full_attention == 1.5
kv_cache_config_sliding_window = KVCacheConfig(
num_blocks=129 * 3,
tensors={},
kv_cache_groups=[
KVCacheGroupSpec([f"layer_{i}" for i in range(32)],
sliding_window_spec),
],
)
max_concurrency_sliding_window = get_max_concurrency_for_kv_cache_config(
vllm_config, kv_cache_config_sliding_window)
assert max_concurrency_sliding_window == 3
kv_cache_config_hybrid_model = KVCacheConfig(
num_blocks=(1024 + 129) * 3,
tensors={},
kv_cache_groups=[
KVCacheGroupSpec([f"layer_{i}" for i in range(32)],
full_attention_spec),
KVCacheGroupSpec([f"layer_{i}" for i in range(32, 64)],
sliding_window_spec),
],
)
max_concurrency_hybrid_model = get_max_concurrency_for_kv_cache_config(
vllm_config, kv_cache_config_hybrid_model)
assert max_concurrency_hybrid_model == 3
def test_allocate_with_lookahead():
"""Verify that lookahead tokens correctly affect block allocation"""
block_size = 4