[Hybrid Allocator] Support full attention with different hidden size (#25101)

Signed-off-by: Chen Zhang <zhangch99@outlook.com>
This commit is contained in:
Chen Zhang
2025-09-19 23:43:59 -07:00
committed by GitHub
parent c60e6137f0
commit 9607d5eb44
6 changed files with 324 additions and 92 deletions
+103 -15
View File
@@ -18,12 +18,14 @@ from vllm.v1.core.kv_cache_manager import KVCacheManager
from vllm.v1.core.kv_cache_utils import (
BlockHash, FreeKVCacheBlockQueue, KVCacheBlock, PrefixCachingMetrics,
estimate_max_model_len, generate_block_hash_extra_keys,
get_kv_cache_configs, get_max_concurrency_for_kv_cache_config,
get_request_block_hasher, hash_block_tokens, init_none_hash,
is_kv_cache_type_uniform, make_block_hash_with_group_id)
generate_scheduler_kv_cache_config, get_kv_cache_configs,
get_max_concurrency_for_kv_cache_config, get_request_block_hasher,
hash_block_tokens, init_none_hash, is_kv_cache_spec_uniform,
make_block_hash_with_group_id)
from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
KVCacheGroupSpec, KVCacheSpec,
KVCacheTensor, SlidingWindowSpec)
KVCacheTensor, SlidingWindowSpec,
UniformTypeKVCacheSpecs)
from vllm.v1.metrics.stats import PrefixCacheStats
from vllm.v1.request import Request
@@ -927,36 +929,36 @@ def test_merge_kv_cache_spec():
assert merged_layer_spec.sliding_window == 1
def test_is_kv_cache_type_uniform():
def test_is_kv_cache_spec_uniform():
kv_cache_spec = {
"layer_1": new_kv_cache_spec(num_kv_heads=32),
"layer_2": new_kv_cache_spec(num_kv_heads=32),
}
assert is_kv_cache_type_uniform(kv_cache_spec)
assert is_kv_cache_spec_uniform(kv_cache_spec)
kv_cache_spec = {
"layer_1": new_kv_cache_spec(num_kv_heads=32),
"layer_2": new_kv_cache_spec(num_kv_heads=32, sliding_window=1),
}
assert is_kv_cache_type_uniform(kv_cache_spec)
assert is_kv_cache_spec_uniform(kv_cache_spec)
kv_cache_spec = {
"layer_1": new_kv_cache_spec(num_kv_heads=32),
"layer_2": new_sliding_window_spec(num_kv_heads=32, sliding_window=1),
}
assert not is_kv_cache_type_uniform(kv_cache_spec)
assert not is_kv_cache_spec_uniform(kv_cache_spec)
kv_cache_spec = {
"layer_1": new_sliding_window_spec(num_kv_heads=32, sliding_window=1),
"layer_2": new_sliding_window_spec(num_kv_heads=32, sliding_window=1),
}
assert is_kv_cache_type_uniform(kv_cache_spec)
assert is_kv_cache_spec_uniform(kv_cache_spec)
kv_cache_spec = {
"layer_1": new_sliding_window_spec(num_kv_heads=32, sliding_window=1),
"layer_2": new_sliding_window_spec(num_kv_heads=32, sliding_window=2),
}
assert not is_kv_cache_type_uniform(kv_cache_spec)
assert not is_kv_cache_spec_uniform(kv_cache_spec)
@pytest.mark.parametrize(
@@ -1286,14 +1288,28 @@ def test_get_kv_cache_config_one_worker():
],
)
# different hidden size, unimplemented
# different hidden size
kv_cache_specs_hybrid = {
'layer_1': new_kv_cache_spec(head_size=128),
'layer_2': new_kv_cache_spec(),
'layer_2': new_kv_cache_spec(head_size=64),
}
with pytest.raises(NotImplementedError):
get_kv_cache_configs(vllm_config, [kv_cache_specs_hybrid],
[mem_per_block_per_layer * 2 * 32])[0]
kv_cache_config_hybrid = get_kv_cache_configs(
vllm_config, [kv_cache_specs_hybrid],
[mem_per_block_per_layer * 3 * 32])[0]
assert kv_cache_config_hybrid == KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[
KVCacheTensor(size=mem_per_block_per_layer * 32 * 2,
shared_by=["layer_1"]),
KVCacheTensor(size=mem_per_block_per_layer * 32,
shared_by=["layer_2"]),
],
kv_cache_groups=[
KVCacheGroupSpec(["layer_1", "layer_2"],
UniformTypeKVCacheSpecs(
block_size=16,
kv_cache_specs=kv_cache_specs_hybrid))
])
# Test num_gpu_blocks_override
vllm_config.cache_config.num_gpu_blocks_override = 16
@@ -1324,3 +1340,75 @@ def test_get_kv_cache_configs_attention_free():
kv_cache_groups=[],
)
]
def test_generate_uniform_type_kv_cache_specs():
# All layers are full attention, can be merged
kv_cache_specs = {
'layer_1': new_kv_cache_spec(),
'layer_2': new_kv_cache_spec(head_size=128),
}
uniform_spec = UniformTypeKVCacheSpecs.from_specs(kv_cache_specs)
assert uniform_spec == UniformTypeKVCacheSpecs(
block_size=16, kv_cache_specs=kv_cache_specs)
# Full attention + sliding window, cannot be merged
kv_cache_specs = {
'layer_1': new_kv_cache_spec(),
'layer_2': new_sliding_window_spec(sliding_window=1),
}
uniform_spec = UniformTypeKVCacheSpecs.from_specs(kv_cache_specs)
assert uniform_spec is None
# different order of full attention + sliding window, cannot be merged
kv_cache_specs = {
'layer_1': new_sliding_window_spec(sliding_window=1),
'layer_2': new_kv_cache_spec(),
}
uniform_spec = UniformTypeKVCacheSpecs.from_specs(kv_cache_specs)
assert uniform_spec is None
# Same-size sliding window, can be merged
kv_cache_specs = {
'layer_1': new_sliding_window_spec(sliding_window=1),
'layer_2': new_sliding_window_spec(sliding_window=1, head_size=128),
}
uniform_spec = UniformTypeKVCacheSpecs.from_specs(kv_cache_specs)
assert uniform_spec == UniformTypeKVCacheSpecs(
block_size=16, kv_cache_specs=kv_cache_specs)
# different block sizes, cannot be merged
kv_cache_specs = {
'layer_1': new_kv_cache_spec(block_size=16),
'layer_2': new_kv_cache_spec(block_size=32),
}
uniform_spec = UniformTypeKVCacheSpecs.from_specs(kv_cache_specs)
assert uniform_spec is None
def test_generate_scheduler_kv_cache_config():
kv_cache_specs = {
'layer_1': new_kv_cache_spec(),
'layer_2': new_kv_cache_spec(head_size=128),
}
kv_cache_configs = [
KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(['layer_1', 'layer_2'],
UniformTypeKVCacheSpecs(
block_size=16,
kv_cache_specs=kv_cache_specs)),
],
)
]
scheduler_kv_cache_config = generate_scheduler_kv_cache_config(
kv_cache_configs)
assert scheduler_kv_cache_config == KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(['layer_1', 'layer_2'], new_kv_cache_spec())
],
)