[BugFix][KV Cache] Fix hybrid prefix caching with hidden-state extraction (#51108)
Signed-off-by: Canlin Guo <canlinguosdu@gmail.com>
This commit is contained in:
@@ -45,6 +45,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
@@ -2196,6 +2197,65 @@ def _grouping_config():
|
||||
)
|
||||
|
||||
|
||||
def test_hidden_state_group_preserves_hybrid_prefix_cache_granularity():
|
||||
block_size = 544
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
# FullAttentionSpec stores both K and V, so this produces a
|
||||
# 544 * 1 * (512 + 512) * 2 = 1,114,112-byte page.
|
||||
head_size=512,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
mamba_spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((557056,),),
|
||||
dtypes=(torch.bfloat16,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
hidden_spec = HiddenStateCacheSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=3,
|
||||
head_size=1024,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
assert full_spec.page_size_bytes == mamba_spec.page_size_bytes
|
||||
|
||||
groups = get_kv_cache_groups(
|
||||
_grouping_config(),
|
||||
{
|
||||
"model.full_attn": full_spec,
|
||||
"model.mamba": mamba_spec,
|
||||
"cache_only_layers.0": hidden_spec,
|
||||
},
|
||||
)
|
||||
|
||||
hidden_group = next(
|
||||
group
|
||||
for group in groups
|
||||
if isinstance(group.kv_cache_spec, HiddenStateCacheSpec)
|
||||
)
|
||||
assert hidden_group.kv_cache_spec.block_size == 136
|
||||
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=16,
|
||||
enable_prefix_caching=True,
|
||||
prefix_match_unit=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
kv_transfer_config=object(),
|
||||
)
|
||||
assert kv_cache_utils.resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
) == (544, 136)
|
||||
|
||||
|
||||
def test_mla_draft_prefers_standard_layout_when_pages_can_be_unified():
|
||||
specs = {
|
||||
"target.0.attn": new_mla_spec(),
|
||||
|
||||
@@ -645,6 +645,9 @@ def resolve_kv_cache_block_sizes(
|
||||
if not (cache_config.enable_prefix_caching or connector_enabled):
|
||||
return scheduler_block_size, scheduler_block_size
|
||||
|
||||
# Mamba groups outside align mode break divisibility; back off to the
|
||||
# scheduler block size. Read the mode from the resolved group spec because
|
||||
# its block size may have been updated independently of cache_config.
|
||||
if any(
|
||||
isinstance(g.kv_cache_spec, MambaSpec)
|
||||
and g.kv_cache_spec.mamba_cache_mode != "align"
|
||||
@@ -1741,6 +1744,13 @@ def _annotate_eagle_groups_deepseek_v4(
|
||||
break
|
||||
|
||||
|
||||
def _largest_divisor_at_most(value: int, limit: int) -> int:
|
||||
for candidate in range(min(value, limit), 0, -1):
|
||||
if value % candidate == 0:
|
||||
return candidate
|
||||
return 1
|
||||
|
||||
|
||||
def get_kv_cache_groups(
|
||||
vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec]
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
@@ -1806,9 +1816,20 @@ def get_kv_cache_groups(
|
||||
# Add hidden-state layers back with page aligned to the common page.
|
||||
if hidden_specs:
|
||||
common_page = get_uniform_page_size([g.kv_cache_spec for g in groups])
|
||||
group_block_size = math.gcd(*(g.kv_cache_spec.block_size for g in groups))
|
||||
for name, spec in hidden_specs.items():
|
||||
per_token = spec.num_kv_heads * spec.head_size * get_dtype_size(spec.dtype)
|
||||
new_bs = max(common_page // per_token, 1)
|
||||
max_block_size = max(common_page // per_token, 1)
|
||||
new_bs = _largest_divisor_at_most(group_block_size, max_block_size)
|
||||
wasted_bytes = common_page - new_bs * per_token
|
||||
logger.info(
|
||||
"Using block size %d for hidden-state cache layer %s; "
|
||||
"page alignment wastes %d bytes (%.2f%%) per block",
|
||||
new_bs,
|
||||
name,
|
||||
wasted_bytes,
|
||||
wasted_bytes / common_page * 100,
|
||||
)
|
||||
aligned = replace(spec, block_size=new_bs, page_size_padded=common_page)
|
||||
groups.append(KVCacheGroupSpec([name], aligned))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user