[BugFix][KV Cache] Fix hybrid prefix caching with hidden-state extraction (#51108)

Signed-off-by: Canlin Guo <canlinguosdu@gmail.com>
This commit is contained in:
Canlin Guo
2026-08-06 06:42:37 +00:00
committed by GitHub
parent 8060260e1d
commit 5370461649
2 changed files with 82 additions and 1 deletions
+60
View File
@@ -45,6 +45,7 @@ from vllm.v1.core.kv_cache_utils import (
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
FullAttentionSpec,
HiddenStateCacheSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
@@ -2196,6 +2197,65 @@ def _grouping_config():
)
def test_hidden_state_group_preserves_hybrid_prefix_cache_granularity():
block_size = 544
full_spec = FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
# FullAttentionSpec stores both K and V, so this produces a
# 544 * 1 * (512 + 512) * 2 = 1,114,112-byte page.
head_size=512,
dtype=torch.bfloat16,
)
mamba_spec = MambaSpec(
block_size=block_size,
shapes=((557056,),),
dtypes=(torch.bfloat16,),
mamba_cache_mode="align",
)
hidden_spec = HiddenStateCacheSpec(
block_size=block_size,
num_kv_heads=3,
head_size=1024,
dtype=torch.bfloat16,
)
assert full_spec.page_size_bytes == mamba_spec.page_size_bytes
groups = get_kv_cache_groups(
_grouping_config(),
{
"model.full_attn": full_spec,
"model.mamba": mamba_spec,
"cache_only_layers.0": hidden_spec,
},
)
hidden_group = next(
group
for group in groups
if isinstance(group.kv_cache_spec, HiddenStateCacheSpec)
)
assert hidden_group.kv_cache_spec.block_size == 136
kv_cache_config = KVCacheConfig(
num_blocks=1,
kv_cache_tensors=[],
kv_cache_groups=groups,
)
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
block_size=16,
enable_prefix_caching=True,
prefix_match_unit=None,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
kv_transfer_config=object(),
)
assert kv_cache_utils.resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
) == (544, 136)
def test_mla_draft_prefers_standard_layout_when_pages_can_be_unified():
specs = {
"target.0.attn": new_mla_spec(),
+22 -1
View File
@@ -645,6 +645,9 @@ def resolve_kv_cache_block_sizes(
if not (cache_config.enable_prefix_caching or connector_enabled):
return scheduler_block_size, scheduler_block_size
# Mamba groups outside align mode break divisibility; back off to the
# scheduler block size. Read the mode from the resolved group spec because
# its block size may have been updated independently of cache_config.
if any(
isinstance(g.kv_cache_spec, MambaSpec)
and g.kv_cache_spec.mamba_cache_mode != "align"
@@ -1741,6 +1744,13 @@ def _annotate_eagle_groups_deepseek_v4(
break
def _largest_divisor_at_most(value: int, limit: int) -> int:
for candidate in range(min(value, limit), 0, -1):
if value % candidate == 0:
return candidate
return 1
def get_kv_cache_groups(
vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec]
) -> list[KVCacheGroupSpec]:
@@ -1806,9 +1816,20 @@ def get_kv_cache_groups(
# Add hidden-state layers back with page aligned to the common page.
if hidden_specs:
common_page = get_uniform_page_size([g.kv_cache_spec for g in groups])
group_block_size = math.gcd(*(g.kv_cache_spec.block_size for g in groups))
for name, spec in hidden_specs.items():
per_token = spec.num_kv_heads * spec.head_size * get_dtype_size(spec.dtype)
new_bs = max(common_page // per_token, 1)
max_block_size = max(common_page // per_token, 1)
new_bs = _largest_divisor_at_most(group_block_size, max_block_size)
wasted_bytes = common_page - new_bs * per_token
logger.info(
"Using block size %d for hidden-state cache layer %s; "
"page alignment wastes %d bytes (%.2f%%) per block",
new_bs,
name,
wasted_bytes,
wasted_bytes / common_page * 100,
)
aligned = replace(spec, block_size=new_bs, page_size_padded=common_page)
groups.append(KVCacheGroupSpec([name], aligned))