diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 857cbdabe4..33430ce850 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -45,6 +45,7 @@ from vllm.v1.core.kv_cache_utils import ( from vllm.v1.kv_cache_interface import ( ChunkedLocalAttentionSpec, FullAttentionSpec, + HiddenStateCacheSpec, KVCacheConfig, KVCacheGroupSpec, KVCacheSpec, @@ -2196,6 +2197,65 @@ def _grouping_config(): ) +def test_hidden_state_group_preserves_hybrid_prefix_cache_granularity(): + block_size = 544 + full_spec = FullAttentionSpec( + block_size=block_size, + num_kv_heads=1, + # FullAttentionSpec stores both K and V, so this produces a + # 544 * 1 * (512 + 512) * 2 = 1,114,112-byte page. + head_size=512, + dtype=torch.bfloat16, + ) + mamba_spec = MambaSpec( + block_size=block_size, + shapes=((557056,),), + dtypes=(torch.bfloat16,), + mamba_cache_mode="align", + ) + hidden_spec = HiddenStateCacheSpec( + block_size=block_size, + num_kv_heads=3, + head_size=1024, + dtype=torch.bfloat16, + ) + assert full_spec.page_size_bytes == mamba_spec.page_size_bytes + + groups = get_kv_cache_groups( + _grouping_config(), + { + "model.full_attn": full_spec, + "model.mamba": mamba_spec, + "cache_only_layers.0": hidden_spec, + }, + ) + + hidden_group = next( + group + for group in groups + if isinstance(group.kv_cache_spec, HiddenStateCacheSpec) + ) + assert hidden_group.kv_cache_spec.block_size == 136 + + kv_cache_config = KVCacheConfig( + num_blocks=1, + kv_cache_tensors=[], + kv_cache_groups=groups, + ) + vllm_config = SimpleNamespace( + cache_config=SimpleNamespace( + block_size=16, + enable_prefix_caching=True, + prefix_match_unit=None, + ), + parallel_config=SimpleNamespace(decode_context_parallel_size=1), + kv_transfer_config=object(), + ) + assert kv_cache_utils.resolve_kv_cache_block_sizes( + kv_cache_config, vllm_config + ) == (544, 136) + + def test_mla_draft_prefers_standard_layout_when_pages_can_be_unified(): specs = { "target.0.attn": new_mla_spec(), diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 04c5f5883a..770dffa4f2 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -645,6 +645,9 @@ def resolve_kv_cache_block_sizes( if not (cache_config.enable_prefix_caching or connector_enabled): return scheduler_block_size, scheduler_block_size + # Mamba groups outside align mode break divisibility; back off to the + # scheduler block size. Read the mode from the resolved group spec because + # its block size may have been updated independently of cache_config. if any( isinstance(g.kv_cache_spec, MambaSpec) and g.kv_cache_spec.mamba_cache_mode != "align" @@ -1741,6 +1744,13 @@ def _annotate_eagle_groups_deepseek_v4( break +def _largest_divisor_at_most(value: int, limit: int) -> int: + for candidate in range(min(value, limit), 0, -1): + if value % candidate == 0: + return candidate + return 1 + + def get_kv_cache_groups( vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec] ) -> list[KVCacheGroupSpec]: @@ -1806,9 +1816,20 @@ def get_kv_cache_groups( # Add hidden-state layers back with page aligned to the common page. if hidden_specs: common_page = get_uniform_page_size([g.kv_cache_spec for g in groups]) + group_block_size = math.gcd(*(g.kv_cache_spec.block_size for g in groups)) for name, spec in hidden_specs.items(): per_token = spec.num_kv_heads * spec.head_size * get_dtype_size(spec.dtype) - new_bs = max(common_page // per_token, 1) + max_block_size = max(common_page // per_token, 1) + new_bs = _largest_divisor_at_most(group_block_size, max_block_size) + wasted_bytes = common_page - new_bs * per_token + logger.info( + "Using block size %d for hidden-state cache layer %s; " + "page alignment wastes %d bytes (%.2f%%) per block", + new_bs, + name, + wasted_bytes, + wasted_bytes / common_page * 100, + ) aligned = replace(spec, block_size=new_bs, page_size_padded=common_page) groups.append(KVCacheGroupSpec([name], aligned))