From 8e958902eee56ca5158728f1dd5a32246f0246f3 Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Wed, 12 Aug 2026 07:08:40 -0400 Subject: [PATCH] [Bugfix] Disable fine-grained prefix-cache hits for incompatible hybrid KV layouts (#51843) Signed-off-by: mgoin --- .../test_partial_prefix_cache_hits.py | 72 +++++++++++++++++++ vllm/v1/core/kv_cache_coordinator.py | 26 ++++++- vllm/v1/core/sched/scheduler.py | 1 + 3 files changed, 96 insertions(+), 3 deletions(-) diff --git a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py index b193f43abd..97199a15ab 100644 --- a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py +++ b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py @@ -24,6 +24,7 @@ from vllm.v1.kv_cache_interface import ( KVCacheConfig, KVCacheGroupSpec, MambaSpec, + SlidingWindowSpec, ) @@ -1353,3 +1354,74 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks(): len(group) * block_size >= num_computed for group in computed_blocks.blocks ) assert manager.allocate_slots(req1, 4, num_computed, computed_blocks) is not None + + +def test_hybrid_sliding_window_group_disables_partial_hash_hits(): + hash_block_size = 2 + sliding_window_block_size = 2 * hash_block_size + mamba_block_size = 2 * sliding_window_block_size + kv_cache_config = KVCacheConfig( + num_blocks=64, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["full"], + FullAttentionSpec( + block_size=hash_block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["mamba"], + MambaSpec( + block_size=mamba_block_size, + shapes=(1, 1), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ), + ), + KVCacheGroupSpec( + ["swa_draft"], + SlidingWindowSpec( + block_size=sliding_window_block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=sliding_window_block_size, + ), + is_eagle_group=True, + ), + ], + ) + manager = make_kv_cache_manager( + kv_cache_config=kv_cache_config, + max_model_len=8192, + enable_caching=True, + hash_block_size=hash_block_size, + use_eagle=True, + ) + + tokens = list(range(3 * sliding_window_block_size)) + request = make_request("0", tokens, hash_block_size, sha256) + computed_blocks, num_computed, _ = manager.get_computed_blocks(request) + assert not manager.coordinator.enable_partial_hash_hits + assert ( + manager.allocate_slots(request, mamba_block_size, num_computed, computed_blocks) + is not None + ) + request.num_computed_tokens = mamba_block_size + manager.new_step_starts() + assert manager.allocate_slots(request, len(tokens) - mamba_block_size) is not None + request.num_computed_tokens = len(tokens) + manager.free(request) + manager.new_step_starts() + + cached_request = make_request( + "1", tokens + [len(tokens), len(tokens) + 1], hash_block_size, sha256 + ) + computed_blocks, num_computed, _ = manager.get_computed_blocks(cached_request) + + assert num_computed == mamba_block_size + assert len(computed_blocks.blocks[0]) * hash_block_size == num_computed diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index 70cd267506..321bbb0a76 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -5,6 +5,7 @@ from collections.abc import Sequence from typing import NamedTuple from vllm import envs +from vllm.logger import init_logger from vllm.utils.math_utils import cdiv from vllm.v1.core.block_pool import BlockPool from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector @@ -26,6 +27,8 @@ from vllm.v1.kv_cache_interface import ( ) from vllm.v1.request import Request +logger = init_logger(__name__) + def _validate_prefix_cache_retention_interval( retention_interval: int | None, @@ -62,6 +65,8 @@ class KVCacheCoordinator(ABC): Coordinate the KV cache of different KV cache groups. """ + enable_partial_hash_hits = False + def __init__( self, kv_cache_config: KVCacheConfig, @@ -578,14 +583,29 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): "full-attention and Mamba groups, got: " f"{type(g.kv_cache_spec).__name__}." ) - # Partial hash hits are limited to full-attention + mamba ("align") - # without context parallelism. - self.enable_partial_hash_hits = dcp_world_size == 1 and any( + # Fine-grained hash hits require Mamba "align", no context + # parallelism, and compatible cache managers in every group. + has_partial_mamba_group = any( isinstance(g.kv_cache_spec, MambaSpec) and g.kv_cache_spec.mamba_cache_mode == "align" and g.kv_cache_spec.block_size > hash_block_size for g in kv_cache_config.kv_cache_groups ) + self.enable_partial_hash_hits = dcp_world_size == 1 and has_partial_mamba_group + if self.enable_partial_hash_hits: + unsupported_partial_hit_managers = { + type(manager).__name__ + for manager in self.single_type_managers + if not manager.supports_fine_grained_hash_lookup + and manager.block_size != hash_block_size + } + if unsupported_partial_hit_managers: + self.enable_partial_hash_hits = False + logger.warning_once( + "Disabling fine-grained prefix-cache hits because these KV " + "cache managers require block-aligned lookups: %s.", + ", ".join(sorted(unsupported_partial_hit_managers)), + ) self.verify_and_split_kv_cache_groups() @property diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index a781b45801..4ebc248b84 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -307,6 +307,7 @@ class Scheduler(SchedulerInterface): self.mamba_partial_cache_hit = ( self.need_mamba_block_aligned_split and self.hash_block_size < self.block_size + and self.kv_cache_manager.coordinator.enable_partial_hash_hits ) # Counts of non-empty steps scheduled / processed. update_from_output