From 4b364f810e12a689dd1862a81a163ccb6df07839 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Fri, 15 May 2026 00:59:18 -0700 Subject: [PATCH] [Core][DSV4] Skip caching SWA blocks that can never serve a prefix-cache hit (#42258) Signed-off-by: Yifan Qiao --- tests/v1/core/test_prefix_caching.py | 139 +++++++++++++++++++ vllm/v1/core/block_pool.py | 21 ++- vllm/v1/core/kv_cache_coordinator.py | 16 +++ vllm/v1/core/single_type_kv_cache_manager.py | 65 ++++++++- 4 files changed, 232 insertions(+), 9 deletions(-) diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 0a58fa2f4f..546412b1d2 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -2528,6 +2528,145 @@ def test_different_block_size(): assert num_computed_tokens == 4 * 16 +def test_hybrid_cache_blocks_swa_tail_window_only(): + """Within each lcm-aligned segment, SWA's ``find_longest_cache_hit`` only + returns the trailing ``ceil((sliding_window - 1) / block_size)`` blocks + (its right-to-left scan stops once a contiguous match is found). Blocks + earlier in the segment can never serve a hit, so + ``HybridKVCacheCoordinator.cache_blocks`` should skip them rather than + polluting the prefix-cache hash map.""" + block_size = 8 + # Full attn block_size=32, SWA block_size=8, sw=8 -> lcm=32. + # tail = ceil(7/8) = 1; per_segment = 32/8 = 4. + # Per-segment template = [F, F, F, T]; only the last SWA block in each + # 32-token segment ends up in the prefix-cache hash map. + kv_cache_config = KVCacheConfig( + num_blocks=100, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["layer1"], + FullAttentionSpec( + block_size=4 * block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float16, + ), + ), + KVCacheGroupSpec( + ["layer2"], + SlidingWindowSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=block_size, + ), + ), + ], + ) + manager = KVCacheManager( + kv_cache_config=kv_cache_config, + max_model_len=8192, + enable_caching=True, + hash_block_size=block_size, + ) + + # 8 hash-blocks of 8 tokens (64 tokens, two lcm-aligned segments). + token_ids = [i for i in range(8) for _ in range(block_size)] + req = make_request("0", token_ids, block_size, sha256) + computed_blocks, _ = manager.get_computed_blocks(req) + blocks = manager.allocate_slots( + req, + 8 * block_size, + len(computed_blocks.blocks[0]) * block_size, + computed_blocks, + ) + assert blocks is not None + assert len(req.block_hashes) == 8 + + pool = manager.block_pool + # SWA group_id=1: only hash 3 and hash 7 (the last block of each + # 32-token segment) should be cached. Hashes 0,1,2,4,5,6 cannot serve + # a hit at any lcm-aligned length, so they must NOT be cached. + expected_cached = {3, 7} + for i in range(8): + cached = pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1]) + if i in expected_cached: + assert cached is not None, f"SWA hash {i} should be cached" + else: + assert cached is None, ( + f"SWA hash {i} cannot serve any lcm-aligned hit; should not be cached" + ) + + +def test_hybrid_cache_blocks_clamped_to_lcm(): + """HybridKVCacheCoordinator.cache_blocks() clamps to lcm_block_size. + Chunks past the last lcm-aligned boundary can never participate in a + cache hit (find_longest_cache_hit always returns lcm-aligned hits), so + caching them only pollutes the prefix-cache hash map and keeps blocks + on the LRU list that could otherwise return to the free pool.""" + block_size = 16 + # Full attn block_size=32, SWA block_size=16 -> lcm=32. + kv_cache_config = KVCacheConfig( + num_blocks=100, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["layer1"], + FullAttentionSpec( + block_size=block_size * 2, + num_kv_heads=1, + head_size=1, + dtype=torch.float16, + ), + ), + KVCacheGroupSpec( + ["layer2"], + SlidingWindowSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=2 * block_size, + ), + ), + ], + ) + manager = KVCacheManager( + kv_cache_config=kv_cache_config, + max_model_len=8192, + enable_caching=True, + hash_block_size=block_size, + ) + + # 7 hash-blocks of 16 tokens (112 tokens). With lcm=32 the clamp truncates + # to 96 tokens — SWA caches 6 hashes, full-attn caches 3. + token_ids = [i for i in range(7) for _ in range(block_size)] + req = make_request("0", token_ids, block_size, sha256) + computed_blocks, _ = manager.get_computed_blocks(req) + blocks = manager.allocate_slots( + req, + 7 * block_size, + len(computed_blocks.blocks[0]) * block_size, + computed_blocks, + ) + assert blocks is not None + assert len(req.block_hashes) == 7 + + pool = manager.block_pool + # SWA group_id=1: hashes 0..5 cached (6 blocks * 16 tokens = 96), hash 6 + # spans tokens [96, 112) past the lcm boundary and must NOT be cached. + for i in range(6): + assert ( + pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1]) + is not None + ), f"SWA hash {i} should be cached" + assert pool.get_cached_block(req.block_hashes[6], kv_cache_group_ids=[1]) is None, ( + "SWA hash 6 spans tokens past the lcm boundary; should not be cached" + ) + + def test_block_lookup_cache_single_block_per_key(): cache = BlockHashToBlockMap() key0 = BlockHashWithGroupId(b"hash0") diff --git a/vllm/v1/core/block_pool.py b/vllm/v1/core/block_pool.py index 9097079ef3..513e4bf380 100644 --- a/vllm/v1/core/block_pool.py +++ b/vllm/v1/core/block_pool.py @@ -216,6 +216,7 @@ class BlockPool: num_full_blocks: int, block_size: int, kv_cache_group_id: int, + block_mask: list[bool] | None = None, ) -> None: """Cache a list of full blocks for prefix caching. This function takes a list of blocks that will have their block hash @@ -233,11 +234,19 @@ class BlockPool: be cached after this function. block_size: Number of tokens in each block. kv_cache_group_id: The id of the KV cache group. + block_mask: Optional mask aligned with + ``blocks[num_cached_blocks:num_full_blocks]``. When provided, + blocks where the mask is False are skipped (treated like null + blocks). Used by groups whose ``find_longest_cache_hit`` only + consults a subset of blocks (e.g. SWA tail-window), so blocks + that can never serve a hit stay out of the prefix-cache hash + map. """ if num_cached_blocks >= num_full_blocks: return new_full_blocks = blocks[num_cached_blocks:num_full_blocks] assert len(request.block_hashes) >= num_full_blocks + assert block_mask is None or len(block_mask) == len(new_full_blocks) if block_size == self.hash_block_size: # Common case. block_hashes: BlockHashList = request.block_hashes @@ -256,10 +265,10 @@ class BlockPool: [] if self.enable_kv_cache_events else None ) for i, blk in enumerate(new_full_blocks): - # Some blocks may be null blocks when enabling sparse attention like - # sliding window attention, or Mamba models with prefix-caching in - # align mode. We skip null blocks here. - if blk.is_null: + # Some blocks may be null or masked out when enabling sparse attention + # like sliding window attention, or Mamba models with prefix-caching + # in align mode. We skip null blocks here. + if blk.is_null or (block_mask is not None and not block_mask[i]): continue assert blk.block_hash is None block_hash = new_block_hashes[i] @@ -288,12 +297,14 @@ class BlockPool: # Generate extra keys for each block individually. # Each block may have different extra_keys (e.g., different MM # features, or cache_salt only for the first block). - # Skip null blocks to match the length of new_hashes. + # Skip null/masked-out blocks to match the length of new_hashes. extra_keys_list: list[tuple[Any, ...] | None] = [] curr_mm_idx = 0 for i in range(num_cached_blocks, num_full_blocks): if blocks[i].is_null: continue + if block_mask is not None and not block_mask[i - num_cached_blocks]: + continue block_start = i * block_size block_end = block_start + block_size extra_keys, curr_mm_idx = generate_block_hash_extra_keys( diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index b5332ea0fd..c5e8953745 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -484,6 +484,22 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): if any(gid in self.eagle_group_ids for gid in group_ids) } + def cache_blocks(self, request: Request, num_computed_tokens: int) -> None: + # Cache hits in this coordinator are always a multiple of + # ``lcm_block_size`` tokens (see ``find_longest_cache_hit``). Within an + # aligned region, SWA groups only consult a subset of blocks per + # ``lcm_block_size``-segment so the unused blocks also stay out of the + # prefix-cache hash map. + num_computed_tokens = ( + num_computed_tokens // self.lcm_block_size * self.lcm_block_size + ) + for manager in self.single_type_managers: + manager.cache_blocks( + request, + num_computed_tokens, + alignment_tokens=self.lcm_block_size, + ) + def find_longest_cache_hit( self, block_hashes: list[BlockHash], diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 7e23f176ec..e29919022e 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -275,7 +275,12 @@ class SingleTypeKVCacheManager(ABC): self.new_block_ids = [] return ids - def cache_blocks(self, request: Request, num_tokens: int) -> None: + def cache_blocks( + self, + request: Request, + num_tokens: int, + alignment_tokens: int | None = None, + ) -> None: """ Cache the blocks for the request. @@ -283,6 +288,12 @@ class SingleTypeKVCacheManager(ABC): request: The request. num_tokens: The total number of tokens that need to be cached (including tokens that are already cached). + alignment_tokens: The cache-hit alignment (in tokens) used by the + coordinator's ``find_longest_cache_hit``. When greater than + this group's ``block_size``, managers whose hit logic only + returns a subset of blocks per alignment-aligned segment + (SWA) skip the rest since they can never participate in a + future cache hit. """ num_cached_blocks = self.num_cached_block.get(request.request_id, 0) num_full_blocks = num_tokens // self.block_size @@ -290,6 +301,13 @@ class SingleTypeKVCacheManager(ABC): if num_cached_blocks >= num_full_blocks: return + # Fast path: when the coordinator imposes no alignment constraint + if alignment_tokens is None or alignment_tokens <= self.block_size: + block_mask = None + else: + block_mask = self._cache_block_mask( + num_cached_blocks, num_full_blocks, alignment_tokens + ) self.block_pool.cache_full_blocks( request=request, blocks=self.req_to_blocks[request.request_id], @@ -297,10 +315,26 @@ class SingleTypeKVCacheManager(ABC): num_full_blocks=num_full_blocks, block_size=self.block_size, kv_cache_group_id=self.kv_cache_group_id, + block_mask=block_mask, ) self.num_cached_block[request.request_id] = num_full_blocks + def _cache_block_mask( + self, + num_cached_blocks: int, + num_full_blocks: int, + alignment_tokens: int, + ) -> list[bool] | None: + """Per-block mask for ``cache_full_blocks``. ``None`` means cache + every (non-null) block — the default for full attention. + + Subclasses with sparse hit semantics (SWA) override this to skip + blocks that can never serve a hit at any alignment-aligned prefix + length. + """ + return None + def free(self, request_id: str) -> None: """ Free the blocks for the request. @@ -604,6 +638,19 @@ class SlidingWindowManager(SingleTypeKVCacheManager): computed.pop() return computed_blocks + def _cache_block_mask( + self, num_cached_blocks: int, num_full_blocks: int, alignment_tokens: int + ) -> list[bool] | None: + assert alignment_tokens > self.block_size + per_segment = alignment_tokens // self.block_size + tail = cdiv(self.sliding_window - 1, self.block_size) + if tail >= per_segment: + return None + skip = per_segment - tail + return [ + i % per_segment >= skip for i in range(num_cached_blocks, num_full_blocks) + ] + def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: """ Get the number of tokens that will be skipped for attention computation. @@ -1050,9 +1097,14 @@ class MambaManager(SingleTypeKVCacheManager): """ return num_computed_tokens - 1 - def cache_blocks(self, request: Request, num_tokens: int) -> None: + def cache_blocks( + self, + request: Request, + num_tokens: int, + alignment_tokens: int | None = None, + ) -> None: num_cached_blocks_before = self.num_cached_block.get(request.request_id, 0) - super().cache_blocks(request, num_tokens) + super().cache_blocks(request, num_tokens, alignment_tokens=alignment_tokens) num_cached_blocks_after = self.num_cached_block.get(request.request_id, 0) if num_cached_blocks_after > num_cached_blocks_before: for block in self.req_to_blocks[request.request_id][ @@ -1081,7 +1133,12 @@ class CrossAttentionManager(SingleTypeKVCacheManager): # requests, so `new_computed_blocks` should always be empty. assert len(new_computed_blocks) == 0 - def cache_blocks(self, request: Request, num_tokens: int) -> None: + def cache_blocks( + self, + request: Request, + num_tokens: int, + alignment_tokens: int | None = None, + ) -> None: # We do not cache blocks for cross-attention to be shared between # requests, so this method is not relevant. raise ValueError("Should not be called as prefix caching is disabled.")