[Core][DSV4] Skip caching SWA blocks that can never serve a prefix-cache hit (#42258)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Yifan Qiao
2026-05-15 15:59:18 +08:00
committed by GitHub
parent 31fa757cf9
commit 4b364f810e
4 changed files with 232 additions and 9 deletions
+139
View File
@@ -2528,6 +2528,145 @@ def test_different_block_size():
assert num_computed_tokens == 4 * 16
def test_hybrid_cache_blocks_swa_tail_window_only():
"""Within each lcm-aligned segment, SWA's ``find_longest_cache_hit`` only
returns the trailing ``ceil((sliding_window - 1) / block_size)`` blocks
(its right-to-left scan stops once a contiguous match is found). Blocks
earlier in the segment can never serve a hit, so
``HybridKVCacheCoordinator.cache_blocks`` should skip them rather than
polluting the prefix-cache hash map."""
block_size = 8
# Full attn block_size=32, SWA block_size=8, sw=8 -> lcm=32.
# tail = ceil(7/8) = 1; per_segment = 32/8 = 4.
# Per-segment template = [F, F, F, T]; only the last SWA block in each
# 32-token segment ends up in the prefix-cache hash map.
kv_cache_config = KVCacheConfig(
num_blocks=100,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["layer1"],
FullAttentionSpec(
block_size=4 * block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float16,
),
),
KVCacheGroupSpec(
["layer2"],
SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=block_size,
),
),
],
)
manager = KVCacheManager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
)
# 8 hash-blocks of 8 tokens (64 tokens, two lcm-aligned segments).
token_ids = [i for i in range(8) for _ in range(block_size)]
req = make_request("0", token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req)
blocks = manager.allocate_slots(
req,
8 * block_size,
len(computed_blocks.blocks[0]) * block_size,
computed_blocks,
)
assert blocks is not None
assert len(req.block_hashes) == 8
pool = manager.block_pool
# SWA group_id=1: only hash 3 and hash 7 (the last block of each
# 32-token segment) should be cached. Hashes 0,1,2,4,5,6 cannot serve
# a hit at any lcm-aligned length, so they must NOT be cached.
expected_cached = {3, 7}
for i in range(8):
cached = pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1])
if i in expected_cached:
assert cached is not None, f"SWA hash {i} should be cached"
else:
assert cached is None, (
f"SWA hash {i} cannot serve any lcm-aligned hit; should not be cached"
)
def test_hybrid_cache_blocks_clamped_to_lcm():
"""HybridKVCacheCoordinator.cache_blocks() clamps to lcm_block_size.
Chunks past the last lcm-aligned boundary can never participate in a
cache hit (find_longest_cache_hit always returns lcm-aligned hits), so
caching them only pollutes the prefix-cache hash map and keeps blocks
on the LRU list that could otherwise return to the free pool."""
block_size = 16
# Full attn block_size=32, SWA block_size=16 -> lcm=32.
kv_cache_config = KVCacheConfig(
num_blocks=100,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["layer1"],
FullAttentionSpec(
block_size=block_size * 2,
num_kv_heads=1,
head_size=1,
dtype=torch.float16,
),
),
KVCacheGroupSpec(
["layer2"],
SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=2 * block_size,
),
),
],
)
manager = KVCacheManager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
)
# 7 hash-blocks of 16 tokens (112 tokens). With lcm=32 the clamp truncates
# to 96 tokens — SWA caches 6 hashes, full-attn caches 3.
token_ids = [i for i in range(7) for _ in range(block_size)]
req = make_request("0", token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req)
blocks = manager.allocate_slots(
req,
7 * block_size,
len(computed_blocks.blocks[0]) * block_size,
computed_blocks,
)
assert blocks is not None
assert len(req.block_hashes) == 7
pool = manager.block_pool
# SWA group_id=1: hashes 0..5 cached (6 blocks * 16 tokens = 96), hash 6
# spans tokens [96, 112) past the lcm boundary and must NOT be cached.
for i in range(6):
assert (
pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1])
is not None
), f"SWA hash {i} should be cached"
assert pool.get_cached_block(req.block_hashes[6], kv_cache_group_ids=[1]) is None, (
"SWA hash 6 spans tokens past the lcm boundary; should not be cached"
)
def test_block_lookup_cache_single_block_per_key():
cache = BlockHashToBlockMap()
key0 = BlockHashWithGroupId(b"hash0")
+16 -5
View File
@@ -216,6 +216,7 @@ class BlockPool:
num_full_blocks: int,
block_size: int,
kv_cache_group_id: int,
block_mask: list[bool] | None = None,
) -> None:
"""Cache a list of full blocks for prefix caching.
This function takes a list of blocks that will have their block hash
@@ -233,11 +234,19 @@ class BlockPool:
be cached after this function.
block_size: Number of tokens in each block.
kv_cache_group_id: The id of the KV cache group.
block_mask: Optional mask aligned with
``blocks[num_cached_blocks:num_full_blocks]``. When provided,
blocks where the mask is False are skipped (treated like null
blocks). Used by groups whose ``find_longest_cache_hit`` only
consults a subset of blocks (e.g. SWA tail-window), so blocks
that can never serve a hit stay out of the prefix-cache hash
map.
"""
if num_cached_blocks >= num_full_blocks:
return
new_full_blocks = blocks[num_cached_blocks:num_full_blocks]
assert len(request.block_hashes) >= num_full_blocks
assert block_mask is None or len(block_mask) == len(new_full_blocks)
if block_size == self.hash_block_size:
# Common case.
block_hashes: BlockHashList = request.block_hashes
@@ -256,10 +265,10 @@ class BlockPool:
[] if self.enable_kv_cache_events else None
)
for i, blk in enumerate(new_full_blocks):
# Some blocks may be null blocks when enabling sparse attention like
# sliding window attention, or Mamba models with prefix-caching in
# align mode. We skip null blocks here.
if blk.is_null:
# Some blocks may be null or masked out when enabling sparse attention
# like sliding window attention, or Mamba models with prefix-caching
# in align mode. We skip null blocks here.
if blk.is_null or (block_mask is not None and not block_mask[i]):
continue
assert blk.block_hash is None
block_hash = new_block_hashes[i]
@@ -288,12 +297,14 @@ class BlockPool:
# Generate extra keys for each block individually.
# Each block may have different extra_keys (e.g., different MM
# features, or cache_salt only for the first block).
# Skip null blocks to match the length of new_hashes.
# Skip null/masked-out blocks to match the length of new_hashes.
extra_keys_list: list[tuple[Any, ...] | None] = []
curr_mm_idx = 0
for i in range(num_cached_blocks, num_full_blocks):
if blocks[i].is_null:
continue
if block_mask is not None and not block_mask[i - num_cached_blocks]:
continue
block_start = i * block_size
block_end = block_start + block_size
extra_keys, curr_mm_idx = generate_block_hash_extra_keys(
+16
View File
@@ -484,6 +484,22 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
if any(gid in self.eagle_group_ids for gid in group_ids)
}
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
# Cache hits in this coordinator are always a multiple of
# ``lcm_block_size`` tokens (see ``find_longest_cache_hit``). Within an
# aligned region, SWA groups only consult a subset of blocks per
# ``lcm_block_size``-segment so the unused blocks also stay out of the
# prefix-cache hash map.
num_computed_tokens = (
num_computed_tokens // self.lcm_block_size * self.lcm_block_size
)
for manager in self.single_type_managers:
manager.cache_blocks(
request,
num_computed_tokens,
alignment_tokens=self.lcm_block_size,
)
def find_longest_cache_hit(
self,
block_hashes: list[BlockHash],
+61 -4
View File
@@ -275,7 +275,12 @@ class SingleTypeKVCacheManager(ABC):
self.new_block_ids = []
return ids
def cache_blocks(self, request: Request, num_tokens: int) -> None:
def cache_blocks(
self,
request: Request,
num_tokens: int,
alignment_tokens: int | None = None,
) -> None:
"""
Cache the blocks for the request.
@@ -283,6 +288,12 @@ class SingleTypeKVCacheManager(ABC):
request: The request.
num_tokens: The total number of tokens that need to be cached
(including tokens that are already cached).
alignment_tokens: The cache-hit alignment (in tokens) used by the
coordinator's ``find_longest_cache_hit``. When greater than
this group's ``block_size``, managers whose hit logic only
returns a subset of blocks per alignment-aligned segment
(SWA) skip the rest since they can never participate in a
future cache hit.
"""
num_cached_blocks = self.num_cached_block.get(request.request_id, 0)
num_full_blocks = num_tokens // self.block_size
@@ -290,6 +301,13 @@ class SingleTypeKVCacheManager(ABC):
if num_cached_blocks >= num_full_blocks:
return
# Fast path: when the coordinator imposes no alignment constraint
if alignment_tokens is None or alignment_tokens <= self.block_size:
block_mask = None
else:
block_mask = self._cache_block_mask(
num_cached_blocks, num_full_blocks, alignment_tokens
)
self.block_pool.cache_full_blocks(
request=request,
blocks=self.req_to_blocks[request.request_id],
@@ -297,10 +315,26 @@ class SingleTypeKVCacheManager(ABC):
num_full_blocks=num_full_blocks,
block_size=self.block_size,
kv_cache_group_id=self.kv_cache_group_id,
block_mask=block_mask,
)
self.num_cached_block[request.request_id] = num_full_blocks
def _cache_block_mask(
self,
num_cached_blocks: int,
num_full_blocks: int,
alignment_tokens: int,
) -> list[bool] | None:
"""Per-block mask for ``cache_full_blocks``. ``None`` means cache
every (non-null) block — the default for full attention.
Subclasses with sparse hit semantics (SWA) override this to skip
blocks that can never serve a hit at any alignment-aligned prefix
length.
"""
return None
def free(self, request_id: str) -> None:
"""
Free the blocks for the request.
@@ -604,6 +638,19 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
computed.pop()
return computed_blocks
def _cache_block_mask(
self, num_cached_blocks: int, num_full_blocks: int, alignment_tokens: int
) -> list[bool] | None:
assert alignment_tokens > self.block_size
per_segment = alignment_tokens // self.block_size
tail = cdiv(self.sliding_window - 1, self.block_size)
if tail >= per_segment:
return None
skip = per_segment - tail
return [
i % per_segment >= skip for i in range(num_cached_blocks, num_full_blocks)
]
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
"""
Get the number of tokens that will be skipped for attention computation.
@@ -1050,9 +1097,14 @@ class MambaManager(SingleTypeKVCacheManager):
"""
return num_computed_tokens - 1
def cache_blocks(self, request: Request, num_tokens: int) -> None:
def cache_blocks(
self,
request: Request,
num_tokens: int,
alignment_tokens: int | None = None,
) -> None:
num_cached_blocks_before = self.num_cached_block.get(request.request_id, 0)
super().cache_blocks(request, num_tokens)
super().cache_blocks(request, num_tokens, alignment_tokens=alignment_tokens)
num_cached_blocks_after = self.num_cached_block.get(request.request_id, 0)
if num_cached_blocks_after > num_cached_blocks_before:
for block in self.req_to_blocks[request.request_id][
@@ -1081,7 +1133,12 @@ class CrossAttentionManager(SingleTypeKVCacheManager):
# requests, so `new_computed_blocks` should always be empty.
assert len(new_computed_blocks) == 0
def cache_blocks(self, request: Request, num_tokens: int) -> None:
def cache_blocks(
self,
request: Request,
num_tokens: int,
alignment_tokens: int | None = None,
) -> None:
# We do not cache blocks for cross-attention to be shared between
# requests, so this method is not relevant.
raise ValueError("Should not be called as prefix caching is disabled.")