[Core][DSV4] Skip caching SWA blocks that can never serve a prefix-cache hit (#42258)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
@@ -2528,6 +2528,145 @@ def test_different_block_size():
|
||||
assert num_computed_tokens == 4 * 16
|
||||
|
||||
|
||||
def test_hybrid_cache_blocks_swa_tail_window_only():
|
||||
"""Within each lcm-aligned segment, SWA's ``find_longest_cache_hit`` only
|
||||
returns the trailing ``ceil((sliding_window - 1) / block_size)`` blocks
|
||||
(its right-to-left scan stops once a contiguous match is found). Blocks
|
||||
earlier in the segment can never serve a hit, so
|
||||
``HybridKVCacheCoordinator.cache_blocks`` should skip them rather than
|
||||
polluting the prefix-cache hash map."""
|
||||
block_size = 8
|
||||
# Full attn block_size=32, SWA block_size=8, sw=8 -> lcm=32.
|
||||
# tail = ceil(7/8) = 1; per_segment = 32/8 = 4.
|
||||
# Per-segment template = [F, F, F, T]; only the last SWA block in each
|
||||
# 32-token segment ends up in the prefix-cache hash map.
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=4 * block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float16,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer2"],
|
||||
SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=block_size,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
|
||||
# 8 hash-blocks of 8 tokens (64 tokens, two lcm-aligned segments).
|
||||
token_ids = [i for i in range(8) for _ in range(block_size)]
|
||||
req = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req)
|
||||
blocks = manager.allocate_slots(
|
||||
req,
|
||||
8 * block_size,
|
||||
len(computed_blocks.blocks[0]) * block_size,
|
||||
computed_blocks,
|
||||
)
|
||||
assert blocks is not None
|
||||
assert len(req.block_hashes) == 8
|
||||
|
||||
pool = manager.block_pool
|
||||
# SWA group_id=1: only hash 3 and hash 7 (the last block of each
|
||||
# 32-token segment) should be cached. Hashes 0,1,2,4,5,6 cannot serve
|
||||
# a hit at any lcm-aligned length, so they must NOT be cached.
|
||||
expected_cached = {3, 7}
|
||||
for i in range(8):
|
||||
cached = pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1])
|
||||
if i in expected_cached:
|
||||
assert cached is not None, f"SWA hash {i} should be cached"
|
||||
else:
|
||||
assert cached is None, (
|
||||
f"SWA hash {i} cannot serve any lcm-aligned hit; should not be cached"
|
||||
)
|
||||
|
||||
|
||||
def test_hybrid_cache_blocks_clamped_to_lcm():
|
||||
"""HybridKVCacheCoordinator.cache_blocks() clamps to lcm_block_size.
|
||||
Chunks past the last lcm-aligned boundary can never participate in a
|
||||
cache hit (find_longest_cache_hit always returns lcm-aligned hits), so
|
||||
caching them only pollutes the prefix-cache hash map and keeps blocks
|
||||
on the LRU list that could otherwise return to the free pool."""
|
||||
block_size = 16
|
||||
# Full attn block_size=32, SWA block_size=16 -> lcm=32.
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size * 2,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float16,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer2"],
|
||||
SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=2 * block_size,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
|
||||
# 7 hash-blocks of 16 tokens (112 tokens). With lcm=32 the clamp truncates
|
||||
# to 96 tokens — SWA caches 6 hashes, full-attn caches 3.
|
||||
token_ids = [i for i in range(7) for _ in range(block_size)]
|
||||
req = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req)
|
||||
blocks = manager.allocate_slots(
|
||||
req,
|
||||
7 * block_size,
|
||||
len(computed_blocks.blocks[0]) * block_size,
|
||||
computed_blocks,
|
||||
)
|
||||
assert blocks is not None
|
||||
assert len(req.block_hashes) == 7
|
||||
|
||||
pool = manager.block_pool
|
||||
# SWA group_id=1: hashes 0..5 cached (6 blocks * 16 tokens = 96), hash 6
|
||||
# spans tokens [96, 112) past the lcm boundary and must NOT be cached.
|
||||
for i in range(6):
|
||||
assert (
|
||||
pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1])
|
||||
is not None
|
||||
), f"SWA hash {i} should be cached"
|
||||
assert pool.get_cached_block(req.block_hashes[6], kv_cache_group_ids=[1]) is None, (
|
||||
"SWA hash 6 spans tokens past the lcm boundary; should not be cached"
|
||||
)
|
||||
|
||||
|
||||
def test_block_lookup_cache_single_block_per_key():
|
||||
cache = BlockHashToBlockMap()
|
||||
key0 = BlockHashWithGroupId(b"hash0")
|
||||
|
||||
@@ -216,6 +216,7 @@ class BlockPool:
|
||||
num_full_blocks: int,
|
||||
block_size: int,
|
||||
kv_cache_group_id: int,
|
||||
block_mask: list[bool] | None = None,
|
||||
) -> None:
|
||||
"""Cache a list of full blocks for prefix caching.
|
||||
This function takes a list of blocks that will have their block hash
|
||||
@@ -233,11 +234,19 @@ class BlockPool:
|
||||
be cached after this function.
|
||||
block_size: Number of tokens in each block.
|
||||
kv_cache_group_id: The id of the KV cache group.
|
||||
block_mask: Optional mask aligned with
|
||||
``blocks[num_cached_blocks:num_full_blocks]``. When provided,
|
||||
blocks where the mask is False are skipped (treated like null
|
||||
blocks). Used by groups whose ``find_longest_cache_hit`` only
|
||||
consults a subset of blocks (e.g. SWA tail-window), so blocks
|
||||
that can never serve a hit stay out of the prefix-cache hash
|
||||
map.
|
||||
"""
|
||||
if num_cached_blocks >= num_full_blocks:
|
||||
return
|
||||
new_full_blocks = blocks[num_cached_blocks:num_full_blocks]
|
||||
assert len(request.block_hashes) >= num_full_blocks
|
||||
assert block_mask is None or len(block_mask) == len(new_full_blocks)
|
||||
if block_size == self.hash_block_size:
|
||||
# Common case.
|
||||
block_hashes: BlockHashList = request.block_hashes
|
||||
@@ -256,10 +265,10 @@ class BlockPool:
|
||||
[] if self.enable_kv_cache_events else None
|
||||
)
|
||||
for i, blk in enumerate(new_full_blocks):
|
||||
# Some blocks may be null blocks when enabling sparse attention like
|
||||
# sliding window attention, or Mamba models with prefix-caching in
|
||||
# align mode. We skip null blocks here.
|
||||
if blk.is_null:
|
||||
# Some blocks may be null or masked out when enabling sparse attention
|
||||
# like sliding window attention, or Mamba models with prefix-caching
|
||||
# in align mode. We skip null blocks here.
|
||||
if blk.is_null or (block_mask is not None and not block_mask[i]):
|
||||
continue
|
||||
assert blk.block_hash is None
|
||||
block_hash = new_block_hashes[i]
|
||||
@@ -288,12 +297,14 @@ class BlockPool:
|
||||
# Generate extra keys for each block individually.
|
||||
# Each block may have different extra_keys (e.g., different MM
|
||||
# features, or cache_salt only for the first block).
|
||||
# Skip null blocks to match the length of new_hashes.
|
||||
# Skip null/masked-out blocks to match the length of new_hashes.
|
||||
extra_keys_list: list[tuple[Any, ...] | None] = []
|
||||
curr_mm_idx = 0
|
||||
for i in range(num_cached_blocks, num_full_blocks):
|
||||
if blocks[i].is_null:
|
||||
continue
|
||||
if block_mask is not None and not block_mask[i - num_cached_blocks]:
|
||||
continue
|
||||
block_start = i * block_size
|
||||
block_end = block_start + block_size
|
||||
extra_keys, curr_mm_idx = generate_block_hash_extra_keys(
|
||||
|
||||
@@ -484,6 +484,22 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
if any(gid in self.eagle_group_ids for gid in group_ids)
|
||||
}
|
||||
|
||||
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
|
||||
# Cache hits in this coordinator are always a multiple of
|
||||
# ``lcm_block_size`` tokens (see ``find_longest_cache_hit``). Within an
|
||||
# aligned region, SWA groups only consult a subset of blocks per
|
||||
# ``lcm_block_size``-segment so the unused blocks also stay out of the
|
||||
# prefix-cache hash map.
|
||||
num_computed_tokens = (
|
||||
num_computed_tokens // self.lcm_block_size * self.lcm_block_size
|
||||
)
|
||||
for manager in self.single_type_managers:
|
||||
manager.cache_blocks(
|
||||
request,
|
||||
num_computed_tokens,
|
||||
alignment_tokens=self.lcm_block_size,
|
||||
)
|
||||
|
||||
def find_longest_cache_hit(
|
||||
self,
|
||||
block_hashes: list[BlockHash],
|
||||
|
||||
@@ -275,7 +275,12 @@ class SingleTypeKVCacheManager(ABC):
|
||||
self.new_block_ids = []
|
||||
return ids
|
||||
|
||||
def cache_blocks(self, request: Request, num_tokens: int) -> None:
|
||||
def cache_blocks(
|
||||
self,
|
||||
request: Request,
|
||||
num_tokens: int,
|
||||
alignment_tokens: int | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Cache the blocks for the request.
|
||||
|
||||
@@ -283,6 +288,12 @@ class SingleTypeKVCacheManager(ABC):
|
||||
request: The request.
|
||||
num_tokens: The total number of tokens that need to be cached
|
||||
(including tokens that are already cached).
|
||||
alignment_tokens: The cache-hit alignment (in tokens) used by the
|
||||
coordinator's ``find_longest_cache_hit``. When greater than
|
||||
this group's ``block_size``, managers whose hit logic only
|
||||
returns a subset of blocks per alignment-aligned segment
|
||||
(SWA) skip the rest since they can never participate in a
|
||||
future cache hit.
|
||||
"""
|
||||
num_cached_blocks = self.num_cached_block.get(request.request_id, 0)
|
||||
num_full_blocks = num_tokens // self.block_size
|
||||
@@ -290,6 +301,13 @@ class SingleTypeKVCacheManager(ABC):
|
||||
if num_cached_blocks >= num_full_blocks:
|
||||
return
|
||||
|
||||
# Fast path: when the coordinator imposes no alignment constraint
|
||||
if alignment_tokens is None or alignment_tokens <= self.block_size:
|
||||
block_mask = None
|
||||
else:
|
||||
block_mask = self._cache_block_mask(
|
||||
num_cached_blocks, num_full_blocks, alignment_tokens
|
||||
)
|
||||
self.block_pool.cache_full_blocks(
|
||||
request=request,
|
||||
blocks=self.req_to_blocks[request.request_id],
|
||||
@@ -297,10 +315,26 @@ class SingleTypeKVCacheManager(ABC):
|
||||
num_full_blocks=num_full_blocks,
|
||||
block_size=self.block_size,
|
||||
kv_cache_group_id=self.kv_cache_group_id,
|
||||
block_mask=block_mask,
|
||||
)
|
||||
|
||||
self.num_cached_block[request.request_id] = num_full_blocks
|
||||
|
||||
def _cache_block_mask(
|
||||
self,
|
||||
num_cached_blocks: int,
|
||||
num_full_blocks: int,
|
||||
alignment_tokens: int,
|
||||
) -> list[bool] | None:
|
||||
"""Per-block mask for ``cache_full_blocks``. ``None`` means cache
|
||||
every (non-null) block — the default for full attention.
|
||||
|
||||
Subclasses with sparse hit semantics (SWA) override this to skip
|
||||
blocks that can never serve a hit at any alignment-aligned prefix
|
||||
length.
|
||||
"""
|
||||
return None
|
||||
|
||||
def free(self, request_id: str) -> None:
|
||||
"""
|
||||
Free the blocks for the request.
|
||||
@@ -604,6 +638,19 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
computed.pop()
|
||||
return computed_blocks
|
||||
|
||||
def _cache_block_mask(
|
||||
self, num_cached_blocks: int, num_full_blocks: int, alignment_tokens: int
|
||||
) -> list[bool] | None:
|
||||
assert alignment_tokens > self.block_size
|
||||
per_segment = alignment_tokens // self.block_size
|
||||
tail = cdiv(self.sliding_window - 1, self.block_size)
|
||||
if tail >= per_segment:
|
||||
return None
|
||||
skip = per_segment - tail
|
||||
return [
|
||||
i % per_segment >= skip for i in range(num_cached_blocks, num_full_blocks)
|
||||
]
|
||||
|
||||
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
|
||||
"""
|
||||
Get the number of tokens that will be skipped for attention computation.
|
||||
@@ -1050,9 +1097,14 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
"""
|
||||
return num_computed_tokens - 1
|
||||
|
||||
def cache_blocks(self, request: Request, num_tokens: int) -> None:
|
||||
def cache_blocks(
|
||||
self,
|
||||
request: Request,
|
||||
num_tokens: int,
|
||||
alignment_tokens: int | None = None,
|
||||
) -> None:
|
||||
num_cached_blocks_before = self.num_cached_block.get(request.request_id, 0)
|
||||
super().cache_blocks(request, num_tokens)
|
||||
super().cache_blocks(request, num_tokens, alignment_tokens=alignment_tokens)
|
||||
num_cached_blocks_after = self.num_cached_block.get(request.request_id, 0)
|
||||
if num_cached_blocks_after > num_cached_blocks_before:
|
||||
for block in self.req_to_blocks[request.request_id][
|
||||
@@ -1081,7 +1133,12 @@ class CrossAttentionManager(SingleTypeKVCacheManager):
|
||||
# requests, so `new_computed_blocks` should always be empty.
|
||||
assert len(new_computed_blocks) == 0
|
||||
|
||||
def cache_blocks(self, request: Request, num_tokens: int) -> None:
|
||||
def cache_blocks(
|
||||
self,
|
||||
request: Request,
|
||||
num_tokens: int,
|
||||
alignment_tokens: int | None = None,
|
||||
) -> None:
|
||||
# We do not cache blocks for cross-attention to be shared between
|
||||
# requests, so this method is not relevant.
|
||||
raise ValueError("Should not be called as prefix caching is disabled.")
|
||||
|
||||
Reference in New Issue
Block a user