[Core][DSV4] Skip caching SWA blocks that can never serve a prefix-cache hit (#42258)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Yifan Qiao
2026-05-15 15:59:18 +08:00
committed by GitHub
parent 31fa757cf9
commit 4b364f810e
4 changed files with 232 additions and 9 deletions
+139
View File
@@ -2528,6 +2528,145 @@ def test_different_block_size():
assert num_computed_tokens == 4 * 16
def test_hybrid_cache_blocks_swa_tail_window_only():
"""Within each lcm-aligned segment, SWA's ``find_longest_cache_hit`` only
returns the trailing ``ceil((sliding_window - 1) / block_size)`` blocks
(its right-to-left scan stops once a contiguous match is found). Blocks
earlier in the segment can never serve a hit, so
``HybridKVCacheCoordinator.cache_blocks`` should skip them rather than
polluting the prefix-cache hash map."""
block_size = 8
# Full attn block_size=32, SWA block_size=8, sw=8 -> lcm=32.
# tail = ceil(7/8) = 1; per_segment = 32/8 = 4.
# Per-segment template = [F, F, F, T]; only the last SWA block in each
# 32-token segment ends up in the prefix-cache hash map.
kv_cache_config = KVCacheConfig(
num_blocks=100,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["layer1"],
FullAttentionSpec(
block_size=4 * block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float16,
),
),
KVCacheGroupSpec(
["layer2"],
SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=block_size,
),
),
],
)
manager = KVCacheManager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
)
# 8 hash-blocks of 8 tokens (64 tokens, two lcm-aligned segments).
token_ids = [i for i in range(8) for _ in range(block_size)]
req = make_request("0", token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req)
blocks = manager.allocate_slots(
req,
8 * block_size,
len(computed_blocks.blocks[0]) * block_size,
computed_blocks,
)
assert blocks is not None
assert len(req.block_hashes) == 8
pool = manager.block_pool
# SWA group_id=1: only hash 3 and hash 7 (the last block of each
# 32-token segment) should be cached. Hashes 0,1,2,4,5,6 cannot serve
# a hit at any lcm-aligned length, so they must NOT be cached.
expected_cached = {3, 7}
for i in range(8):
cached = pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1])
if i in expected_cached:
assert cached is not None, f"SWA hash {i} should be cached"
else:
assert cached is None, (
f"SWA hash {i} cannot serve any lcm-aligned hit; should not be cached"
)
def test_hybrid_cache_blocks_clamped_to_lcm():
"""HybridKVCacheCoordinator.cache_blocks() clamps to lcm_block_size.
Chunks past the last lcm-aligned boundary can never participate in a
cache hit (find_longest_cache_hit always returns lcm-aligned hits), so
caching them only pollutes the prefix-cache hash map and keeps blocks
on the LRU list that could otherwise return to the free pool."""
block_size = 16
# Full attn block_size=32, SWA block_size=16 -> lcm=32.
kv_cache_config = KVCacheConfig(
num_blocks=100,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["layer1"],
FullAttentionSpec(
block_size=block_size * 2,
num_kv_heads=1,
head_size=1,
dtype=torch.float16,
),
),
KVCacheGroupSpec(
["layer2"],
SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=2 * block_size,
),
),
],
)
manager = KVCacheManager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
)
# 7 hash-blocks of 16 tokens (112 tokens). With lcm=32 the clamp truncates
# to 96 tokens — SWA caches 6 hashes, full-attn caches 3.
token_ids = [i for i in range(7) for _ in range(block_size)]
req = make_request("0", token_ids, block_size, sha256)
computed_blocks, _ = manager.get_computed_blocks(req)
blocks = manager.allocate_slots(
req,
7 * block_size,
len(computed_blocks.blocks[0]) * block_size,
computed_blocks,
)
assert blocks is not None
assert len(req.block_hashes) == 7
pool = manager.block_pool
# SWA group_id=1: hashes 0..5 cached (6 blocks * 16 tokens = 96), hash 6
# spans tokens [96, 112) past the lcm boundary and must NOT be cached.
for i in range(6):
assert (
pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1])
is not None
), f"SWA hash {i} should be cached"
assert pool.get_cached_block(req.block_hashes[6], kv_cache_group_ids=[1]) is None, (
"SWA hash 6 spans tokens past the lcm boundary; should not be cached"
)
def test_block_lookup_cache_single_block_per_key():
cache = BlockHashToBlockMap()
key0 = BlockHashWithGroupId(b"hash0")