[Core][DSV4] Skip caching SWA blocks that can never serve a prefix-cache hit (#42258)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
@@ -2528,6 +2528,145 @@ def test_different_block_size():
|
||||
assert num_computed_tokens == 4 * 16
|
||||
|
||||
|
||||
def test_hybrid_cache_blocks_swa_tail_window_only():
|
||||
"""Within each lcm-aligned segment, SWA's ``find_longest_cache_hit`` only
|
||||
returns the trailing ``ceil((sliding_window - 1) / block_size)`` blocks
|
||||
(its right-to-left scan stops once a contiguous match is found). Blocks
|
||||
earlier in the segment can never serve a hit, so
|
||||
``HybridKVCacheCoordinator.cache_blocks`` should skip them rather than
|
||||
polluting the prefix-cache hash map."""
|
||||
block_size = 8
|
||||
# Full attn block_size=32, SWA block_size=8, sw=8 -> lcm=32.
|
||||
# tail = ceil(7/8) = 1; per_segment = 32/8 = 4.
|
||||
# Per-segment template = [F, F, F, T]; only the last SWA block in each
|
||||
# 32-token segment ends up in the prefix-cache hash map.
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=4 * block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float16,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer2"],
|
||||
SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=block_size,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
|
||||
# 8 hash-blocks of 8 tokens (64 tokens, two lcm-aligned segments).
|
||||
token_ids = [i for i in range(8) for _ in range(block_size)]
|
||||
req = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req)
|
||||
blocks = manager.allocate_slots(
|
||||
req,
|
||||
8 * block_size,
|
||||
len(computed_blocks.blocks[0]) * block_size,
|
||||
computed_blocks,
|
||||
)
|
||||
assert blocks is not None
|
||||
assert len(req.block_hashes) == 8
|
||||
|
||||
pool = manager.block_pool
|
||||
# SWA group_id=1: only hash 3 and hash 7 (the last block of each
|
||||
# 32-token segment) should be cached. Hashes 0,1,2,4,5,6 cannot serve
|
||||
# a hit at any lcm-aligned length, so they must NOT be cached.
|
||||
expected_cached = {3, 7}
|
||||
for i in range(8):
|
||||
cached = pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1])
|
||||
if i in expected_cached:
|
||||
assert cached is not None, f"SWA hash {i} should be cached"
|
||||
else:
|
||||
assert cached is None, (
|
||||
f"SWA hash {i} cannot serve any lcm-aligned hit; should not be cached"
|
||||
)
|
||||
|
||||
|
||||
def test_hybrid_cache_blocks_clamped_to_lcm():
|
||||
"""HybridKVCacheCoordinator.cache_blocks() clamps to lcm_block_size.
|
||||
Chunks past the last lcm-aligned boundary can never participate in a
|
||||
cache hit (find_longest_cache_hit always returns lcm-aligned hits), so
|
||||
caching them only pollutes the prefix-cache hash map and keeps blocks
|
||||
on the LRU list that could otherwise return to the free pool."""
|
||||
block_size = 16
|
||||
# Full attn block_size=32, SWA block_size=16 -> lcm=32.
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size * 2,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float16,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer2"],
|
||||
SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=2 * block_size,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
|
||||
# 7 hash-blocks of 16 tokens (112 tokens). With lcm=32 the clamp truncates
|
||||
# to 96 tokens — SWA caches 6 hashes, full-attn caches 3.
|
||||
token_ids = [i for i in range(7) for _ in range(block_size)]
|
||||
req = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req)
|
||||
blocks = manager.allocate_slots(
|
||||
req,
|
||||
7 * block_size,
|
||||
len(computed_blocks.blocks[0]) * block_size,
|
||||
computed_blocks,
|
||||
)
|
||||
assert blocks is not None
|
||||
assert len(req.block_hashes) == 7
|
||||
|
||||
pool = manager.block_pool
|
||||
# SWA group_id=1: hashes 0..5 cached (6 blocks * 16 tokens = 96), hash 6
|
||||
# spans tokens [96, 112) past the lcm boundary and must NOT be cached.
|
||||
for i in range(6):
|
||||
assert (
|
||||
pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1])
|
||||
is not None
|
||||
), f"SWA hash {i} should be cached"
|
||||
assert pool.get_cached_block(req.block_hashes[6], kv_cache_group_ids=[1]) is None, (
|
||||
"SWA hash 6 spans tokens past the lcm boundary; should not be cached"
|
||||
)
|
||||
|
||||
|
||||
def test_block_lookup_cache_single_block_per_key():
|
||||
cache = BlockHashToBlockMap()
|
||||
key0 = BlockHashWithGroupId(b"hash0")
|
||||
|
||||
Reference in New Issue
Block a user