Apply LRU policy only to proper cache entries (#42656)

Signed-off-by: Stanislaw Wozniak <stw@zurich.ibm.com>
This commit is contained in:
Stan Wozniak
2026-06-16 21:49:15 +00:00
committed by GitHub
parent 9d4dc4ca2f
commit 520828789c
3 changed files with 29 additions and 56 deletions
+12 -10
View File
@@ -291,13 +291,12 @@ def test_prefill(hash_fn):
# All blocks should be available.
assert free_block_queue.num_free_blocks == 10
# The order should be
# [partial without hashes from req1 and req0 (5, 4) - prepended for immediate reuse]
# [unallocated (6, 7, 8, 9, 10)]
# [unique_req0 (4)]
# [unique_req1 (5)]
# [common (3, 2, 1)]
assert [
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
] == [6, 7, 8, 9, 10, 4, 5, 3, 2, 1]
] == [5, 4, 6, 7, 8, 9, 10, 3, 2, 1]
# Cache hit in the common prefix when the original block is already free.
# Incomplete 1 block (6 tokens)
@@ -311,7 +310,7 @@ def test_prefill(hash_fn):
blocks = manager.allocate_slots(
req2, num_new_tokens, len(computed_blocks.blocks[0]) * 16, computed_blocks
)
assert blocks is not None and blocks.get_block_ids() == ([6],)
assert blocks is not None and blocks.get_block_ids() == ([5],) # reuse partial [5]
# Although we only have 6 free blocks, we have 8 blocks in
# the free block queue due to lazy removal.
@@ -331,7 +330,7 @@ def test_prefill(hash_fn):
)
# This block ID order also checks the eviction order.
assert blocks is not None and blocks.get_block_ids() == (
[7, 8, 9, 10, 4, 5, 6, 3, 2, 1],
[5, 4, 6, 7, 8, 9, 10, 3, 2, 1],
)
assert free_block_queue.num_free_blocks == 0
@@ -1216,13 +1215,12 @@ def test_prefill_plp():
# All blocks should be available.
assert manager.block_pool.free_block_queue.num_free_blocks == 10
# The order should be
# [partial without hashes from req1 and req0 (5, 4) - prepended for immediate reuse]
# [unallocated (6, 7, 8, 9, 10)]
# [unique_req0 (4)]
# [unique_req1 (5)]
# [common (3, 2, 1)]
assert [
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
] == [6, 7, 8, 9, 10, 4, 5, 3, 2, 1]
] == [5, 4, 6, 7, 8, 9, 10, 3, 2, 1]
# Request #2 is a prompt-logprobs request:
# NO cache hit in the common prefix; duplicates request #0 cached blocks
@@ -1351,11 +1349,15 @@ def test_evict():
assert manager.block_pool.free_block_queue.num_free_blocks == 1
manager.free(req0)
# partial blocks (without hash) at head, other at tail (LRU policy):
assert [
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
] == [6, 10, 5, 4, 3, 2, 1]
manager.free(req1)
assert manager.block_pool.free_block_queue.num_free_blocks == 10
assert [
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
] == [10, 6, 5, 4, 3, 2, 1, 9, 8, 7]
] == [6, 10, 5, 4, 3, 2, 1, 9, 8, 7]
# Touch the first 2 blocks.
req2 = make_request("2", list(range(2 * 16 + 3)), block_size, sha256)
@@ -1365,7 +1367,7 @@ def test_evict():
blocks = manager.allocate_slots(
req2, 3, len(computed_blocks.blocks[0]) * 16, computed_blocks
)
assert blocks is not None and blocks.get_block_ids() == ([10],)
assert blocks is not None and blocks.get_block_ids() == ([6],)
assert manager.block_pool.free_block_queue.num_free_blocks == 7
+14 -15
View File
@@ -416,29 +416,28 @@ class BlockPool:
if self.metrics_collector:
self.metrics_collector.on_block_accessed(block)
def free_blocks(
self, ordered_blocks: Iterable[KVCacheBlock], prepend: bool = False
) -> None:
def free_blocks(self, ordered_blocks: Iterable[KVCacheBlock]) -> None:
"""Free a list of blocks. The blocks should be ordered by their
eviction priority, where the first block will be evicted first.
Args:
ordered_blocks: A list of blocks to free ordered by their eviction
priority.
prepend: Whether to put newly-free blocks at the front of the free
queue to be prioritized for reuse.
"""
# Materialize the iterable to allow multiple passes.
blocks_list = list(ordered_blocks)
for block in blocks_list:
# Identify blocks with hash (LRU cache) and without it (will never match in APC)
blocks_with_hash = []
blocks_without_hash = []
for block in ordered_blocks:
block.ref_cnt -= 1
freed_blocks = [
block for block in blocks_list if block.ref_cnt == 0 and not block.is_null
]
if prepend:
self.free_block_queue.prepend_n(freed_blocks)
else:
self.free_block_queue.append_n(freed_blocks)
if block.ref_cnt == 0 and not block.is_null:
if block.block_hash is None:
blocks_without_hash.append(block)
else:
blocks_with_hash.append(block)
# Blocks without hash always get evicted first - prepend them last to the tail
self.free_block_queue.prepend_n(blocks_without_hash)
self.free_block_queue.append_n(blocks_with_hash)
def evict_blocks(self, block_ids: set[int]) -> None:
"""evict blocks from the prefix cache by their block IDs.
+3 -31
View File
@@ -504,12 +504,7 @@ class SingleTypeKVCacheManager(ABC):
# range), so we must cap to the number of blocks that currently exist for
# this request.
num_skipped_blocks = min(num_skipped_blocks, len(blocks))
# Reuse skipped local blocks in order:
# scratch blocks: no prefix-cache value, reuse first.
# cached blocks: reusable prefix-cache value, reuse last.
removed_cached_blocks: list[KVCacheBlock] = []
removed_uncached_blocks: list[KVCacheBlock] = []
removed_blocks: list[KVCacheBlock] = []
# Because the block starts from index 0, the num_skipped_block-th block
# corresponds to index num_skipped_blocks - 1.
for i in range(num_skipped_blocks - 1, -1, -1):
@@ -518,16 +513,9 @@ class SingleTypeKVCacheManager(ABC):
# should also have been set to null blocks by the previous calls
# to this function.
break
if blocks[i].block_hash is None:
removed_uncached_blocks.append(blocks[i])
else:
removed_cached_blocks.append(blocks[i])
removed_blocks.append(blocks[i])
blocks[i] = self._null_block
# `prepend=True` makes uncached scratch blocks the next allocation
# candidates, while cached blocks stay behind them as best-effort
# prefix-cache entries.
self.block_pool.free_blocks(removed_cached_blocks)
self.block_pool.free_blocks(removed_uncached_blocks, prepend=True)
self.block_pool.free_blocks(removed_blocks)
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
"""
@@ -777,22 +765,6 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
return mask
def free(self, request_id: str) -> None:
# similar to remove_skipped_blocks(), prepend the uncached blocks
# and append the cached blocks to the free queue
req_blocks = self.req_to_blocks.pop(request_id, [])
if req_blocks:
cached_blocks: list[KVCacheBlock] = []
uncached_blocks: list[KVCacheBlock] = []
for block in reversed(req_blocks):
if block.block_hash is None:
uncached_blocks.append(block)
else:
cached_blocks.append(block)
self.block_pool.free_blocks(cached_blocks)
self.block_pool.free_blocks(uncached_blocks, prepend=True)
self.num_cached_block.pop(request_id, None)
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
"""
Get the number of tokens that will be skipped for attention computation.