Apply LRU policy only to proper cache entries (#42656)
Signed-off-by: Stanislaw Wozniak <stw@zurich.ibm.com>
This commit is contained in:
@@ -291,13 +291,12 @@ def test_prefill(hash_fn):
|
||||
# All blocks should be available.
|
||||
assert free_block_queue.num_free_blocks == 10
|
||||
# The order should be
|
||||
# [partial without hashes from req1 and req0 (5, 4) - prepended for immediate reuse]
|
||||
# [unallocated (6, 7, 8, 9, 10)]
|
||||
# [unique_req0 (4)]
|
||||
# [unique_req1 (5)]
|
||||
# [common (3, 2, 1)]
|
||||
assert [
|
||||
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
|
||||
] == [6, 7, 8, 9, 10, 4, 5, 3, 2, 1]
|
||||
] == [5, 4, 6, 7, 8, 9, 10, 3, 2, 1]
|
||||
|
||||
# Cache hit in the common prefix when the original block is already free.
|
||||
# Incomplete 1 block (6 tokens)
|
||||
@@ -311,7 +310,7 @@ def test_prefill(hash_fn):
|
||||
blocks = manager.allocate_slots(
|
||||
req2, num_new_tokens, len(computed_blocks.blocks[0]) * 16, computed_blocks
|
||||
)
|
||||
assert blocks is not None and blocks.get_block_ids() == ([6],)
|
||||
assert blocks is not None and blocks.get_block_ids() == ([5],) # reuse partial [5]
|
||||
|
||||
# Although we only have 6 free blocks, we have 8 blocks in
|
||||
# the free block queue due to lazy removal.
|
||||
@@ -331,7 +330,7 @@ def test_prefill(hash_fn):
|
||||
)
|
||||
# This block ID order also checks the eviction order.
|
||||
assert blocks is not None and blocks.get_block_ids() == (
|
||||
[7, 8, 9, 10, 4, 5, 6, 3, 2, 1],
|
||||
[5, 4, 6, 7, 8, 9, 10, 3, 2, 1],
|
||||
)
|
||||
|
||||
assert free_block_queue.num_free_blocks == 0
|
||||
@@ -1216,13 +1215,12 @@ def test_prefill_plp():
|
||||
# All blocks should be available.
|
||||
assert manager.block_pool.free_block_queue.num_free_blocks == 10
|
||||
# The order should be
|
||||
# [partial without hashes from req1 and req0 (5, 4) - prepended for immediate reuse]
|
||||
# [unallocated (6, 7, 8, 9, 10)]
|
||||
# [unique_req0 (4)]
|
||||
# [unique_req1 (5)]
|
||||
# [common (3, 2, 1)]
|
||||
assert [
|
||||
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
|
||||
] == [6, 7, 8, 9, 10, 4, 5, 3, 2, 1]
|
||||
] == [5, 4, 6, 7, 8, 9, 10, 3, 2, 1]
|
||||
|
||||
# Request #2 is a prompt-logprobs request:
|
||||
# NO cache hit in the common prefix; duplicates request #0 cached blocks
|
||||
@@ -1351,11 +1349,15 @@ def test_evict():
|
||||
assert manager.block_pool.free_block_queue.num_free_blocks == 1
|
||||
|
||||
manager.free(req0)
|
||||
# partial blocks (without hash) at head, other at tail (LRU policy):
|
||||
assert [
|
||||
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
|
||||
] == [6, 10, 5, 4, 3, 2, 1]
|
||||
manager.free(req1)
|
||||
assert manager.block_pool.free_block_queue.num_free_blocks == 10
|
||||
assert [
|
||||
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
|
||||
] == [10, 6, 5, 4, 3, 2, 1, 9, 8, 7]
|
||||
] == [6, 10, 5, 4, 3, 2, 1, 9, 8, 7]
|
||||
|
||||
# Touch the first 2 blocks.
|
||||
req2 = make_request("2", list(range(2 * 16 + 3)), block_size, sha256)
|
||||
@@ -1365,7 +1367,7 @@ def test_evict():
|
||||
blocks = manager.allocate_slots(
|
||||
req2, 3, len(computed_blocks.blocks[0]) * 16, computed_blocks
|
||||
)
|
||||
assert blocks is not None and blocks.get_block_ids() == ([10],)
|
||||
assert blocks is not None and blocks.get_block_ids() == ([6],)
|
||||
assert manager.block_pool.free_block_queue.num_free_blocks == 7
|
||||
|
||||
|
||||
|
||||
+14
-15
@@ -416,29 +416,28 @@ class BlockPool:
|
||||
if self.metrics_collector:
|
||||
self.metrics_collector.on_block_accessed(block)
|
||||
|
||||
def free_blocks(
|
||||
self, ordered_blocks: Iterable[KVCacheBlock], prepend: bool = False
|
||||
) -> None:
|
||||
def free_blocks(self, ordered_blocks: Iterable[KVCacheBlock]) -> None:
|
||||
"""Free a list of blocks. The blocks should be ordered by their
|
||||
eviction priority, where the first block will be evicted first.
|
||||
|
||||
Args:
|
||||
ordered_blocks: A list of blocks to free ordered by their eviction
|
||||
priority.
|
||||
prepend: Whether to put newly-free blocks at the front of the free
|
||||
queue to be prioritized for reuse.
|
||||
"""
|
||||
# Materialize the iterable to allow multiple passes.
|
||||
blocks_list = list(ordered_blocks)
|
||||
for block in blocks_list:
|
||||
# Identify blocks with hash (LRU cache) and without it (will never match in APC)
|
||||
blocks_with_hash = []
|
||||
blocks_without_hash = []
|
||||
for block in ordered_blocks:
|
||||
block.ref_cnt -= 1
|
||||
freed_blocks = [
|
||||
block for block in blocks_list if block.ref_cnt == 0 and not block.is_null
|
||||
]
|
||||
if prepend:
|
||||
self.free_block_queue.prepend_n(freed_blocks)
|
||||
else:
|
||||
self.free_block_queue.append_n(freed_blocks)
|
||||
if block.ref_cnt == 0 and not block.is_null:
|
||||
if block.block_hash is None:
|
||||
blocks_without_hash.append(block)
|
||||
else:
|
||||
blocks_with_hash.append(block)
|
||||
|
||||
# Blocks without hash always get evicted first - prepend them last to the tail
|
||||
self.free_block_queue.prepend_n(blocks_without_hash)
|
||||
self.free_block_queue.append_n(blocks_with_hash)
|
||||
|
||||
def evict_blocks(self, block_ids: set[int]) -> None:
|
||||
"""evict blocks from the prefix cache by their block IDs.
|
||||
|
||||
@@ -504,12 +504,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
# range), so we must cap to the number of blocks that currently exist for
|
||||
# this request.
|
||||
num_skipped_blocks = min(num_skipped_blocks, len(blocks))
|
||||
|
||||
# Reuse skipped local blocks in order:
|
||||
# scratch blocks: no prefix-cache value, reuse first.
|
||||
# cached blocks: reusable prefix-cache value, reuse last.
|
||||
removed_cached_blocks: list[KVCacheBlock] = []
|
||||
removed_uncached_blocks: list[KVCacheBlock] = []
|
||||
removed_blocks: list[KVCacheBlock] = []
|
||||
# Because the block starts from index 0, the num_skipped_block-th block
|
||||
# corresponds to index num_skipped_blocks - 1.
|
||||
for i in range(num_skipped_blocks - 1, -1, -1):
|
||||
@@ -518,16 +513,9 @@ class SingleTypeKVCacheManager(ABC):
|
||||
# should also have been set to null blocks by the previous calls
|
||||
# to this function.
|
||||
break
|
||||
if blocks[i].block_hash is None:
|
||||
removed_uncached_blocks.append(blocks[i])
|
||||
else:
|
||||
removed_cached_blocks.append(blocks[i])
|
||||
removed_blocks.append(blocks[i])
|
||||
blocks[i] = self._null_block
|
||||
# `prepend=True` makes uncached scratch blocks the next allocation
|
||||
# candidates, while cached blocks stay behind them as best-effort
|
||||
# prefix-cache entries.
|
||||
self.block_pool.free_blocks(removed_cached_blocks)
|
||||
self.block_pool.free_blocks(removed_uncached_blocks, prepend=True)
|
||||
self.block_pool.free_blocks(removed_blocks)
|
||||
|
||||
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
|
||||
"""
|
||||
@@ -777,22 +765,6 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
|
||||
return mask
|
||||
|
||||
def free(self, request_id: str) -> None:
|
||||
# similar to remove_skipped_blocks(), prepend the uncached blocks
|
||||
# and append the cached blocks to the free queue
|
||||
req_blocks = self.req_to_blocks.pop(request_id, [])
|
||||
if req_blocks:
|
||||
cached_blocks: list[KVCacheBlock] = []
|
||||
uncached_blocks: list[KVCacheBlock] = []
|
||||
for block in reversed(req_blocks):
|
||||
if block.block_hash is None:
|
||||
uncached_blocks.append(block)
|
||||
else:
|
||||
cached_blocks.append(block)
|
||||
self.block_pool.free_blocks(cached_blocks)
|
||||
self.block_pool.free_blocks(uncached_blocks, prepend=True)
|
||||
self.num_cached_block.pop(request_id, None)
|
||||
|
||||
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
|
||||
"""
|
||||
Get the number of tokens that will be skipped for attention computation.
|
||||
|
||||
Reference in New Issue
Block a user