Apply LRU policy only to proper cache entries (#42656)

Signed-off-by: Stanislaw Wozniak <stw@zurich.ibm.com>
This commit is contained in:
Stan Wozniak
2026-06-16 21:49:15 +00:00
committed by GitHub
parent 9d4dc4ca2f
commit 520828789c
3 changed files with 29 additions and 56 deletions
+12 -10
View File
@@ -291,13 +291,12 @@ def test_prefill(hash_fn):
# All blocks should be available.
assert free_block_queue.num_free_blocks == 10
# The order should be
# [partial without hashes from req1 and req0 (5, 4) - prepended for immediate reuse]
# [unallocated (6, 7, 8, 9, 10)]
# [unique_req0 (4)]
# [unique_req1 (5)]
# [common (3, 2, 1)]
assert [
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
] == [6, 7, 8, 9, 10, 4, 5, 3, 2, 1]
] == [5, 4, 6, 7, 8, 9, 10, 3, 2, 1]
# Cache hit in the common prefix when the original block is already free.
# Incomplete 1 block (6 tokens)
@@ -311,7 +310,7 @@ def test_prefill(hash_fn):
blocks = manager.allocate_slots(
req2, num_new_tokens, len(computed_blocks.blocks[0]) * 16, computed_blocks
)
assert blocks is not None and blocks.get_block_ids() == ([6],)
assert blocks is not None and blocks.get_block_ids() == ([5],) # reuse partial [5]
# Although we only have 6 free blocks, we have 8 blocks in
# the free block queue due to lazy removal.
@@ -331,7 +330,7 @@ def test_prefill(hash_fn):
)
# This block ID order also checks the eviction order.
assert blocks is not None and blocks.get_block_ids() == (
[7, 8, 9, 10, 4, 5, 6, 3, 2, 1],
[5, 4, 6, 7, 8, 9, 10, 3, 2, 1],
)
assert free_block_queue.num_free_blocks == 0
@@ -1216,13 +1215,12 @@ def test_prefill_plp():
# All blocks should be available.
assert manager.block_pool.free_block_queue.num_free_blocks == 10
# The order should be
# [partial without hashes from req1 and req0 (5, 4) - prepended for immediate reuse]
# [unallocated (6, 7, 8, 9, 10)]
# [unique_req0 (4)]
# [unique_req1 (5)]
# [common (3, 2, 1)]
assert [
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
] == [6, 7, 8, 9, 10, 4, 5, 3, 2, 1]
] == [5, 4, 6, 7, 8, 9, 10, 3, 2, 1]
# Request #2 is a prompt-logprobs request:
# NO cache hit in the common prefix; duplicates request #0 cached blocks
@@ -1351,11 +1349,15 @@ def test_evict():
assert manager.block_pool.free_block_queue.num_free_blocks == 1
manager.free(req0)
# partial blocks (without hash) at head, other at tail (LRU policy):
assert [
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
] == [6, 10, 5, 4, 3, 2, 1]
manager.free(req1)
assert manager.block_pool.free_block_queue.num_free_blocks == 10
assert [
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
] == [10, 6, 5, 4, 3, 2, 1, 9, 8, 7]
] == [6, 10, 5, 4, 3, 2, 1, 9, 8, 7]
# Touch the first 2 blocks.
req2 = make_request("2", list(range(2 * 16 + 3)), block_size, sha256)
@@ -1365,7 +1367,7 @@ def test_evict():
blocks = manager.allocate_slots(
req2, 3, len(computed_blocks.blocks[0]) * 16, computed_blocks
)
assert blocks is not None and blocks.get_block_ids() == ([10],)
assert blocks is not None and blocks.get_block_ids() == ([6],)
assert manager.block_pool.free_block_queue.num_free_blocks == 7