Apply LRU policy only to proper cache entries (#42656)
Signed-off-by: Stanislaw Wozniak <stw@zurich.ibm.com>
This commit is contained in:
@@ -291,13 +291,12 @@ def test_prefill(hash_fn):
|
||||
# All blocks should be available.
|
||||
assert free_block_queue.num_free_blocks == 10
|
||||
# The order should be
|
||||
# [partial without hashes from req1 and req0 (5, 4) - prepended for immediate reuse]
|
||||
# [unallocated (6, 7, 8, 9, 10)]
|
||||
# [unique_req0 (4)]
|
||||
# [unique_req1 (5)]
|
||||
# [common (3, 2, 1)]
|
||||
assert [
|
||||
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
|
||||
] == [6, 7, 8, 9, 10, 4, 5, 3, 2, 1]
|
||||
] == [5, 4, 6, 7, 8, 9, 10, 3, 2, 1]
|
||||
|
||||
# Cache hit in the common prefix when the original block is already free.
|
||||
# Incomplete 1 block (6 tokens)
|
||||
@@ -311,7 +310,7 @@ def test_prefill(hash_fn):
|
||||
blocks = manager.allocate_slots(
|
||||
req2, num_new_tokens, len(computed_blocks.blocks[0]) * 16, computed_blocks
|
||||
)
|
||||
assert blocks is not None and blocks.get_block_ids() == ([6],)
|
||||
assert blocks is not None and blocks.get_block_ids() == ([5],) # reuse partial [5]
|
||||
|
||||
# Although we only have 6 free blocks, we have 8 blocks in
|
||||
# the free block queue due to lazy removal.
|
||||
@@ -331,7 +330,7 @@ def test_prefill(hash_fn):
|
||||
)
|
||||
# This block ID order also checks the eviction order.
|
||||
assert blocks is not None and blocks.get_block_ids() == (
|
||||
[7, 8, 9, 10, 4, 5, 6, 3, 2, 1],
|
||||
[5, 4, 6, 7, 8, 9, 10, 3, 2, 1],
|
||||
)
|
||||
|
||||
assert free_block_queue.num_free_blocks == 0
|
||||
@@ -1216,13 +1215,12 @@ def test_prefill_plp():
|
||||
# All blocks should be available.
|
||||
assert manager.block_pool.free_block_queue.num_free_blocks == 10
|
||||
# The order should be
|
||||
# [partial without hashes from req1 and req0 (5, 4) - prepended for immediate reuse]
|
||||
# [unallocated (6, 7, 8, 9, 10)]
|
||||
# [unique_req0 (4)]
|
||||
# [unique_req1 (5)]
|
||||
# [common (3, 2, 1)]
|
||||
assert [
|
||||
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
|
||||
] == [6, 7, 8, 9, 10, 4, 5, 3, 2, 1]
|
||||
] == [5, 4, 6, 7, 8, 9, 10, 3, 2, 1]
|
||||
|
||||
# Request #2 is a prompt-logprobs request:
|
||||
# NO cache hit in the common prefix; duplicates request #0 cached blocks
|
||||
@@ -1351,11 +1349,15 @@ def test_evict():
|
||||
assert manager.block_pool.free_block_queue.num_free_blocks == 1
|
||||
|
||||
manager.free(req0)
|
||||
# partial blocks (without hash) at head, other at tail (LRU policy):
|
||||
assert [
|
||||
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
|
||||
] == [6, 10, 5, 4, 3, 2, 1]
|
||||
manager.free(req1)
|
||||
assert manager.block_pool.free_block_queue.num_free_blocks == 10
|
||||
assert [
|
||||
b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks()
|
||||
] == [10, 6, 5, 4, 3, 2, 1, 9, 8, 7]
|
||||
] == [6, 10, 5, 4, 3, 2, 1, 9, 8, 7]
|
||||
|
||||
# Touch the first 2 blocks.
|
||||
req2 = make_request("2", list(range(2 * 16 + 3)), block_size, sha256)
|
||||
@@ -1365,7 +1367,7 @@ def test_evict():
|
||||
blocks = manager.allocate_slots(
|
||||
req2, 3, len(computed_blocks.blocks[0]) * 16, computed_blocks
|
||||
)
|
||||
assert blocks is not None and blocks.get_block_ids() == ([10],)
|
||||
assert blocks is not None and blocks.get_block_ids() == ([6],)
|
||||
assert manager.block_pool.free_block_queue.num_free_blocks == 7
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user