[Core] Support reset_prefix_cache (#12284)
This commit is contained in:
@@ -796,6 +796,44 @@ class TestPrefixCachingBlockAllocator:
|
||||
block_hashes=block_hashes_seq1)
|
||||
assert len(cached_blocks) == len(blocks_seq1) - num_evicted_blocks
|
||||
|
||||
# Test reset prefix cache
|
||||
@staticmethod
|
||||
@pytest.mark.parametrize("num_blocks", [10])
|
||||
@pytest.mark.parametrize("block_size", [16])
|
||||
def test_reset_prefix_cache(num_blocks: int, block_size: int):
|
||||
"""This test case simulates the case of resetting the prefix cache."""
|
||||
|
||||
allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
|
||||
block_size=block_size)
|
||||
token_ids = list(range(3 * block_size))
|
||||
|
||||
first_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
|
||||
block_size=block_size,
|
||||
token_ids=token_ids,
|
||||
allocator=allocator,
|
||||
)
|
||||
second_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
|
||||
block_size=block_size,
|
||||
token_ids=token_ids,
|
||||
allocator=allocator,
|
||||
)
|
||||
|
||||
# Free each block in the first chain.
|
||||
for block in first_chain:
|
||||
allocator.free(block)
|
||||
|
||||
# Failed to reset prefix cache because some blocks are not freed yet.
|
||||
assert not allocator.reset_prefix_cache()
|
||||
assert allocator.get_prefix_cache_hit_rate() > 0.0
|
||||
|
||||
# Free each block in the second chain.
|
||||
for block in second_chain:
|
||||
allocator.free(block)
|
||||
|
||||
# Reset prefix cache.
|
||||
assert allocator.reset_prefix_cache()
|
||||
assert allocator.get_prefix_cache_hit_rate() == 0.0
|
||||
|
||||
@staticmethod
|
||||
def create_immutable_chain(
|
||||
block_size: int,
|
||||
|
||||
@@ -587,3 +587,42 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
|
||||
assert {block.ref_cnt for block in block_part1[:3]} == {1}
|
||||
# Block 3-5 are free.
|
||||
assert {block.ref_cnt for block in block_part1[3:]} == {0}
|
||||
|
||||
|
||||
def test_reset_prefix_cache():
|
||||
manager = KVCacheManager(
|
||||
block_size=16,
|
||||
num_gpu_blocks=10,
|
||||
max_model_len=8192,
|
||||
sliding_window=None,
|
||||
enable_caching=True,
|
||||
num_preallocate_tokens=0,
|
||||
)
|
||||
|
||||
full_block_token_ids = [i for i in range(3) for _ in range(16)]
|
||||
unique_token_ids = [3] * 7
|
||||
all_token_ids = full_block_token_ids + unique_token_ids
|
||||
req0 = make_request("0", all_token_ids)
|
||||
blocks = manager.allocate_slots(req0, 55, [])
|
||||
assert [b.block_id for b in blocks] == [0, 1, 2, 3]
|
||||
|
||||
unique_token_ids = [4] * 7
|
||||
all_token_ids = full_block_token_ids + unique_token_ids
|
||||
req1 = make_request("1", all_token_ids)
|
||||
computed_blocks, _ = manager.get_computed_blocks(req1)
|
||||
assert len(req1.kv_block_hashes) == 3
|
||||
assert len(computed_blocks) == 3
|
||||
blocks = manager.allocate_slots(req1, 7, computed_blocks)
|
||||
assert [b.block_id for b in blocks] == [4]
|
||||
|
||||
# Failed to reset prefix cache because some blocks are not freed yet.
|
||||
assert not manager.reset_prefix_cache()
|
||||
assert manager.cached_block_hash_to_block
|
||||
|
||||
# Free the blocks.
|
||||
manager.free(req0)
|
||||
manager.free(req1)
|
||||
|
||||
assert manager.reset_prefix_cache()
|
||||
assert not manager.cached_block_hash_to_block
|
||||
assert all([blk.block_hash is None for blk in manager.block_pool])
|
||||
|
||||
Reference in New Issue
Block a user