diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index b0be55bb49..0871a15d08 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -291,13 +291,12 @@ def test_prefill(hash_fn): # All blocks should be available. assert free_block_queue.num_free_blocks == 10 # The order should be + # [partial without hashes from req1 and req0 (5, 4) - prepended for immediate reuse] # [unallocated (6, 7, 8, 9, 10)] - # [unique_req0 (4)] - # [unique_req1 (5)] # [common (3, 2, 1)] assert [ b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks() - ] == [6, 7, 8, 9, 10, 4, 5, 3, 2, 1] + ] == [5, 4, 6, 7, 8, 9, 10, 3, 2, 1] # Cache hit in the common prefix when the original block is already free. # Incomplete 1 block (6 tokens) @@ -311,7 +310,7 @@ def test_prefill(hash_fn): blocks = manager.allocate_slots( req2, num_new_tokens, len(computed_blocks.blocks[0]) * 16, computed_blocks ) - assert blocks is not None and blocks.get_block_ids() == ([6],) + assert blocks is not None and blocks.get_block_ids() == ([5],) # reuse partial [5] # Although we only have 6 free blocks, we have 8 blocks in # the free block queue due to lazy removal. @@ -331,7 +330,7 @@ def test_prefill(hash_fn): ) # This block ID order also checks the eviction order. assert blocks is not None and blocks.get_block_ids() == ( - [7, 8, 9, 10, 4, 5, 6, 3, 2, 1], + [5, 4, 6, 7, 8, 9, 10, 3, 2, 1], ) assert free_block_queue.num_free_blocks == 0 @@ -1216,13 +1215,12 @@ def test_prefill_plp(): # All blocks should be available. assert manager.block_pool.free_block_queue.num_free_blocks == 10 # The order should be + # [partial without hashes from req1 and req0 (5, 4) - prepended for immediate reuse] # [unallocated (6, 7, 8, 9, 10)] - # [unique_req0 (4)] - # [unique_req1 (5)] # [common (3, 2, 1)] assert [ b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks() - ] == [6, 7, 8, 9, 10, 4, 5, 3, 2, 1] + ] == [5, 4, 6, 7, 8, 9, 10, 3, 2, 1] # Request #2 is a prompt-logprobs request: # NO cache hit in the common prefix; duplicates request #0 cached blocks @@ -1351,11 +1349,15 @@ def test_evict(): assert manager.block_pool.free_block_queue.num_free_blocks == 1 manager.free(req0) + # partial blocks (without hash) at head, other at tail (LRU policy): + assert [ + b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks() + ] == [6, 10, 5, 4, 3, 2, 1] manager.free(req1) assert manager.block_pool.free_block_queue.num_free_blocks == 10 assert [ b.block_id for b in manager.block_pool.free_block_queue.get_all_free_blocks() - ] == [10, 6, 5, 4, 3, 2, 1, 9, 8, 7] + ] == [6, 10, 5, 4, 3, 2, 1, 9, 8, 7] # Touch the first 2 blocks. req2 = make_request("2", list(range(2 * 16 + 3)), block_size, sha256) @@ -1365,7 +1367,7 @@ def test_evict(): blocks = manager.allocate_slots( req2, 3, len(computed_blocks.blocks[0]) * 16, computed_blocks ) - assert blocks is not None and blocks.get_block_ids() == ([10],) + assert blocks is not None and blocks.get_block_ids() == ([6],) assert manager.block_pool.free_block_queue.num_free_blocks == 7 diff --git a/vllm/v1/core/block_pool.py b/vllm/v1/core/block_pool.py index 4202f52708..e6bbba1466 100644 --- a/vllm/v1/core/block_pool.py +++ b/vllm/v1/core/block_pool.py @@ -416,29 +416,28 @@ class BlockPool: if self.metrics_collector: self.metrics_collector.on_block_accessed(block) - def free_blocks( - self, ordered_blocks: Iterable[KVCacheBlock], prepend: bool = False - ) -> None: + def free_blocks(self, ordered_blocks: Iterable[KVCacheBlock]) -> None: """Free a list of blocks. The blocks should be ordered by their eviction priority, where the first block will be evicted first. Args: ordered_blocks: A list of blocks to free ordered by their eviction priority. - prepend: Whether to put newly-free blocks at the front of the free - queue to be prioritized for reuse. """ - # Materialize the iterable to allow multiple passes. - blocks_list = list(ordered_blocks) - for block in blocks_list: + # Identify blocks with hash (LRU cache) and without it (will never match in APC) + blocks_with_hash = [] + blocks_without_hash = [] + for block in ordered_blocks: block.ref_cnt -= 1 - freed_blocks = [ - block for block in blocks_list if block.ref_cnt == 0 and not block.is_null - ] - if prepend: - self.free_block_queue.prepend_n(freed_blocks) - else: - self.free_block_queue.append_n(freed_blocks) + if block.ref_cnt == 0 and not block.is_null: + if block.block_hash is None: + blocks_without_hash.append(block) + else: + blocks_with_hash.append(block) + + # Blocks without hash always get evicted first - prepend them last to the tail + self.free_block_queue.prepend_n(blocks_without_hash) + self.free_block_queue.append_n(blocks_with_hash) def evict_blocks(self, block_ids: set[int]) -> None: """evict blocks from the prefix cache by their block IDs. diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index ad47e321e1..580b58814b 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -504,12 +504,7 @@ class SingleTypeKVCacheManager(ABC): # range), so we must cap to the number of blocks that currently exist for # this request. num_skipped_blocks = min(num_skipped_blocks, len(blocks)) - - # Reuse skipped local blocks in order: - # scratch blocks: no prefix-cache value, reuse first. - # cached blocks: reusable prefix-cache value, reuse last. - removed_cached_blocks: list[KVCacheBlock] = [] - removed_uncached_blocks: list[KVCacheBlock] = [] + removed_blocks: list[KVCacheBlock] = [] # Because the block starts from index 0, the num_skipped_block-th block # corresponds to index num_skipped_blocks - 1. for i in range(num_skipped_blocks - 1, -1, -1): @@ -518,16 +513,9 @@ class SingleTypeKVCacheManager(ABC): # should also have been set to null blocks by the previous calls # to this function. break - if blocks[i].block_hash is None: - removed_uncached_blocks.append(blocks[i]) - else: - removed_cached_blocks.append(blocks[i]) + removed_blocks.append(blocks[i]) blocks[i] = self._null_block - # `prepend=True` makes uncached scratch blocks the next allocation - # candidates, while cached blocks stay behind them as best-effort - # prefix-cache entries. - self.block_pool.free_blocks(removed_cached_blocks) - self.block_pool.free_blocks(removed_uncached_blocks, prepend=True) + self.block_pool.free_blocks(removed_blocks) def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: """ @@ -777,22 +765,6 @@ class SlidingWindowManager(SingleTypeKVCacheManager): return mask - def free(self, request_id: str) -> None: - # similar to remove_skipped_blocks(), prepend the uncached blocks - # and append the cached blocks to the free queue - req_blocks = self.req_to_blocks.pop(request_id, []) - if req_blocks: - cached_blocks: list[KVCacheBlock] = [] - uncached_blocks: list[KVCacheBlock] = [] - for block in reversed(req_blocks): - if block.block_hash is None: - uncached_blocks.append(block) - else: - cached_blocks.append(block) - self.block_pool.free_blocks(cached_blocks) - self.block_pool.free_blocks(uncached_blocks, prepend=True) - self.num_cached_block.pop(request_id, None) - def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: """ Get the number of tokens that will be skipped for attention computation.