From 6bc6f2d86d7800f878a4a13495b51a6e3b728c37 Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Mon, 22 Jun 2026 14:43:10 +0800 Subject: [PATCH] [1/N][Core] add partial prefix cache primitives (#45939) Signed-off-by: zjy0516 Co-authored-by: Yifan Qiao --- .../test_partial_prefix_cache_primitives.py | 460 ++++++++++++++++++ tests/v1/core/test_kv_cache_utils.py | 2 +- tests/v1/core/test_prefix_caching.py | 2 +- vllm/v1/core/block_pool.py | 242 ++++++++- vllm/v1/core/kv_cache_utils.py | 54 +- 5 files changed, 719 insertions(+), 41 deletions(-) create mode 100644 tests/v1/core/prefix_cache/test_partial_prefix_cache_primitives.py diff --git a/tests/v1/core/prefix_cache/test_partial_prefix_cache_primitives.py b/tests/v1/core/prefix_cache/test_partial_prefix_cache_primitives.py new file mode 100644 index 0000000000..225d87e867 --- /dev/null +++ b/tests/v1/core/prefix_cache/test_partial_prefix_cache_primitives.py @@ -0,0 +1,460 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from collections.abc import Callable + +import pytest + +import vllm.v1.core.kv_cache_utils as kv_cache_utils +from vllm.distributed.kv_events import BlockRemoved, BlockStored +from vllm.sampling_params import SamplingParams +from vllm.utils.hashing import sha256 +from vllm.v1.core.block_pool import BlockPool +from vllm.v1.core.kv_cache_utils import ( + BlockHash, + BlockHashListWithBlockSize, + KVCacheBlock, + get_request_block_hasher, + hash_block_tokens, + init_none_hash, +) +from vllm.v1.request import Request + +pytestmark = pytest.mark.cpu_test + + +@pytest.fixture(autouse=True) +def _auto_init_hash_fn(): + init_none_hash(sha256) + + +def make_request( + request_id: str, + prompt_token_ids: list[int], + hash_block_size: int, + hash_fn: Callable, +) -> Request: + sampling_params = SamplingParams(max_tokens=17) + sampling_params.update_from_generation_config({}, eos_token_id=100) + return Request( + request_id=request_id, + prompt_token_ids=prompt_token_ids, + sampling_params=sampling_params, + pooling_params=None, + block_hasher=get_request_block_hasher(hash_block_size, hash_fn), + ) + + +def boundary_hash(req: Request, hash_block_size: int, num_tokens: int) -> BlockHash: + # Every boundary at a hash_block_size multiple is just the fine-grained + # chain hash ending there. + return req.block_hashes[num_tokens // hash_block_size - 1] + + +def cache_full_block_and_partial_tail( + token_ids: list[int], + *, + enable_kv_cache_events: bool = False, +) -> tuple[BlockPool, Request, list[KVCacheBlock], BlockHash]: + hash_block_size = 2 + block_size = 6 + kv_cache_group_id = 0 + req = make_request("0", token_ids, hash_block_size, sha256) + pool = BlockPool( + num_gpu_blocks=3, + enable_caching=True, + hash_block_size=hash_block_size, + enable_kv_cache_events=enable_kv_cache_events, + ) + blocks = pool.get_new_blocks(2) + + pool.cache_full_blocks( + request=req, + blocks=blocks, + num_cached_blocks=0, + num_full_blocks=1, + block_size=block_size, + kv_cache_group_id=kv_cache_group_id, + ) + partial_hash = boundary_hash(req, hash_block_size, len(token_ids)) + assert pool.cache_partial_block( + request=req, + block=blocks[1], + num_tokens=len(token_ids), + kv_cache_group_id=kv_cache_group_id, + block_size=block_size, + ) + return pool, req, blocks, partial_hash + + +def test_boundary_hashes_reuse_fine_grained_chain(): + hash_block_size = 2 + block_size = 6 + token_ids = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4] + req = make_request("0", token_ids, hash_block_size, sha256) + + coarse = BlockHashListWithBlockSize(req.block_hashes, hash_block_size, block_size) + # The block_size=6 full-block hash is the fine hash at the 6-token boundary, + # not a concatenation of the three fine hashes inside the block. + assert coarse[0] == req.block_hashes[6 // hash_block_size - 1] + assert coarse[0] != BlockHash( + req.block_hashes[0] + req.block_hashes[1] + req.block_hashes[2] + ) + # A partial tail at 10 tokens is the fine hash at the 10-token boundary, + # which chains over the entire prefix. + tail_hash = boundary_hash(req, hash_block_size, 10) + assert tail_hash == req.block_hashes[4] + assert tail_hash == hash_block_tokens(sha256, req.block_hashes[3], token_ids[8:10]) + + +def test_cache_partial_block_kv_cache_events(): + hash_block_size = 4 + block_size = 12 + kv_cache_group_id = 2 + + pool = BlockPool( + num_gpu_blocks=2, + enable_caching=True, + hash_block_size=hash_block_size, + enable_kv_cache_events=True, + ) + req = make_request( + "req_partial_events", + prompt_token_ids=list(range(hash_block_size * 2)), + hash_block_size=hash_block_size, + hash_fn=sha256, + ) + + block = pool.get_new_blocks(1)[0] + partial_entry_hash = pool.cache_partial_block( + request=req, + block=block, + num_tokens=hash_block_size * 2, + kv_cache_group_id=kv_cache_group_id, + block_size=block_size, + ) + + events = pool.take_events() + assert len(events) == 1 + stored_event = events[0] + assert isinstance(stored_event, BlockStored) + assert partial_entry_hash is not None + assert stored_event.block_hashes == [ + kv_cache_utils.maybe_convert_block_hash(req.block_hashes[1]) + ] + assert stored_event.parent_block_hash == kv_cache_utils.maybe_convert_block_hash( + req.block_hashes[0] + ) + assert stored_event.token_ids == req.all_token_ids[hash_block_size:] + assert stored_event.block_size == 4 + assert stored_event.group_idx == kv_cache_group_id + + duplicate_entry_hash = pool.cache_partial_block( + request=req, + block=block, + num_tokens=hash_block_size * 2, + kv_cache_group_id=kv_cache_group_id, + block_size=block_size, + ) + assert duplicate_entry_hash == partial_entry_hash + assert pool.take_events() == [] + + pool.free_blocks([block]) + pool.get_new_blocks(1) + events = pool.take_events() + assert len(events) == 1 + removed_event = events[0] + assert isinstance(removed_event, BlockRemoved) + assert removed_event.block_hashes == stored_event.block_hashes + assert removed_event.group_idx == kv_cache_group_id + + +def test_partial_block_replacement_emits_remove_then_store_events(): + hash_block_size = 2 + block_size = 6 + kv_cache_group_id = 0 + req = make_request("0", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256) + pool = BlockPool( + num_gpu_blocks=3, + enable_caching=True, + hash_block_size=hash_block_size, + enable_kv_cache_events=True, + ) + blocks = pool.get_new_blocks(2) + + pool.cache_full_blocks( + request=req, + blocks=blocks, + num_cached_blocks=0, + num_full_blocks=1, + block_size=block_size, + kv_cache_group_id=kv_cache_group_id, + ) + partial_hash_8 = boundary_hash(req, hash_block_size, 8) + assert pool.cache_partial_block( + request=req, + block=blocks[1], + num_tokens=8, + kv_cache_group_id=kv_cache_group_id, + block_size=block_size, + ) + assert pool.get_cached_block(partial_hash_8, [kv_cache_group_id]) == [blocks[1]] + pool.take_events() + + req.append_output_token_ids([4, 4]) + partial_hash_10 = boundary_hash(req, hash_block_size, 10) + assert pool.cache_partial_block( + request=req, + block=blocks[1], + num_tokens=10, + kv_cache_group_id=kv_cache_group_id, + block_size=block_size, + ) + events = pool.take_events() + + assert len(events) == 2 + removed_event, stored_event = events + assert isinstance(removed_event, BlockRemoved) + assert removed_event.block_hashes == [ + kv_cache_utils.maybe_convert_block_hash(partial_hash_8) + ] + assert removed_event.group_idx == kv_cache_group_id + assert isinstance(stored_event, BlockStored) + assert stored_event.block_hashes == [ + kv_cache_utils.maybe_convert_block_hash(partial_hash_10) + ] + assert stored_event.parent_block_hash == kv_cache_utils.maybe_convert_block_hash( + boundary_hash(req, hash_block_size, 8) + ) + assert stored_event.token_ids == req.all_token_ids[8:10] + assert stored_event.block_size == hash_block_size + assert stored_event.group_idx == kv_cache_group_id + assert pool.get_cached_block(partial_hash_8, [kv_cache_group_id]) is None + assert pool.get_cached_block(partial_hash_10, [kv_cache_group_id]) == [blocks[1]] + + +def test_later_request_hits_cached_partial_tail(): + hash_block_size = 2 + block_size = 6 + kv_cache_group_id = 0 + cached_token_ids = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4] + req = make_request("0", cached_token_ids, hash_block_size, sha256) + pool = BlockPool( + num_gpu_blocks=3, + enable_caching=True, + hash_block_size=hash_block_size, + ) + blocks = pool.get_new_blocks(2) + + pool.cache_full_blocks( + request=req, + blocks=blocks, + num_cached_blocks=0, + num_full_blocks=1, + block_size=block_size, + kv_cache_group_id=kv_cache_group_id, + ) + partial_hash_10 = boundary_hash(req, hash_block_size, 10) + assert pool.cache_partial_block( + request=req, + block=blocks[1], + num_tokens=10, + kv_cache_group_id=kv_cache_group_id, + block_size=block_size, + ) + + replay = make_request("1", cached_token_ids, hash_block_size, sha256) + replay_hash_10 = boundary_hash(replay, hash_block_size, 10) + assert replay_hash_10 == partial_hash_10 + assert pool.get_cached_block(replay_hash_10, [kv_cache_group_id]) == [blocks[1]] + + extended = make_request("2", cached_token_ids + [10], hash_block_size, sha256) + extended_hash_10 = boundary_hash(extended, hash_block_size, 10) + assert extended_hash_10 == partial_hash_10 + assert pool.get_cached_block(extended_hash_10, [kv_cache_group_id]) == [blocks[1]] + + +def test_cache_partial_block_uses_fine_grained_boundary_hash(): + hash_block_size = 2 + block_size = 6 + kv_cache_group_id = 0 + token_ids = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4] + req = make_request("0", token_ids, hash_block_size, sha256) + pool = BlockPool( + num_gpu_blocks=3, + enable_caching=True, + hash_block_size=hash_block_size, + ) + blocks = pool.get_new_blocks(2) + + pool.cache_full_blocks( + request=req, + blocks=blocks, + num_cached_blocks=0, + num_full_blocks=1, + block_size=block_size, + kv_cache_group_id=kv_cache_group_id, + ) + + partial_entry_hash = pool.cache_partial_block( + request=req, + block=blocks[1], + num_tokens=10, + kv_cache_group_id=kv_cache_group_id, + block_size=block_size, + ) + # The partial entry is keyed by the fine-grained hash at the 10-token + # boundary, regardless of the owning group's block_size. + expected = boundary_hash(req, hash_block_size, 10) + assert partial_entry_hash == kv_cache_utils.make_block_hash_with_group_id( + expected, kv_cache_group_id + ) + assert pool.get_cached_block(expected, [kv_cache_group_id]) == [blocks[1]] + + +def test_cache_partial_block_requires_hash_boundary(): + hash_block_size = 2 + block_size = 4 + req = make_request("0", [0, 0, 1, 1], hash_block_size, sha256) + pool = BlockPool( + num_gpu_blocks=2, + enable_caching=True, + hash_block_size=hash_block_size, + ) + block = pool.get_new_blocks(1)[0] + + with pytest.raises(AssertionError): + pool.cache_partial_block( + request=req, + block=block, + num_tokens=3, + kv_cache_group_id=0, + block_size=block_size, + ) + + +def test_cache_partial_block_duplicate_checks_all_blocks_for_hash(): + hash_block_size = 2 + block_size = 4 + kv_cache_group_id = 0 + req = make_request("0", [0, 0, 1, 1], hash_block_size, sha256) + pool = BlockPool( + num_gpu_blocks=4, + enable_caching=True, + hash_block_size=hash_block_size, + ) + blocks = pool.get_new_blocks(2) + + first_entry_hash = pool.cache_partial_block( + request=req, + block=blocks[0], + num_tokens=2, + kv_cache_group_id=kv_cache_group_id, + block_size=block_size, + ) + second_entry_hash = pool.cache_partial_block( + request=req, + block=blocks[1], + num_tokens=2, + kv_cache_group_id=kv_cache_group_id, + block_size=block_size, + ) + assert first_entry_hash == second_entry_hash + + duplicate_entry_hash = pool.cache_partial_block( + request=req, + block=blocks[1], + num_tokens=2, + kv_cache_group_id=kv_cache_group_id, + block_size=block_size, + ) + assert duplicate_entry_hash == second_entry_hash + assert pool.cached_block_hashes_by_block == {} + + +def test_reset_prefix_cache_clears_partial_entry_metadata(): + pool, req, blocks, partial_hash_10 = cache_full_block_and_partial_tail( + [0, 0, 1, 1, 2, 2, 3, 3, 4, 4] + ) + full_hash = BlockHashListWithBlockSize(req.block_hashes, 2, 6)[0] + + assert pool.get_cached_block(full_hash, [0]) == [blocks[0]] + assert pool.get_cached_block(partial_hash_10, [0]) == [blocks[1]] + + pool.free_blocks(blocks) + assert pool.reset_prefix_cache() + + assert pool.get_cached_block(full_hash, [0]) is None + assert pool.get_cached_block(partial_hash_10, [0]) is None + assert pool.cached_block_hashes_by_block == {} + + +def test_evict_cached_block_removes_full_hash_and_partial_entry(): + pool, req, blocks, partial_hash_10 = cache_full_block_and_partial_tail( + [0, 0, 1, 1, 2, 2, 3, 3, 4, 4] + ) + full_hash = BlockHashListWithBlockSize(req.block_hashes, 2, 6)[0] + + assert pool.get_cached_block(full_hash, [0]) == [blocks[0]] + assert pool.get_cached_block(partial_hash_10, [0]) == [blocks[1]] + + pool.evict_blocks({blocks[0].block_id, blocks[1].block_id}) + + assert pool.get_cached_block(full_hash, [0]) is None + assert pool.get_cached_block(partial_hash_10, [0]) is None + assert pool.cached_block_hashes_by_block == {} + + +def test_partial_block_promotes_to_direct_full_block_hash(): + hash_block_size = 2 + block_size = 6 + kv_cache_group_id = 0 + token_ids = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4] + req = make_request("0", token_ids, hash_block_size, sha256) + pool = BlockPool( + num_gpu_blocks=3, + enable_caching=True, + hash_block_size=hash_block_size, + ) + blocks = pool.get_new_blocks(2) + + pool.cache_full_blocks( + request=req, + blocks=blocks, + num_cached_blocks=0, + num_full_blocks=1, + block_size=block_size, + kv_cache_group_id=kv_cache_group_id, + ) + partial_hash_10 = boundary_hash(req, hash_block_size, 10) + assert pool.cache_partial_block( + request=req, + block=blocks[1], + num_tokens=10, + kv_cache_group_id=kv_cache_group_id, + block_size=block_size, + ) + assert pool.get_cached_block(partial_hash_10, [kv_cache_group_id]) == [blocks[1]] + + req.append_output_token_ids([5, 5]) + full_hashes = BlockHashListWithBlockSize( + req.block_hashes, hash_block_size, block_size + ) + promoted_full_hash = full_hashes[1] + # The promoted full-block hash is the fine hash at the 12-token boundary, + # not a concatenation of the fine hashes inside the block. + assert promoted_full_hash == req.block_hashes[12 // hash_block_size - 1] + assert promoted_full_hash != BlockHash( + req.block_hashes[3] + req.block_hashes[4] + req.block_hashes[5] + ) + + pool.cache_full_blocks( + request=req, + blocks=blocks, + num_cached_blocks=1, + num_full_blocks=2, + block_size=block_size, + kv_cache_group_id=kv_cache_group_id, + ) + assert pool.get_cached_block(promoted_full_hash, [kv_cache_group_id]) == [blocks[1]] + assert pool.get_cached_block(partial_hash_10, [kv_cache_group_id]) is None diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 3f5b7a1243..95237fa272 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -225,7 +225,7 @@ def test_kv_cache_block(): # Test block hash setting and resetting block_hash = make_block_hash_with_group_id(BlockHash(b"abc"), 0) - block.block_hash = block_hash + block.set_block_hash(block_hash) assert block.block_hash == block_hash block.reset_hash() diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 0871a15d08..3e375b8720 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -2003,7 +2003,7 @@ def test_maybe_evict_cached_block(): assert len(pool.blocks) == len(block_hashes) # Manually add all blocks to cached_blocks for block, block_hash in zip(pool.blocks, block_hashes): - block.block_hash = block_hash + block.set_block_hash(block_hash) pool.cached_block_hash_to_block.insert(block_hash, block) block0, block1, block2, block3 = pool.blocks diff --git a/vllm/v1/core/block_pool.py b/vllm/v1/core/block_pool.py index e6bbba1466..81ac05f365 100644 --- a/vllm/v1/core/block_pool.py +++ b/vllm/v1/core/block_pool.py @@ -72,6 +72,20 @@ class BlockHashToBlockMap: self._unexpected_blocks_type(blocks) return None + def contain(self, key: BlockHashWithGroupId, block_id: int) -> bool: + """ + Checks whether the key maps to the given block ID. + """ + blocks = self._cache.get(key) + if blocks is None: + return False + if isinstance(blocks, KVCacheBlock): + return blocks.block_id == block_id + if isinstance(blocks, dict): + return block_id in blocks + self._unexpected_blocks_type(blocks) + return False + def insert(self, key: BlockHashWithGroupId, block: KVCacheBlock) -> None: """ Inserts the KVCacheBlock to the cache @@ -169,6 +183,7 @@ class BlockPool: # Cache for block lookup self.cached_block_hash_to_block: BlockHashToBlockMap = BlockHashToBlockMap() + self.cached_block_hashes_by_block: dict[int, set[BlockHashWithGroupId]] = {} # To represent a placeholder block with block_id=0. # The ref_cnt of null_block is not maintained, needs special care to @@ -245,7 +260,6 @@ class BlockPool: if num_cached_blocks >= num_full_blocks: return new_full_blocks = blocks[num_cached_blocks:num_full_blocks] - assert len(request.block_hashes) >= num_full_blocks assert block_mask is None or len(block_mask) == len(new_full_blocks) if block_size == self.hash_block_size: # Common case. @@ -254,11 +268,10 @@ class BlockPool: # block_size is a multiple of hash_block_size. This happens when # different KV cache groups have different block sizes. assert block_size % self.hash_block_size == 0 - # Recalculate block_hashes at the granularity of block_size, using - # the original block_hashes (at the granularity of hash_block_size). block_hashes = BlockHashListWithBlockSize( request.block_hashes, self.hash_block_size, block_size ) + assert len(block_hashes) >= num_full_blocks new_block_hashes = block_hashes[num_cached_blocks:] new_hashes: list[ExternalBlockHash] | None = ( @@ -270,15 +283,27 @@ class BlockPool: # in align mode. We skip null blocks here. if blk.is_null or (block_mask is not None and not block_mask[i]): continue - assert blk.block_hash is None block_hash = new_block_hashes[i] + num_hash_tokens = (num_cached_blocks + i + 1) * block_size # Update and added the full block to the cache. block_hash_with_group_id = make_block_hash_with_group_id( block_hash, kv_cache_group_id ) - blk.block_hash = block_hash_with_group_id - self.cached_block_hash_to_block.insert(block_hash_with_group_id, blk) + if blk.block_hash is not None: + # The only valid case where a "new full block" already has a + # hash is partial->full promotion of the same cache block. + assert ( + blk.block_hash_num_tokens is not None + and blk.block_hash_num_tokens < num_hash_tokens + ) + removed_hashes = self._remove_cached_block_hashes(blk) + self._emit_block_removed_events(removed_hashes) + self._insert_block_hash( + block_hash_with_group_id, + blk, + num_tokens=num_hash_tokens, + ) if new_hashes is not None: new_hashes.append(maybe_convert_block_hash(block_hash)) @@ -330,6 +355,190 @@ class BlockPool: ) ) + def cache_partial_block( + self, + request: Request, + block: KVCacheBlock, + num_tokens: int, + kv_cache_group_id: int, + block_size: int, + ) -> BlockHashWithGroupId | None: + """Register a partial prefix-cache entry for an existing block. + + Prefix-cache keys normally identify full cache blocks. A partial entry + makes an existing cache block reachable from a fine-grained prefix + boundary inside that block without allocating or copying a new + ``KVCacheBlock``. + + The partial entry is lookup metadata owned by ``block``. If ``block`` + has no primary hash, the key becomes its primary hash. If the block + already has a primary hash, the partial entry is tracked in + ``cached_block_hashes_by_block`` so eviction, reset, and promotion can + remove every hash key that points to the block. + + Args: + request: Request whose token IDs and block hashes define the + partial entry. + block: Existing cache block to make reachable from the partial + prefix boundary. + num_tokens: Prefix length represented by the partial entry. It + must be a positive multiple of ``self.hash_block_size`` and + cannot exceed the request's computed block hashes. + kv_cache_group_id: KV cache group that owns the partial entry. + block_size: Cache block size for the owning group. The partial + entry hash itself is always the prefix-chain hash at + ``num_tokens``; ``block_size`` is used to assert that the + entry is partial within the owning cache block. + + Returns: + The hash key with group ID if a partial entry can be registered; + otherwise ``None`` for null blocks. + """ + if block.is_null: + return None + + assert block_size > self.hash_block_size + assert block_size % self.hash_block_size == 0 + assert num_tokens % block_size != 0 + block_hash = self._get_partial_block_hash(request, num_tokens) + num_hash_blocks = num_tokens // self.hash_block_size + block_hash_with_group_id = make_block_hash_with_group_id( + block_hash, kv_cache_group_id + ) + already_cached = block.block_hash == block_hash_with_group_id or ( + self.cached_block_hash_to_block.contain( + block_hash_with_group_id, block.block_id + ) + ) + if ( + not already_cached + and block.block_hash is not None + and block.block_hash_num_tokens is not None + and block.block_hash_num_tokens < num_hash_blocks * self.hash_block_size + ): + removed_hashes = self._remove_cached_block_hashes(block) + self._emit_block_removed_events(removed_hashes) + self._insert_block_hash( + block_hash_with_group_id, + block, + num_tokens=num_hash_blocks * self.hash_block_size, + ) + if self.enable_kv_cache_events and not already_cached: + parent_hash, block_start = self._get_partial_block_parent_hash_and_start( + request, num_tokens + ) + parent_block_hash = ( + maybe_convert_block_hash(parent_hash) + if parent_hash is not None + else None + ) + block_end = num_tokens + curr_mm_idx = -1 if block_start > 0 else 0 + extra_keys, _ = generate_block_hash_extra_keys( + request, block_start, block_end, curr_mm_idx + ) + self.kv_event_queue.append( + BlockStored( + block_hashes=[maybe_convert_block_hash(block_hash)], + parent_block_hash=parent_block_hash, + token_ids=request.all_token_ids[block_start:block_end], + block_size=block_end - block_start, + lora_id=request.lora_request.adapter_id + if request.lora_request + else None, + medium=MEDIUM_GPU, + lora_name=request.lora_request.name + if request.lora_request + else None, + extra_keys=[extra_keys], + group_idx=kv_cache_group_id, + ) + ) + return block_hash_with_group_id + + def _get_partial_block_hash( + self, + request: Request, + num_tokens: int, + ) -> BlockHash: + assert num_tokens % self.hash_block_size == 0 + num_hash_blocks = num_tokens // self.hash_block_size + assert 0 < num_hash_blocks <= len(request.block_hashes) + + # Each hash_block_size hash chains over its full prefix, so the partial + # entry for any group block size is the hash at that prefix boundary. + return request.block_hashes[num_hash_blocks - 1] + + def _get_partial_block_parent_hash_and_start( + self, + request: Request, + num_tokens: int, + ) -> tuple[BlockHash | None, int]: + num_hash_blocks = num_tokens // self.hash_block_size + parent_hash = ( + request.block_hashes[num_hash_blocks - 2] if num_hash_blocks > 1 else None + ) + block_start = (num_hash_blocks - 1) * self.hash_block_size + return parent_hash, block_start + + def _remove_cached_block_hashes( + self, + block: KVCacheBlock, + ) -> list[BlockHashWithGroupId]: + block_hashes: list[BlockHashWithGroupId] = [] + if block.block_hash is not None: + block_hashes.append(block.block_hash) + block_hashes.extend(self.cached_block_hashes_by_block.pop(block.block_id, ())) + if not block_hashes: + return [] + + removed_hashes: list[BlockHashWithGroupId] = [] + for block_hash in block_hashes: + if ( + self.cached_block_hash_to_block.pop(block_hash, block.block_id) + is not None + ): + removed_hashes.append(block_hash) + block.reset_hash() + return removed_hashes + + def _emit_block_removed_events( + self, + block_hashes: list[BlockHashWithGroupId], + ) -> None: + if not self.enable_kv_cache_events: + return + for block_hash in block_hashes: + self.kv_event_queue.append( + BlockRemoved( + block_hashes=[maybe_convert_block_hash(get_block_hash(block_hash))], + medium=MEDIUM_GPU, + group_idx=get_group_id(block_hash), + ) + ) + + def _insert_block_hash( + self, + block_hash_with_group_id: BlockHashWithGroupId, + block: KVCacheBlock, + num_tokens: int | None, + ) -> None: + if block.block_hash == block_hash_with_group_id: + return + + if self.cached_block_hash_to_block.contain( + block_hash_with_group_id, block.block_id + ): + return + + if block.block_hash is None: + block.set_block_hash(block_hash_with_group_id, num_tokens=num_tokens) + else: + self.cached_block_hashes_by_block.setdefault(block.block_id, set()).add( + block_hash_with_group_id + ) + self.cached_block_hash_to_block.insert(block_hash_with_group_id, block) + def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]: """Get new blocks from the free block pool. @@ -377,26 +586,12 @@ class BlockPool: if self.metrics_collector: self.metrics_collector.on_block_evicted(block) - block_hash = block.block_hash - if block_hash is None: + evicted_hashes = self._remove_cached_block_hashes(block) + if not evicted_hashes: # The block doesn't have hash, eviction is not needed return False - if self.cached_block_hash_to_block.pop(block_hash, block.block_id) is None: - # block not found in cached_block_hash_to_block, - # eviction is not needed - return False - - block.reset_hash() - - if self.enable_kv_cache_events: - self.kv_event_queue.append( - BlockRemoved( - block_hashes=[maybe_convert_block_hash(get_block_hash(block_hash))], - medium=MEDIUM_GPU, - group_idx=get_group_id(block_hash), - ) - ) + self._emit_block_removed_events(evicted_hashes) return True def touch(self, blocks: Sequence[KVCacheBlock]) -> None: @@ -478,6 +673,7 @@ class BlockPool: # Remove all hashes so that no new blocks will hit. self.cached_block_hash_to_block = BlockHashToBlockMap() + self.cached_block_hashes_by_block.clear() # Remove all hashes from all blocks. for block in self.blocks: diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 95b8fba4cc..a3822e7fc4 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -125,6 +125,9 @@ class KVCacheBlock: # The hash key (block hash + group id) of the block, only available # when the block is full and cached. _block_hash: BlockHashWithGroupId | None = None + # Number of prefix tokens covered by _block_hash. For full blocks this is + # the full block boundary; partial aliases can end inside a cache block. + _block_hash_num_tokens: int | None = None # Used to construct a doubly linked list for free blocks. # These two attributes should only be manipulated by FreeKVCacheBlockQueue. @@ -138,16 +141,25 @@ class KVCacheBlock: def block_hash(self) -> BlockHashWithGroupId | None: return self._block_hash - @block_hash.setter - def block_hash(self, block_hash: BlockHashWithGroupId): - assert self.block_hash is None, ( + @property + def block_hash_num_tokens(self) -> int | None: + return self._block_hash_num_tokens + + def set_block_hash( + self, + block_hash: BlockHashWithGroupId, + num_tokens: int | None = None, + ) -> None: + assert self.block_hash is None and self._block_hash_num_tokens is None, ( "The block already has a hash. This should not happen." ) self._block_hash = block_hash + self._block_hash_num_tokens = num_tokens def reset_hash(self): """Reset the block hash when the block is evicted.""" self._block_hash = None + self._block_hash_num_tokens = None def __repr__(self) -> str: # Use block_id instead of KVCacheBlock object to avoid calling __repr__ @@ -158,6 +170,7 @@ class KVCacheBlock: f"KVCacheBlock(block_id={self.block_id}, " f"ref_cnt={self.ref_cnt}, " f"_block_hash={self._block_hash!r}, " + f"_block_hash_num_tokens={self._block_hash_num_tokens}, " f"prev_free_block={prev_block_id}, " f"next_free_block={next_block_id})" ) @@ -658,18 +671,24 @@ def resolve_kv_cache_block_sizes( def get_request_block_hasher( - block_size: int, + hash_block_size: int, caching_hash_fn: Callable[[Any], bytes], ) -> Callable[[Request], list[BlockHash]]: """ Returns a function which computes the list of un-computed block hashes - of a request.""" + of a request. + + Hashes are computed at ``hash_block_size`` granularity and chained over the + full prefix, so each hash uniquely fingerprints the prefix ending at its + boundary. Coarser group block sizes and partial-cache boundaries reuse + these hashes directly (see ``BlockHashListWithBlockSize``). + """ def request_block_hasher(request: Request) -> list[BlockHash]: - start_token_idx = len(request.block_hashes) * block_size + start_token_idx = len(request.block_hashes) * hash_block_size num_tokens = request.num_tokens - if start_token_idx + block_size > num_tokens: + if start_token_idx + hash_block_size > num_tokens: # Early stop when there no new full blocks created. return [] @@ -686,7 +705,7 @@ def get_request_block_hasher( ) new_block_hashes: list[BlockHash] = [] while True: - end_token_idx = start_token_idx + block_size + end_token_idx = start_token_idx + hash_block_size if end_token_idx > num_tokens: # We only hash full blocks break @@ -703,7 +722,7 @@ def get_request_block_hasher( ) new_block_hashes.append(block_hash) - start_token_idx += block_size + start_token_idx += hash_block_size prev_block_hash_value = block_hash return new_block_hashes @@ -2132,11 +2151,14 @@ class BlockHashListWithBlockSize: Currently, only scaling up by an integer factor is supported (i.e., `target_block_size` is a multiple of `hash_block_size`). Conversion is - performed lazily on access for efficiency, by concatenating consecutive - hashes at `hash_block_size` to form each hash at `target_block_size`. + performed lazily on access for efficiency. Each `hash_block_size` hash is + already chained over its entire prefix, so the hash at the last + `hash_block_size` boundary of a `target_block_size` block uniquely + fingerprints that block's prefix; we use it directly. Example (`hash_block_size` = 16, `target_block_size` = 32): - concatenating two 16-size hashes yields one 32-size hash: + the second 16-size hash already covers tokens 0-31, so it is the 32-size + hash: Block hashes with block_size 16: | Token Range | 0-15 | 16-31 | 32-47 | 48-63 | @@ -2146,7 +2168,7 @@ class BlockHashListWithBlockSize: Block hashes with block_size 32: | Token Range | 0-31 | 32-63 | |-------------|------|-------| - | Hash | AB | CD | + | Hash | B | D | Args: block_hashes: Block hashes to convert, computed at `hash_block_size`. @@ -2188,9 +2210,9 @@ class BlockHashListWithBlockSize: yield self._get_value_at(i) def _get_value_at(self, idx: int) -> BlockHash: - base = idx * self.scale_factor - end = base + self.scale_factor - return BlockHash(b"".join(self.block_hashes[base:end])) + # The last hash_block_size hash within the target block already chains + # over the whole prefix, so it is the target block's hash. + return self.block_hashes[(idx + 1) * self.scale_factor - 1] BlockHashList = list[BlockHash] | BlockHashListWithBlockSize