[2/N][Core] support partial prefix cache hit for hybrid model (#46384)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Jiangyun Zhu
2026-07-12 05:37:51 +00:00
committed by GitHub
co-authored by Yifan Qiao
parent 8e981630c9
commit 481e481be7
21 changed files with 1653 additions and 177 deletions
+24 -2
View File
@@ -260,7 +260,9 @@ class BlockPool:
return
new_full_blocks = blocks[num_cached_blocks:num_full_blocks]
assert block_mask is None or len(block_mask) == len(new_full_blocks)
block_hashes = resolve_block_hashes(request, self.hash_block_size, block_size)
block_hashes = resolve_block_hashes(
request.block_hashes, self.hash_block_size, block_size
)
new_block_hashes = block_hashes[num_cached_blocks:]
new_hashes: list[ExternalBlockHash] | None = (
@@ -390,7 +392,9 @@ class BlockPool:
if not self.enable_kv_cache_events or num_cached_blocks == 0:
return
block_hashes = resolve_block_hashes(request, self.hash_block_size, block_size)
block_hashes = resolve_block_hashes(
request.block_hashes, self.hash_block_size, block_size
)
# Collect external hashes and extra_keys for cached blocks.
cached_hashes: list[ExternalBlockHash] = []
@@ -622,6 +626,24 @@ class BlockPool:
)
self.cached_block_hash_to_block.insert(block_hash_with_group_id, block)
def move_block_hashes(
self,
src_block: KVCacheBlock,
dst_block: KVCacheBlock,
) -> None:
"""Re-point ``src_block``'s prefix-cache entries to ``dst_block``.
Used when the request owning ``src_block`` keeps writing into it
: the prefix cache holds a private copy (``dst_block``)
under the same hashes instead. Entries stay live; no events emitted.
"""
assert dst_block.block_hash is None
assert dst_block.block_id not in self.cached_block_hashes_by_block
num_tokens = src_block.block_hash_num_tokens
for block_hash in self._remove_cached_block_hashes(src_block):
# `num_tokens` only applies to the first (primary) insertion.
self._insert_block_hash(block_hash, dst_block, num_tokens=num_tokens)
def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]:
"""Get new blocks from the free block pool.