[2/N][Core] support partial prefix cache hit for hybrid model (#46384)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Yifan Qiao
parent
8e981630c9
commit
481e481be7
@@ -260,7 +260,9 @@ class BlockPool:
|
||||
return
|
||||
new_full_blocks = blocks[num_cached_blocks:num_full_blocks]
|
||||
assert block_mask is None or len(block_mask) == len(new_full_blocks)
|
||||
block_hashes = resolve_block_hashes(request, self.hash_block_size, block_size)
|
||||
block_hashes = resolve_block_hashes(
|
||||
request.block_hashes, self.hash_block_size, block_size
|
||||
)
|
||||
|
||||
new_block_hashes = block_hashes[num_cached_blocks:]
|
||||
new_hashes: list[ExternalBlockHash] | None = (
|
||||
@@ -390,7 +392,9 @@ class BlockPool:
|
||||
if not self.enable_kv_cache_events or num_cached_blocks == 0:
|
||||
return
|
||||
|
||||
block_hashes = resolve_block_hashes(request, self.hash_block_size, block_size)
|
||||
block_hashes = resolve_block_hashes(
|
||||
request.block_hashes, self.hash_block_size, block_size
|
||||
)
|
||||
|
||||
# Collect external hashes and extra_keys for cached blocks.
|
||||
cached_hashes: list[ExternalBlockHash] = []
|
||||
@@ -622,6 +626,24 @@ class BlockPool:
|
||||
)
|
||||
self.cached_block_hash_to_block.insert(block_hash_with_group_id, block)
|
||||
|
||||
def move_block_hashes(
|
||||
self,
|
||||
src_block: KVCacheBlock,
|
||||
dst_block: KVCacheBlock,
|
||||
) -> None:
|
||||
"""Re-point ``src_block``'s prefix-cache entries to ``dst_block``.
|
||||
|
||||
Used when the request owning ``src_block`` keeps writing into it
|
||||
: the prefix cache holds a private copy (``dst_block``)
|
||||
under the same hashes instead. Entries stay live; no events emitted.
|
||||
"""
|
||||
assert dst_block.block_hash is None
|
||||
assert dst_block.block_id not in self.cached_block_hashes_by_block
|
||||
num_tokens = src_block.block_hash_num_tokens
|
||||
for block_hash in self._remove_cached_block_hashes(src_block):
|
||||
# `num_tokens` only applies to the first (primary) insertion.
|
||||
self._insert_block_hash(block_hash, dst_block, num_tokens=num_tokens)
|
||||
|
||||
def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]:
|
||||
"""Get new blocks from the free block pool.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user