[Core] support LoRA and prompt adapter in content-based hashing for Block Manager v2 prefix caching (#8240)

This commit is contained in:
Sungjae Lee
2024-12-13 07:51:25 -08:00
committed by GitHub
parent d1fa714cb1
commit c31d4a57a6
10 changed files with 244 additions and 53 deletions
+7 -1
View File
@@ -151,8 +151,13 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
max_block_sliding_window=self.max_block_sliding_window,
)
if seq.get_token_ids():
# NOTE: If there are any factors affecting the block besides
# token_ids, they should be added as input to extra_hash.
extra_hash = seq.extra_hash()
# Add blocks to the block table only if the sequence is non empty.
block_table.allocate(seq.get_token_ids())
block_table.allocate(token_ids=seq.get_token_ids(),
extra_hash=extra_hash)
return block_table
@@ -238,6 +243,7 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
token_ids=block_table.get_unseen_token_ids(seq.get_token_ids()),
num_lookahead_slots=num_lookahead_slots,
num_computed_slots=seq.data.get_num_computed_tokens(),
extra_hash=seq.extra_hash(),
)
# Return any new copy-on-writes.
new_cows = self.block_allocator.clear_copy_on_writes()