[Core] support LoRA and prompt adapter in content-based hashing for Block Manager v2 prefix caching (#8240)

This commit is contained in:
Sungjae Lee
2024-12-13 07:51:25 -08:00
committed by GitHub
parent d1fa714cb1
commit c31d4a57a6
10 changed files with 244 additions and 53 deletions
+12 -5
View File
@@ -177,7 +177,8 @@ class BlockPool:
token_ids=[],
block_size=self._block_size,
allocator=self._allocator,
block_id=None))
block_id=None,
extra_hash=None))
def increase_pool(self):
"""Doubles the internal pool size
@@ -194,10 +195,15 @@ class BlockPool:
token_ids=[],
block_size=self._block_size,
allocator=self._allocator,
block_id=None))
block_id=None,
extra_hash=None))
def init_block(self, prev_block: Optional[Block], token_ids: List[int],
block_size: int, physical_block_id: Optional[int]) -> Block:
def init_block(self,
prev_block: Optional[Block],
token_ids: List[int],
block_size: int,
physical_block_id: Optional[int],
extra_hash: Optional[int] = None) -> Block:
if len(self._free_ids) == 0:
self.increase_pool()
assert len(self._free_ids) > 0
@@ -210,7 +216,8 @@ class BlockPool:
token_ids=token_ids,
block_size=block_size,
allocator=block._allocator, # type: ignore[attr-defined]
block_id=physical_block_id)
block_id=physical_block_id,
extra_hash=extra_hash)
block.pool_id = pool_id # type: ignore[attr-defined]
return block