Possible fix for conflict between Automated Prefix Caching (#2762) and multi-LoRA support (#1804) (#3263)

This commit is contained in:
jacobthebanana
2024-03-07 23:03:22 +00:00
committed by GitHub
parent 385da2dae2
commit 8cbba4622c
2 changed files with 31 additions and 14 deletions
+2 -1
View File
@@ -175,7 +175,8 @@ class Sequence:
# TODO: The current hashing function is O(L^2). We should optimize
# this in the future.
num_tokens = self.num_hashed_tokens_of_block(logical_idx)
return hash(tuple(self.data.get_token_ids()[0:num_tokens]))
return hash(
(tuple(self.data.get_token_ids()[0:num_tokens]), self.lora_int_id))
def num_hashed_tokens_of_block(self, logical_idx: int):
return logical_idx * self.block_size + self.block_size