[FIX] Fix styles in automatic prefix caching & add a automatic prefix caching benchmark (#3158)

This commit is contained in:
Zhuohan Li
2024-03-03 14:37:18 -08:00
committed by GitHub
parent d65fac2738
commit 996d095c54
4 changed files with 69 additions and 18 deletions
+4 -11
View File
@@ -236,13 +236,6 @@ class BlockSpaceManager:
token_ids_len = len(seq.data.get_token_ids())
return token_ids_len > 0 and token_ids_len % seq.block_size == 0
def _is_last_block(
self,
seq: Sequence,
index: int,
) -> bool:
return index == len(seq.logical_token_blocks) - 1
def _maybe_promote_last_block(
self,
seq: Sequence,
@@ -436,7 +429,7 @@ class BlockSpaceManager:
def compute_last_full_block_in_seq(self, seq: Sequence):
if seq.seq_id not in self.block_tables:
return
max_full_block = seq.get_len() // seq.block_size - 1
max_full_block = seq.get_len() // self.block_size - 1
block_table = self.block_tables[seq.seq_id]
if max_full_block == -1:
return
@@ -451,9 +444,9 @@ class BlockSpaceManager:
return [b.block_number for b in block_table[:block_idx + 1]]
return []
# Can return non-empty result only with prefix caching enabled.
def get_common_computed_block_ids(self,
seq_group: SequenceGroup) -> List[int]:
# Can return non-empty result only with prefix caching enabled.
if not self.enable_caching:
return []
@@ -463,9 +456,9 @@ class BlockSpaceManager:
]
return commonprefix([ids for ids in ids_list if ids != []])
# We only mark the last full block because with prefix caching,
# all blocks until the marked one are guaranteed to be computed.
def mark_blocks_as_computed(self, seq_group: SequenceGroup):
# NOTE: We only mark the last full block because with prefix caching,
# all blocks until the marked one are guaranteed to be computed.
if self.enable_caching:
for seq in seq_group.seqs_dict.values():
self.compute_last_full_block_in_seq(seq)
+2 -6
View File
@@ -160,10 +160,10 @@ class Sequence:
def lora_int_id(self) -> int:
return self.lora_request.lora_int_id if self.lora_request else 0
# TODO The current hashing function is O(L^2). We should optimize this in
# the future.
def hash_of_block(self, logical_idx: int) -> int:
# Compute the number of tokens in the sequence
# TODO: The current hashing function is O(L^2). We should optimize
# this in the future.
num_tokens = self.num_hashed_tokens_of_block(logical_idx)
return hash(tuple(self.data.get_token_ids()[0:num_tokens]))
@@ -308,10 +308,6 @@ class SequenceGroup:
# We use the prompt of an arbitrary sequence.
return next(iter(self.seqs_dict.values())).data.prompt_token_ids
@property
def block_size(self) -> int:
return next(iter(self.seqs_dict.values())).block_size
@property
def lora_int_id(self) -> int:
return self.lora_request.lora_int_id if self.lora_request else 0