[Bugfix][Core] Prefix caching causes incorrect outputs due to outdated ComputedBlocksTracker (#18957)

Signed-off-by: 刘全 <quan.liu2@dbappsecurity.com.cn>
Co-authored-by: 刘全 <quan.liu2@dbappsecurity.com.cn>
This commit is contained in:
quanliu
2025-06-15 21:56:37 -07:00
committed by GitHub
co-authored by 刘全
parent c6703d1e0d
commit 92183b41f3
3 changed files with 331 additions and 0 deletions
+4
View File
@@ -270,6 +270,10 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
self.block_tables[seq_id].free()
del self.block_tables[seq_id]
def remove_seq_from_computed_blocks_tracker(self, seq: Sequence) -> None:
seq_id = seq.seq_id
self._computed_blocks_tracker.remove_seq(seq_id)
def free_cross(self, seq_group: SequenceGroup) -> None:
request_id = seq_group.request_id
if request_id not in self.cross_block_tables:
+33
View File
@@ -901,6 +901,8 @@ class Scheduler:
num_new_tokens=num_new_tokens_uncached,
num_new_seqs=num_new_seqs,
):
self.remove_seq_from_computed_blocks_tracker(
seq_group, SequenceStatus.SWAPPED)
break
if lora_int_id > 0 and curr_loras is not None:
@@ -1024,6 +1026,9 @@ class Scheduler:
# Put the sequence back into the waiting queue
waiting_queue.appendleft(seq_group)
self.remove_seq_from_computed_blocks_tracker(
seq_group, SequenceStatus.WAITING)
waiting_queue = deque(sorted(waiting_queue, key=self._get_priority))
self.waiting = waiting_queue
@@ -1113,6 +1118,8 @@ class Scheduler:
)
for seq in waiting_seqs:
seq.status = SequenceStatus.FINISHED_IGNORED
self.remove_seq_from_computed_blocks_tracker(
seq_group, SequenceStatus.FINISHED_IGNORED)
ignored_seq_groups.append(seq_group)
waiting_queue.popleft()
continue
@@ -1126,6 +1133,8 @@ class Scheduler:
can_allocate = self.block_manager.can_allocate(
seq_group, num_lookahead_slots=num_lookahead_slots)
if can_allocate == AllocStatus.LATER:
self.remove_seq_from_computed_blocks_tracker(
seq_group, SequenceStatus.WAITING)
break
elif can_allocate == AllocStatus.NEVER:
logger.warning(
@@ -1136,6 +1145,8 @@ class Scheduler:
)
for seq in waiting_seqs:
seq.status = SequenceStatus.FINISHED_IGNORED
self.remove_seq_from_computed_blocks_tracker(
seq_group, SequenceStatus.FINISHED_IGNORED)
ignored_seq_groups.append(seq_group)
waiting_queue.popleft()
continue
@@ -1145,6 +1156,8 @@ class Scheduler:
if len(seq_groups) == 0:
using_prompt_embeds = seq_group.uses_prompt_embeds()
if using_prompt_embeds != seq_group.uses_prompt_embeds():
self.remove_seq_from_computed_blocks_tracker(
seq_group, SequenceStatus.WAITING)
leftover_waiting_sequences.appendleft(seq_group)
waiting_queue.popleft()
continue
@@ -1159,6 +1172,8 @@ class Scheduler:
and len(curr_loras) >= self.lora_config.max_loras):
# We don't have a space for another LoRA, so
# we ignore this request for now.
self.remove_seq_from_computed_blocks_tracker(
seq_group, SequenceStatus.WAITING)
leftover_waiting_sequences.appendleft(seq_group)
waiting_queue.popleft()
continue
@@ -1168,6 +1183,8 @@ class Scheduler:
# We've reached the budget limit - since there might be
# continuous prefills in the running queue, we should break
# to avoid scheduling any new prefills.
self.remove_seq_from_computed_blocks_tracker(
seq_group, SequenceStatus.WAITING)
break
num_new_seqs = seq_group.get_max_num_running_seqs()
@@ -1175,6 +1192,8 @@ class Scheduler:
num_new_tokens=num_new_tokens_uncached,
num_new_seqs=num_new_seqs,
):
self.remove_seq_from_computed_blocks_tracker(
seq_group, SequenceStatus.WAITING)
break
# Can schedule this request.
@@ -1688,6 +1707,20 @@ class Scheduler:
"""Free a sequence from a block table."""
self.block_manager.free(seq)
def remove_seq_from_computed_blocks_tracker(
self, seq_group: SequenceGroup,
status: Optional[SequenceStatus]) -> None:
seqs = seq_group.get_seqs(status=status)
for seq in seqs:
self._remove_seq_from_computed_blocks_tracker(seq)
def _remove_seq_from_computed_blocks_tracker(self, seq: Sequence) -> None:
"""
Free a sequence computed blocks tracker _seq_id_to_blocks_hashes
and _seq_id_to_num_tokens_computed.
"""
self.block_manager.remove_seq_from_computed_blocks_tracker(seq)
def _free_finished_seqs(self, seq_group: SequenceGroup) -> None:
"""Free finished seqs in a sequence group."""
for seq in seq_group.get_seqs():