Make AsyncLLMEngine more robust & fix batched abort (#969)

Signed-off-by: Antoni Baum <antoni.baum@protonmail.com>
Co-authored-by: Avnish Narayan <38871737+avnishn@users.noreply.github.com>
This commit is contained in:
Antoni Baum
2023-09-07 13:43:45 -07:00
committed by GitHub
co-authored by Avnish Narayan
parent 7a9c20c715
commit c07ece5ca4
7 changed files with 345 additions and 55 deletions
+4 -1
View File
@@ -92,7 +92,10 @@ class Scheduler:
request_id = (request_id, )
request_ids = set(request_id)
for state_queue in [self.waiting, self.running, self.swapped]:
for seq_group in state_queue:
# We need to reverse the list as we are removing elements
# from it as we iterate over it. If we don't do it,
# indices will get messed up and we will skip over elements.
for seq_group in reversed(state_queue):
if seq_group.request_id in request_ids:
# Remove the sequence group from the state queue.
state_queue.remove(seq_group)