[Core] Scheduler perf fix (#4270)

This commit is contained in:
SangBin Cho
2024-04-22 21:11:06 +00:00
committed by GitHub
parent 3d925165f2
commit ad8d696a99
2 changed files with 11 additions and 14 deletions
+2 -5
View File
@@ -297,7 +297,6 @@ class Scheduler:
def add_seq_group(self, seq_group: SequenceGroup) -> None:
# Add sequence groups to the waiting queue.
logger.debug(f"add_seq_group {seq_group.request_id}")
self.waiting.append(seq_group)
def abort_seq_group(self, request_id: Union[str, Iterable[str]]) -> None:
@@ -427,7 +426,6 @@ class Scheduler:
swapped_out.append(seq_group)
break
else:
logger.debug(f"append slot for {seq_group}")
self._append_slots(seq_group, blocks_to_copy)
is_prefill = seq_group.is_prefill()
if is_prefill:
@@ -659,7 +657,7 @@ class Scheduler:
if curr_loras is not None and lora_int_id > 0:
curr_loras.add(lora_int_id)
waiting_queue.popleft()
self._allocate_and_set_running(seq_group, num_new_tokens)
self._allocate_and_set_running(seq_group)
seq_groups.append(
ScheduledSequenceGroup(seq_group=seq_group,
token_chunk_size=num_new_tokens))
@@ -952,8 +950,7 @@ class Scheduler:
self.running = deque(seq_group for seq_group in self.running
if not seq_group.is_finished())
def _allocate_and_set_running(self, seq_group: SequenceGroup,
num_new_tokens: int) -> None:
def _allocate_and_set_running(self, seq_group: SequenceGroup) -> None:
self.block_manager.allocate(seq_group)
for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
seq.status = SequenceStatus.RUNNING