[Core] Scheduling optimization 2 (#4280)
This commit is contained in:
@@ -395,12 +395,12 @@ class Scheduler:
|
||||
# We can have up to 1 running prefill at any given time in running
|
||||
# queue, which means we can guarantee chunk size is at least 1.
|
||||
assert num_running_tokens != 0
|
||||
num_running_seqs = seq_group.get_max_num_running_seqs()
|
||||
|
||||
running_queue.popleft()
|
||||
while not self._can_append_slots(seq_group):
|
||||
budget.subtract_num_batched_tokens(seq_group.request_id,
|
||||
num_running_tokens)
|
||||
num_running_seqs = seq_group.get_max_num_running_seqs()
|
||||
budget.subtract_num_seqs(seq_group.request_id,
|
||||
num_running_seqs)
|
||||
if curr_loras is not None and seq_group.lora_int_id > 0:
|
||||
@@ -439,7 +439,13 @@ class Scheduler:
|
||||
token_chunk_size=1))
|
||||
budget.add_num_batched_tokens(seq_group.request_id,
|
||||
num_running_tokens)
|
||||
budget.add_num_seqs(seq_group.request_id, num_running_seqs)
|
||||
# OPTIMIZATION: Note that get_max_num_running_seqs is
|
||||
# expensive. For the default scheduling chase where
|
||||
# enable_chunking is False, num_seqs are updated before running
|
||||
# this method, so we don't have to update it again here.
|
||||
if enable_chunking:
|
||||
num_running_seqs = seq_group.get_max_num_running_seqs()
|
||||
budget.add_num_seqs(seq_group.request_id, num_running_seqs)
|
||||
if curr_loras is not None and seq_group.lora_int_id > 0:
|
||||
curr_loras.add(seq_group.lora_int_id)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user