[Bugfix] Fix for Spec model TP + Chunked Prefill (#10232)
Signed-off-by: andoorve <37849411+andoorve@users.noreply.github.com> Signed-off-by: Sourashis Roy <sroy@roblox.com> Co-authored-by: Sourashis Roy <sroy@roblox.com>
This commit is contained in:
co-authored by
Sourashis Roy
parent
1f6584ee85
commit
db66e018ea
+19
-9
@@ -1201,15 +1201,25 @@ class Scheduler:
|
||||
# Update swapped requests.
|
||||
self.swapped.extend(running_scheduled.swapped_out)
|
||||
# Put prefills first due to Attention backend ordering assumption.
|
||||
scheduled_seq_groups = (prefills.seq_groups +
|
||||
running_scheduled.prefill_seq_groups +
|
||||
swapped_in.prefill_seq_groups +
|
||||
running_scheduled.decode_seq_groups +
|
||||
swapped_in.decode_seq_groups)
|
||||
num_prefill_groups = (len(prefills.seq_groups) +
|
||||
len(swapped_in.prefill_seq_groups) +
|
||||
len(running_scheduled.prefill_seq_groups))
|
||||
# If all prompts, then we set num_lookahead_slots to 0
|
||||
# this allows us to go through the `no_spec` path in
|
||||
# `spec_decode_worker.py`
|
||||
all_prefills = (len(scheduled_seq_groups) == num_prefill_groups)
|
||||
num_lookahead_slots = (0 if
|
||||
(all_prefills
|
||||
and not self.scheduler_config.is_multi_step)
|
||||
else running_scheduled.num_lookahead_slots)
|
||||
return SchedulerOutputs(
|
||||
scheduled_seq_groups=(prefills.seq_groups +
|
||||
running_scheduled.prefill_seq_groups +
|
||||
swapped_in.prefill_seq_groups +
|
||||
running_scheduled.decode_seq_groups +
|
||||
swapped_in.decode_seq_groups),
|
||||
num_prefill_groups=(len(prefills.seq_groups) +
|
||||
len(swapped_in.prefill_seq_groups) +
|
||||
len(running_scheduled.prefill_seq_groups)),
|
||||
scheduled_seq_groups=scheduled_seq_groups,
|
||||
num_prefill_groups=num_prefill_groups,
|
||||
num_batched_tokens=budget.num_batched_tokens +
|
||||
budget.num_cached_tokens,
|
||||
blocks_to_swap_in=swapped_in.blocks_to_swap_in,
|
||||
@@ -1218,7 +1228,7 @@ class Scheduler:
|
||||
swapped_in.blocks_to_copy,
|
||||
ignored_seq_groups=prefills.ignored_seq_groups +
|
||||
swapped_in.infeasible_seq_groups,
|
||||
num_lookahead_slots=running_scheduled.num_lookahead_slots,
|
||||
num_lookahead_slots=num_lookahead_slots,
|
||||
running_queue_size=len(self.running),
|
||||
preempted=(len(running_scheduled.preempted) +
|
||||
len(running_scheduled.swapped_out)),
|
||||
|
||||
Reference in New Issue
Block a user