[Bugfix][Scheduler] Don't pad spec decode up to max_model_len (#53962)
This commit is contained in:
@@ -964,14 +964,19 @@ class Scheduler(SchedulerInterface):
|
||||
and num_new_tokens == 1
|
||||
and (scheduled_running_reqs and not prefill_scheduled)
|
||||
):
|
||||
num_new_tokens = 1 + self.num_spec_tokens
|
||||
padded_num_tokens = 1 + self.num_spec_tokens
|
||||
# Pad only when there is room for the sampled token(s).
|
||||
if (
|
||||
num_new_tokens > request_token_budget
|
||||
or num_computed_tokens + num_new_tokens > self.max_model_len
|
||||
num_computed_tokens
|
||||
+ padded_num_tokens
|
||||
+ self.num_sampled_tokens_per_step
|
||||
<= self.max_model_len
|
||||
):
|
||||
# Prefer to not schedule than schedule un-padded here.
|
||||
break
|
||||
pad_spec_decode = True
|
||||
if padded_num_tokens > request_token_budget:
|
||||
# Prefer to not schedule than schedule un-padded.
|
||||
break
|
||||
num_new_tokens = padded_num_tokens
|
||||
pad_spec_decode = True
|
||||
|
||||
threshold = self.scheduler_config.long_prefill_token_threshold
|
||||
if 0 < threshold < num_new_tokens:
|
||||
|
||||
Reference in New Issue
Block a user