[Bugfix][Scheduler] Don't pad spec decode up to max_model_len (#53962)

This commit is contained in:
Nick Hill
2026-08-27 08:30:22 -07:00
committed by GitHub
parent f18e29834b
commit 76c0c6530e
3 changed files with 57 additions and 6 deletions
+11 -6
View File
@@ -964,14 +964,19 @@ class Scheduler(SchedulerInterface):
and num_new_tokens == 1
and (scheduled_running_reqs and not prefill_scheduled)
):
num_new_tokens = 1 + self.num_spec_tokens
padded_num_tokens = 1 + self.num_spec_tokens
# Pad only when there is room for the sampled token(s).
if (
num_new_tokens > request_token_budget
or num_computed_tokens + num_new_tokens > self.max_model_len
num_computed_tokens
+ padded_num_tokens
+ self.num_sampled_tokens_per_step
<= self.max_model_len
):
# Prefer to not schedule than schedule un-padded here.
break
pad_spec_decode = True
if padded_num_tokens > request_token_budget:
# Prefer to not schedule than schedule un-padded.
break
num_new_tokens = padded_num_tokens
pad_spec_decode = True
threshold = self.scheduler_config.long_prefill_token_threshold
if 0 < threshold < num_new_tokens: