[Bugfix][Core] Uncount stop-trimmed spec tokens from num_computed_tokens (#58668)

Signed-off-by: David Cheung <d7cheung@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
David Cheung
2026-10-15 12:50:06 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Nick Hill
parent 49f944be3e
commit d486219fc2
3 changed files with 49 additions and 4 deletions
+4 -2
View File
@@ -60,14 +60,16 @@ class AsyncScheduler(Scheduler):
self, request: Request, new_token_ids: list[int], is_stale: bool = False
) -> tuple[list[int], bool]:
status_before_update = request.status
# Tokens trimmed at a stop resolve their placeholders too.
num_new_tokens = len(new_token_ids)
new_token_ids, stopped = super()._update_request_with_output(
request, new_token_ids
request, new_token_ids, is_stale
)
# Placeholders were zeroed at preemption; a stale delivery must not
# decrement them (it would underflow).
if not is_stale:
request.num_output_placeholders -= len(new_token_ids)
request.num_output_placeholders -= num_new_tokens
assert request.num_output_placeholders >= 0
# Cache the new tokens. Preempted requests should be skipped.
+4 -1
View File
@@ -2574,7 +2574,6 @@ class Scheduler(SchedulerInterface):
def _update_request_with_output(
self, request: Request, new_token_ids: list[int], is_stale: bool = False
) -> tuple[list[int], bool]:
# is_stale is only used by the AsyncScheduler override.
# Append generated tokens and check for stop. Note that if
# a request is still being prefilled, we expect the model runner
# to return empty token ids for the request.
@@ -2586,6 +2585,10 @@ class Scheduler(SchedulerInterface):
# This must be called before we make the EngineCoreOutput.
stopped = check_stop(request, self.max_model_len)
if stopped:
# num_computed_tokens covers all new tokens but the last one.
# Keep that true after trimming: resumable requests resume from it.
if not is_stale:
request.num_computed_tokens -= len(new_token_ids) - num_new
del new_token_ids[num_new:] # Trim new tokens if needed.
break
return new_token_ids, stopped