[Bugfix][Core] Uncount stop-trimmed spec tokens from num_computed_tokens (#58668)
Signed-off-by: David Cheung <d7cheung@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Nick Hill
parent
49f944be3e
commit
d486219fc2
@@ -60,14 +60,16 @@ class AsyncScheduler(Scheduler):
|
||||
self, request: Request, new_token_ids: list[int], is_stale: bool = False
|
||||
) -> tuple[list[int], bool]:
|
||||
status_before_update = request.status
|
||||
# Tokens trimmed at a stop resolve their placeholders too.
|
||||
num_new_tokens = len(new_token_ids)
|
||||
new_token_ids, stopped = super()._update_request_with_output(
|
||||
request, new_token_ids
|
||||
request, new_token_ids, is_stale
|
||||
)
|
||||
|
||||
# Placeholders were zeroed at preemption; a stale delivery must not
|
||||
# decrement them (it would underflow).
|
||||
if not is_stale:
|
||||
request.num_output_placeholders -= len(new_token_ids)
|
||||
request.num_output_placeholders -= num_new_tokens
|
||||
assert request.num_output_placeholders >= 0
|
||||
|
||||
# Cache the new tokens. Preempted requests should be skipped.
|
||||
|
||||
@@ -2574,7 +2574,6 @@ class Scheduler(SchedulerInterface):
|
||||
def _update_request_with_output(
|
||||
self, request: Request, new_token_ids: list[int], is_stale: bool = False
|
||||
) -> tuple[list[int], bool]:
|
||||
# is_stale is only used by the AsyncScheduler override.
|
||||
# Append generated tokens and check for stop. Note that if
|
||||
# a request is still being prefilled, we expect the model runner
|
||||
# to return empty token ids for the request.
|
||||
@@ -2586,6 +2585,10 @@ class Scheduler(SchedulerInterface):
|
||||
# This must be called before we make the EngineCoreOutput.
|
||||
stopped = check_stop(request, self.max_model_len)
|
||||
if stopped:
|
||||
# num_computed_tokens covers all new tokens but the last one.
|
||||
# Keep that true after trimming: resumable requests resume from it.
|
||||
if not is_stale:
|
||||
request.num_computed_tokens -= len(new_token_ids) - num_new
|
||||
del new_token_ids[num_new:] # Trim new tokens if needed.
|
||||
break
|
||||
return new_token_ids, stopped
|
||||
|
||||
Reference in New Issue
Block a user