[BugFix] Fix num_output_placeholders preemption underflow (#48245)

Signed-off-by: Chris Eastwood <chris.eastwood@pwn4g3.dev>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Chris Eastwood <chris.eastwood@pwn4g3.dev>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Nick Hill
2026-07-29 06:41:54 -07:00
committed by GitHub
co-authored by Chris Eastwood Yifan Qiao
parent 43eaefba5a
commit a0c092ee72
5 changed files with 385 additions and 77 deletions
+6 -11
View File
@@ -49,23 +49,18 @@ class AsyncScheduler(Scheduler):
request.next_decode_eligible_step = self.current_step + self.pp_size
def _update_request_with_output(
self, request: Request, new_token_ids: list[int]
self, request: Request, new_token_ids: list[int], is_stale: bool = False
) -> tuple[list[int], bool]:
if request.async_tokens_to_discard > 0:
# The request was force-preempted in reset_prefix_cache; drop one
# stale in-flight async output frame per call until the counter
# is drained.
request.async_tokens_to_discard -= 1
return [], False
status_before_update = request.status
new_token_ids, stopped = super()._update_request_with_output(
request, new_token_ids
)
# Update the number of output placeholders.
request.num_output_placeholders -= len(new_token_ids)
assert request.num_output_placeholders >= 0
# Placeholders were zeroed at preemption; a stale delivery must not
# decrement them (it would underflow).
if not is_stale:
request.num_output_placeholders -= len(new_token_ids)
assert request.num_output_placeholders >= 0
# Cache the new tokens. Preempted requests should be skipped.
if status_before_update == RequestStatus.RUNNING:
+56 -29
View File
@@ -694,6 +694,17 @@ class Scheduler(SchedulerInterface):
step_skipped_waiting.prepend_request(request)
continue
if (
request.num_stale_output_tokens > 0
and not request.drop_stale_output
):
# Deliverable stale output still in flight: resuming now
# could resample a position that output later delivers.
# It drains within the pipeline depth.
request_queue.pop_request()
step_skipped_waiting.prepend_request(request)
continue
# Check that adding the request still respects the max_loras
# constraint.
if (
@@ -1244,11 +1255,17 @@ class Scheduler(SchedulerInterface):
return new_block_ids_to_zero or None
def _preempt_request(self, request: Request, timestamp: float) -> None:
def _preempt_request(
self, request: Request, timestamp: float, drop_stale_output: bool = False
) -> None:
"""Preempt a request and put it back to the waiting queue.
NOTE: The request should be popped from the running queue outside of this
method.
drop_stale_output: drop (rather than deliver) any in-flight output; used
by reset_prefix_cache, whose same-step resume would otherwise deliver
tokens out of order.
"""
assert request.status == RequestStatus.RUNNING, (
"Only running requests can be preempted"
@@ -1260,6 +1277,18 @@ class Scheduler(SchedulerInterface):
request.num_computed_tokens = 0
if request.spec_token_ids:
request.spec_token_ids = []
# Async scheduling: mark all in-flight output as stale. Its tokens are
# still delivered on return (dropping them would perturb spec-decode
# acceptance) but must not mutate the reset counters; each step drains
# its share in update_from_output. num_in_flight_tokens already
# includes any undrained stale share, so assign rather than accumulate.
# An undrained drop-mode share stays dropped: its positions have
# already been resampled.
request.drop_stale_output = drop_stale_output or (
request.drop_stale_output and request.num_stale_output_tokens > 0
)
request.num_stale_output_tokens = request.num_in_flight_tokens
request.num_output_placeholders = 0
request.num_preemptions += 1
if self.log_stats:
request.record_event(EngineCoreEventType.PREEMPTED, timestamp)
@@ -1687,8 +1716,14 @@ class Scheduler(SchedulerInterface):
for req_id, num_tokens_scheduled in num_scheduled_tokens.items():
assert num_tokens_scheduled > 0
request = self.requests.get(req_id)
output_is_stale = False
if request is not None:
request.num_in_flight_tokens -= num_tokens_scheduled
# Drain any stale share (see _preempt_request) in lockstep.
if request.num_stale_output_tokens > 0:
output_is_stale = True
request.num_stale_output_tokens -= num_tokens_scheduled
assert request.num_stale_output_tokens >= 0
if failed_kv_load_req_ids and req_id in failed_kv_load_req_ids:
# skip failed or rescheduled requests from KV load failure
continue
@@ -1702,6 +1737,10 @@ class Scheduler(SchedulerInterface):
# In this case, we use is_finished() to check.
continue
# Drop-mode stale output (same-step resume) is discarded entirely.
if output_is_stale and request.drop_stale_output:
continue
req_index = model_runner_output.req_id_to_index[req_id]
generated_token_ids = (
sampled_token_ids[req_index] if sampled_token_ids else []
@@ -1710,28 +1749,22 @@ class Scheduler(SchedulerInterface):
scheduled_spec_token_ids = (
scheduler_output.scheduled_spec_decode_tokens.get(req_id)
)
# Skip a stale frame still pending discard (async_tokens_to_discard
# > 0): its pre-reset rejection count would underflow the counters.
if (
scheduled_spec_token_ids
and (generated_token_ids or self.num_sampled_tokens_per_step == 0)
and request.async_tokens_to_discard == 0
if scheduled_spec_token_ids and (
generated_token_ids or self.num_sampled_tokens_per_step == 0
):
num_draft_tokens = len(scheduled_spec_token_ids)
num_sampled = self.num_sampled_tokens_per_step
num_accepted = max(len(generated_token_ids) - num_sampled, 0)
num_rejected = num_draft_tokens - num_accepted
# num_computed_tokens represents the number of tokens
# processed in the current step, considering scheduled
# tokens and rejections. If some tokens are rejected,
# num_computed_tokens is decreased by the number of rejected
# tokens.
if request.num_computed_tokens > 0:
request.num_computed_tokens -= num_rejected
# If async scheduling, num_output_placeholders also includes
# the scheduled spec tokens count and so is similarly adjusted.
if request.num_output_placeholders > 0:
request.num_output_placeholders -= num_rejected
# Rejections roll back num_computed_tokens (and, under async
# scheduling, num_output_placeholders, which covers the spec
# tokens). A stale rejection count predates the preemption
# rollback and must not apply.
if not output_is_stale:
if request.num_computed_tokens > 0:
request.num_computed_tokens -= num_rejected
if request.num_output_placeholders > 0:
request.num_output_placeholders -= num_rejected
spec_decoding_stats = self.make_spec_decoding_stats(
spec_decoding_stats,
num_draft_tokens=num_draft_tokens,
@@ -1757,7 +1790,7 @@ class Scheduler(SchedulerInterface):
# Check for stop and update request status.
if new_token_ids:
new_token_ids, stopped = self._update_request_with_output(
request, new_token_ids
request, new_token_ids, is_stale=output_is_stale
)
elif request.pooling_params and pooler_output is not None:
# Pooling stops as soon as there is output.
@@ -1899,6 +1932,7 @@ class Scheduler(SchedulerInterface):
if stopped_preempted_reqs:
# This is a rare case and unlikely to impact performance.
self.waiting.remove_requests(stopped_preempted_reqs)
self.skipped_waiting.remove_requests(stopped_preempted_reqs)
error_req_ids = set(self.grammar_compile_error_reqs)
self.grammar_compile_error_reqs.clear()
@@ -2041,8 +2075,9 @@ class Scheduler(SchedulerInterface):
return False
def _update_request_with_output(
self, request: Request, new_token_ids: list[int]
self, request: Request, new_token_ids: list[int], is_stale: bool = False
) -> tuple[list[int], bool]:
# is_stale is only used by the AsyncScheduler override.
# Append generated tokens and check for stop. Note that if
# a request is still being prefilled, we expect the model runner
# to return empty token ids for the request.
@@ -2388,15 +2423,7 @@ class Scheduler(SchedulerInterface):
# running queue in FIFO order.
while self.running:
request = self.running.pop()
self._preempt_request(request, timestamp)
# For async scheduling, any output frames already in flight at
# preemption time are now stale and must be discarded when they
# return. num_output_placeholders is exactly that count: 0 if
# the engine has drained (e.g. pause_generation(keep) waited
# for idle), 1 for vanilla async mid-step, or 1 + spec/PP frames
# otherwise.
request.async_tokens_to_discard = request.num_output_placeholders
request.num_output_placeholders = 0
self._preempt_request(request, timestamp, drop_stale_output=True)
# Clear scheduled request ids cache. Since we are forcing preemption
# + resumption in the same step, we must act as if these requests were
+6 -1
View File
@@ -149,7 +149,12 @@ class Request:
# Used in async scheduling.
self.num_output_placeholders = 0
self.async_tokens_to_discard = 0
# Tokens of output in flight when the request was preempted: delivered
# on return, but must not mutate the reset counters.
self.num_stale_output_tokens = 0
# Drop the stale output instead, for same-step preempt + resume
# (reset_prefix_cache).
self.drop_stale_output = False
# Tokens of steps whose output is not yet processed (async scheduling
# and PP run ahead of the GPU); `num_computed_tokens` counts them