[BugFix] Fix num_output_placeholders preemption underflow (#48245)
Signed-off-by: Chris Eastwood <chris.eastwood@pwn4g3.dev> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Chris Eastwood <chris.eastwood@pwn4g3.dev> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Chris Eastwood
Yifan Qiao
parent
43eaefba5a
commit
a0c092ee72
@@ -49,23 +49,18 @@ class AsyncScheduler(Scheduler):
|
||||
request.next_decode_eligible_step = self.current_step + self.pp_size
|
||||
|
||||
def _update_request_with_output(
|
||||
self, request: Request, new_token_ids: list[int]
|
||||
self, request: Request, new_token_ids: list[int], is_stale: bool = False
|
||||
) -> tuple[list[int], bool]:
|
||||
if request.async_tokens_to_discard > 0:
|
||||
# The request was force-preempted in reset_prefix_cache; drop one
|
||||
# stale in-flight async output frame per call until the counter
|
||||
# is drained.
|
||||
request.async_tokens_to_discard -= 1
|
||||
return [], False
|
||||
|
||||
status_before_update = request.status
|
||||
new_token_ids, stopped = super()._update_request_with_output(
|
||||
request, new_token_ids
|
||||
)
|
||||
|
||||
# Update the number of output placeholders.
|
||||
request.num_output_placeholders -= len(new_token_ids)
|
||||
assert request.num_output_placeholders >= 0
|
||||
# Placeholders were zeroed at preemption; a stale delivery must not
|
||||
# decrement them (it would underflow).
|
||||
if not is_stale:
|
||||
request.num_output_placeholders -= len(new_token_ids)
|
||||
assert request.num_output_placeholders >= 0
|
||||
|
||||
# Cache the new tokens. Preempted requests should be skipped.
|
||||
if status_before_update == RequestStatus.RUNNING:
|
||||
|
||||
@@ -694,6 +694,17 @@ class Scheduler(SchedulerInterface):
|
||||
step_skipped_waiting.prepend_request(request)
|
||||
continue
|
||||
|
||||
if (
|
||||
request.num_stale_output_tokens > 0
|
||||
and not request.drop_stale_output
|
||||
):
|
||||
# Deliverable stale output still in flight: resuming now
|
||||
# could resample a position that output later delivers.
|
||||
# It drains within the pipeline depth.
|
||||
request_queue.pop_request()
|
||||
step_skipped_waiting.prepend_request(request)
|
||||
continue
|
||||
|
||||
# Check that adding the request still respects the max_loras
|
||||
# constraint.
|
||||
if (
|
||||
@@ -1244,11 +1255,17 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
return new_block_ids_to_zero or None
|
||||
|
||||
def _preempt_request(self, request: Request, timestamp: float) -> None:
|
||||
def _preempt_request(
|
||||
self, request: Request, timestamp: float, drop_stale_output: bool = False
|
||||
) -> None:
|
||||
"""Preempt a request and put it back to the waiting queue.
|
||||
|
||||
NOTE: The request should be popped from the running queue outside of this
|
||||
method.
|
||||
|
||||
drop_stale_output: drop (rather than deliver) any in-flight output; used
|
||||
by reset_prefix_cache, whose same-step resume would otherwise deliver
|
||||
tokens out of order.
|
||||
"""
|
||||
assert request.status == RequestStatus.RUNNING, (
|
||||
"Only running requests can be preempted"
|
||||
@@ -1260,6 +1277,18 @@ class Scheduler(SchedulerInterface):
|
||||
request.num_computed_tokens = 0
|
||||
if request.spec_token_ids:
|
||||
request.spec_token_ids = []
|
||||
# Async scheduling: mark all in-flight output as stale. Its tokens are
|
||||
# still delivered on return (dropping them would perturb spec-decode
|
||||
# acceptance) but must not mutate the reset counters; each step drains
|
||||
# its share in update_from_output. num_in_flight_tokens already
|
||||
# includes any undrained stale share, so assign rather than accumulate.
|
||||
# An undrained drop-mode share stays dropped: its positions have
|
||||
# already been resampled.
|
||||
request.drop_stale_output = drop_stale_output or (
|
||||
request.drop_stale_output and request.num_stale_output_tokens > 0
|
||||
)
|
||||
request.num_stale_output_tokens = request.num_in_flight_tokens
|
||||
request.num_output_placeholders = 0
|
||||
request.num_preemptions += 1
|
||||
if self.log_stats:
|
||||
request.record_event(EngineCoreEventType.PREEMPTED, timestamp)
|
||||
@@ -1687,8 +1716,14 @@ class Scheduler(SchedulerInterface):
|
||||
for req_id, num_tokens_scheduled in num_scheduled_tokens.items():
|
||||
assert num_tokens_scheduled > 0
|
||||
request = self.requests.get(req_id)
|
||||
output_is_stale = False
|
||||
if request is not None:
|
||||
request.num_in_flight_tokens -= num_tokens_scheduled
|
||||
# Drain any stale share (see _preempt_request) in lockstep.
|
||||
if request.num_stale_output_tokens > 0:
|
||||
output_is_stale = True
|
||||
request.num_stale_output_tokens -= num_tokens_scheduled
|
||||
assert request.num_stale_output_tokens >= 0
|
||||
if failed_kv_load_req_ids and req_id in failed_kv_load_req_ids:
|
||||
# skip failed or rescheduled requests from KV load failure
|
||||
continue
|
||||
@@ -1702,6 +1737,10 @@ class Scheduler(SchedulerInterface):
|
||||
# In this case, we use is_finished() to check.
|
||||
continue
|
||||
|
||||
# Drop-mode stale output (same-step resume) is discarded entirely.
|
||||
if output_is_stale and request.drop_stale_output:
|
||||
continue
|
||||
|
||||
req_index = model_runner_output.req_id_to_index[req_id]
|
||||
generated_token_ids = (
|
||||
sampled_token_ids[req_index] if sampled_token_ids else []
|
||||
@@ -1710,28 +1749,22 @@ class Scheduler(SchedulerInterface):
|
||||
scheduled_spec_token_ids = (
|
||||
scheduler_output.scheduled_spec_decode_tokens.get(req_id)
|
||||
)
|
||||
# Skip a stale frame still pending discard (async_tokens_to_discard
|
||||
# > 0): its pre-reset rejection count would underflow the counters.
|
||||
if (
|
||||
scheduled_spec_token_ids
|
||||
and (generated_token_ids or self.num_sampled_tokens_per_step == 0)
|
||||
and request.async_tokens_to_discard == 0
|
||||
if scheduled_spec_token_ids and (
|
||||
generated_token_ids or self.num_sampled_tokens_per_step == 0
|
||||
):
|
||||
num_draft_tokens = len(scheduled_spec_token_ids)
|
||||
num_sampled = self.num_sampled_tokens_per_step
|
||||
num_accepted = max(len(generated_token_ids) - num_sampled, 0)
|
||||
num_rejected = num_draft_tokens - num_accepted
|
||||
# num_computed_tokens represents the number of tokens
|
||||
# processed in the current step, considering scheduled
|
||||
# tokens and rejections. If some tokens are rejected,
|
||||
# num_computed_tokens is decreased by the number of rejected
|
||||
# tokens.
|
||||
if request.num_computed_tokens > 0:
|
||||
request.num_computed_tokens -= num_rejected
|
||||
# If async scheduling, num_output_placeholders also includes
|
||||
# the scheduled spec tokens count and so is similarly adjusted.
|
||||
if request.num_output_placeholders > 0:
|
||||
request.num_output_placeholders -= num_rejected
|
||||
# Rejections roll back num_computed_tokens (and, under async
|
||||
# scheduling, num_output_placeholders, which covers the spec
|
||||
# tokens). A stale rejection count predates the preemption
|
||||
# rollback and must not apply.
|
||||
if not output_is_stale:
|
||||
if request.num_computed_tokens > 0:
|
||||
request.num_computed_tokens -= num_rejected
|
||||
if request.num_output_placeholders > 0:
|
||||
request.num_output_placeholders -= num_rejected
|
||||
spec_decoding_stats = self.make_spec_decoding_stats(
|
||||
spec_decoding_stats,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
@@ -1757,7 +1790,7 @@ class Scheduler(SchedulerInterface):
|
||||
# Check for stop and update request status.
|
||||
if new_token_ids:
|
||||
new_token_ids, stopped = self._update_request_with_output(
|
||||
request, new_token_ids
|
||||
request, new_token_ids, is_stale=output_is_stale
|
||||
)
|
||||
elif request.pooling_params and pooler_output is not None:
|
||||
# Pooling stops as soon as there is output.
|
||||
@@ -1899,6 +1932,7 @@ class Scheduler(SchedulerInterface):
|
||||
if stopped_preempted_reqs:
|
||||
# This is a rare case and unlikely to impact performance.
|
||||
self.waiting.remove_requests(stopped_preempted_reqs)
|
||||
self.skipped_waiting.remove_requests(stopped_preempted_reqs)
|
||||
|
||||
error_req_ids = set(self.grammar_compile_error_reqs)
|
||||
self.grammar_compile_error_reqs.clear()
|
||||
@@ -2041,8 +2075,9 @@ class Scheduler(SchedulerInterface):
|
||||
return False
|
||||
|
||||
def _update_request_with_output(
|
||||
self, request: Request, new_token_ids: list[int]
|
||||
self, request: Request, new_token_ids: list[int], is_stale: bool = False
|
||||
) -> tuple[list[int], bool]:
|
||||
# is_stale is only used by the AsyncScheduler override.
|
||||
# Append generated tokens and check for stop. Note that if
|
||||
# a request is still being prefilled, we expect the model runner
|
||||
# to return empty token ids for the request.
|
||||
@@ -2388,15 +2423,7 @@ class Scheduler(SchedulerInterface):
|
||||
# running queue in FIFO order.
|
||||
while self.running:
|
||||
request = self.running.pop()
|
||||
self._preempt_request(request, timestamp)
|
||||
# For async scheduling, any output frames already in flight at
|
||||
# preemption time are now stale and must be discarded when they
|
||||
# return. num_output_placeholders is exactly that count: 0 if
|
||||
# the engine has drained (e.g. pause_generation(keep) waited
|
||||
# for idle), 1 for vanilla async mid-step, or 1 + spec/PP frames
|
||||
# otherwise.
|
||||
request.async_tokens_to_discard = request.num_output_placeholders
|
||||
request.num_output_placeholders = 0
|
||||
self._preempt_request(request, timestamp, drop_stale_output=True)
|
||||
|
||||
# Clear scheduled request ids cache. Since we are forcing preemption
|
||||
# + resumption in the same step, we must act as if these requests were
|
||||
|
||||
+6
-1
@@ -149,7 +149,12 @@ class Request:
|
||||
|
||||
# Used in async scheduling.
|
||||
self.num_output_placeholders = 0
|
||||
self.async_tokens_to_discard = 0
|
||||
# Tokens of output in flight when the request was preempted: delivered
|
||||
# on return, but must not mutate the reset counters.
|
||||
self.num_stale_output_tokens = 0
|
||||
# Drop the stale output instead, for same-step preempt + resume
|
||||
# (reset_prefix_cache).
|
||||
self.drop_stale_output = False
|
||||
|
||||
# Tokens of steps whose output is not yet processed (async scheduling
|
||||
# and PP run ahead of the GPU); `num_computed_tokens` counts them
|
||||
|
||||
Reference in New Issue
Block a user