[Bugfix] Fix resumable request + async scheduling handoff race (#58259)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-09-28 15:26:57 -04:00
committed by GitHub
parent 764413559a
commit 2f4a975901
2 changed files with 56 additions and 1 deletions
+10
View File
@@ -2403,6 +2403,16 @@ class Scheduler(SchedulerInterface):
if not request.resumable:
return True
# Drop the finished turn's in-flight work and resume from the
# materialized frontier.
safe_frontier = request.num_computed_tokens - request.num_output_placeholders
assert safe_frontier >= 0
request.num_computed_tokens = safe_frontier
request.spec_token_ids = []
request.drop_stale_output = True
request.num_stale_output_tokens = request.num_in_flight_tokens
request.num_output_placeholders = 0
if request.streaming_queue:
update = request.streaming_queue.popleft()
if update is None: