[Bugfix][Scheduler] Preserve logprobs across streaming continuations (#57447)

Signed-off-by: 0xsensei <prblmslvr.aditya@gmail.com>
This commit is contained in:
0xsensei
2026-09-28 19:50:27 +00:00
committed by GitHub
parent e7c903609f
commit 2840ca7a33
2 changed files with 66 additions and 9 deletions
+9 -8
View File
@@ -2152,6 +2152,15 @@ class Scheduler(SchedulerInterface):
self.kv_cache_manager.estimate_cached_tokens(request)
)
# Extract sample logprobs before stop handling can replace the
# sampling parameters for a streaming continuation.
if (
request.sampling_params is not None
and request.sampling_params.num_logprobs is not None
and logprobs
):
new_logprobs = logprobs.slice_request(req_index, len(new_token_ids))
finish_reason = None
if stopped:
# Capture finish_reason BEFORE _handle_stopped_request, which may
@@ -2177,14 +2186,6 @@ class Scheduler(SchedulerInterface):
else:
stopped_preempted_reqs.add(request)
# Extract sample logprobs if needed.
if (
request.sampling_params is not None
and request.sampling_params.num_logprobs is not None
and logprobs
):
new_logprobs = logprobs.slice_request(req_index, len(new_token_ids))
if self.return_sampling_mask:
sampling_masks = model_runner_output.sampling_masks
if new_token_ids and sampling_masks is not None: