[Bugfix][Scheduler] Preserve logprobs across streaming continuations (#57447)
Signed-off-by: 0xsensei <prblmslvr.aditya@gmail.com>
This commit is contained in:
@@ -2152,6 +2152,15 @@ class Scheduler(SchedulerInterface):
|
||||
self.kv_cache_manager.estimate_cached_tokens(request)
|
||||
)
|
||||
|
||||
# Extract sample logprobs before stop handling can replace the
|
||||
# sampling parameters for a streaming continuation.
|
||||
if (
|
||||
request.sampling_params is not None
|
||||
and request.sampling_params.num_logprobs is not None
|
||||
and logprobs
|
||||
):
|
||||
new_logprobs = logprobs.slice_request(req_index, len(new_token_ids))
|
||||
|
||||
finish_reason = None
|
||||
if stopped:
|
||||
# Capture finish_reason BEFORE _handle_stopped_request, which may
|
||||
@@ -2177,14 +2186,6 @@ class Scheduler(SchedulerInterface):
|
||||
else:
|
||||
stopped_preempted_reqs.add(request)
|
||||
|
||||
# Extract sample logprobs if needed.
|
||||
if (
|
||||
request.sampling_params is not None
|
||||
and request.sampling_params.num_logprobs is not None
|
||||
and logprobs
|
||||
):
|
||||
new_logprobs = logprobs.slice_request(req_index, len(new_token_ids))
|
||||
|
||||
if self.return_sampling_mask:
|
||||
sampling_masks = model_runner_output.sampling_masks
|
||||
if new_token_ids and sampling_masks is not None:
|
||||
|
||||
Reference in New Issue
Block a user