[Bugfix][Spec Decode] Trim token_ids/logprobs left past a stop string under speculative decoding (#47616)

Signed-off-by: Ting Sun <suntcrick@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ting SUN
2026-10-09 14:22:31 -07:00
committed by GitHub
co-authored by Nick Hill Claude Opus 5.5
parent 35e920d671
commit 51144dd83e
3 changed files with 158 additions and 13 deletions
+30 -4
View File
@@ -736,17 +736,22 @@ class OutputProcessor:
if pooling_output is None:
assert req_state.detokenizer is not None
assert req_state.logprobs_processor is not None
if engine_core_output.new_sampling_mask is not None:
req_state.sampling_mask_chunks.append(
engine_core_output.new_sampling_mask
)
# 2) Detokenize the token ids into text and perform stop checks.
num_prev_tokens = req_state.detokenizer.num_output_tokens()
stop_string = req_state.detokenizer.update(
new_token_ids, finish_reason == FinishReason.STOP
)
if stop_string:
finish_reason = FinishReason.STOP
stop_reason = stop_string
self._trim_surplus_tokens(
req_state, engine_core_output, num_prev_tokens
)
if engine_core_output.new_sampling_mask is not None:
req_state.sampling_mask_chunks.append(
engine_core_output.new_sampling_mask
)
# 3) Compute sample and prompt logprobs for request,
# if required.
@@ -799,6 +804,27 @@ class OutputProcessor:
reqs_to_abort=reqs_to_abort,
)
@staticmethod
def _trim_surplus_tokens(
req_state: RequestState, output: EngineCoreOutput, num_prev_tokens: int
) -> None:
"""Drop new tokens generated past a stop string matched by the
detokenizer, e.g. when multiple tokens are accepted in one step."""
assert req_state.detokenizer is not None
num_kept = req_state.detokenizer.num_output_tokens() - num_prev_tokens
num_dropped = len(output.new_token_ids) - num_kept
if num_dropped <= 0:
return
del output.new_token_ids[num_kept:]
if output.new_logprobs is not None:
output.new_logprobs = output.new_logprobs.slice_request(0, num_kept)
if output.new_sampling_mask is not None:
output.new_sampling_mask = output.new_sampling_mask.slice_request(
0, num_kept
)
if output.routed_experts is not None:
req_state.routed_experts_chunks[-1] = output.routed_experts[:-num_dropped]
def _finish_request(self, req_state: RequestState) -> None:
req_id = req_state.request_id
self.request_states.pop(req_id)