[Bugfix][Spec Decode] Trim token_ids/logprobs left past a stop string under speculative decoding (#47616)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Nick Hill
Claude Opus 5.5
parent
35e920d671
commit
51144dd83e
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import sys
|
||||
from abc import ABC, abstractmethod
|
||||
from bisect import bisect_left
|
||||
|
||||
import tokenizers.decoders
|
||||
from tokenizers import Tokenizer
|
||||
@@ -112,21 +113,27 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
|
||||
skipped_stop_token_id = None
|
||||
|
||||
# 1) Detokenize the new token ids incrementally.
|
||||
stop_check_offset = len(self.output_text)
|
||||
num_prev_tokens = self.num_output_tokens()
|
||||
first_new_token_index = len(self.token_ids)
|
||||
# Output text offsets at the boundaries of the new tokens.
|
||||
token_offsets = [len(self.output_text)] if self.stop else None
|
||||
for new_token_id in new_token_ids:
|
||||
self.token_ids.append(new_token_id)
|
||||
self.output_text += self.decode_next(new_token_id)
|
||||
# Support min_tokens, see https://github.com/vllm-project/vllm/pull/22014
|
||||
if self.min_tokens and self.num_output_tokens() <= self.min_tokens:
|
||||
stop_check_offset = len(self.output_text)
|
||||
if token_offsets is not None:
|
||||
token_offsets.append(len(self.output_text))
|
||||
|
||||
if skipped_stop_token_id is not None:
|
||||
# Cleanup after skipping detokenization.
|
||||
self.token_ids.append(skipped_stop_token_id)
|
||||
if token_offsets is not None:
|
||||
token_offsets.append(len(self.output_text))
|
||||
|
||||
# 2) Evaluate stop strings.
|
||||
stop_string = None
|
||||
if self.stop and self.num_output_tokens() > self.min_tokens:
|
||||
if token_offsets is not None and self.num_output_tokens() > self.min_tokens:
|
||||
# Support min_tokens, see https://github.com/vllm-project/vllm/pull/22014
|
||||
stop_check_offset = token_offsets[max(self.min_tokens - num_prev_tokens, 0)]
|
||||
stop = check_stop_strings(
|
||||
output_text=self.output_text,
|
||||
new_char_count=len(self.output_text) - stop_check_offset,
|
||||
@@ -135,6 +142,13 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
|
||||
)
|
||||
if stop is not None:
|
||||
stop_string, truncate_to = stop
|
||||
stop_end = len(self.output_text) if truncate_to == -1 else truncate_to
|
||||
if not self.include_stop_str_in_output:
|
||||
stop_end += len(stop_string)
|
||||
# Drop tokens generated after the one completing the stop string
|
||||
# (possible when multiple tokens are appended in one step).
|
||||
num_kept = bisect_left(token_offsets, stop_end, lo=1)
|
||||
del self.token_ids[first_new_token_index + num_kept :]
|
||||
if truncate_to != -1:
|
||||
self.output_text = self.output_text[:truncate_to]
|
||||
|
||||
|
||||
@@ -736,17 +736,22 @@ class OutputProcessor:
|
||||
if pooling_output is None:
|
||||
assert req_state.detokenizer is not None
|
||||
assert req_state.logprobs_processor is not None
|
||||
if engine_core_output.new_sampling_mask is not None:
|
||||
req_state.sampling_mask_chunks.append(
|
||||
engine_core_output.new_sampling_mask
|
||||
)
|
||||
# 2) Detokenize the token ids into text and perform stop checks.
|
||||
num_prev_tokens = req_state.detokenizer.num_output_tokens()
|
||||
stop_string = req_state.detokenizer.update(
|
||||
new_token_ids, finish_reason == FinishReason.STOP
|
||||
)
|
||||
if stop_string:
|
||||
finish_reason = FinishReason.STOP
|
||||
stop_reason = stop_string
|
||||
self._trim_surplus_tokens(
|
||||
req_state, engine_core_output, num_prev_tokens
|
||||
)
|
||||
|
||||
if engine_core_output.new_sampling_mask is not None:
|
||||
req_state.sampling_mask_chunks.append(
|
||||
engine_core_output.new_sampling_mask
|
||||
)
|
||||
|
||||
# 3) Compute sample and prompt logprobs for request,
|
||||
# if required.
|
||||
@@ -799,6 +804,27 @@ class OutputProcessor:
|
||||
reqs_to_abort=reqs_to_abort,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _trim_surplus_tokens(
|
||||
req_state: RequestState, output: EngineCoreOutput, num_prev_tokens: int
|
||||
) -> None:
|
||||
"""Drop new tokens generated past a stop string matched by the
|
||||
detokenizer, e.g. when multiple tokens are accepted in one step."""
|
||||
assert req_state.detokenizer is not None
|
||||
num_kept = req_state.detokenizer.num_output_tokens() - num_prev_tokens
|
||||
num_dropped = len(output.new_token_ids) - num_kept
|
||||
if num_dropped <= 0:
|
||||
return
|
||||
del output.new_token_ids[num_kept:]
|
||||
if output.new_logprobs is not None:
|
||||
output.new_logprobs = output.new_logprobs.slice_request(0, num_kept)
|
||||
if output.new_sampling_mask is not None:
|
||||
output.new_sampling_mask = output.new_sampling_mask.slice_request(
|
||||
0, num_kept
|
||||
)
|
||||
if output.routed_experts is not None:
|
||||
req_state.routed_experts_chunks[-1] = output.routed_experts[:-num_dropped]
|
||||
|
||||
def _finish_request(self, req_state: RequestState) -> None:
|
||||
req_id = req_state.request_id
|
||||
self.request_states.pop(req_id)
|
||||
|
||||
Reference in New Issue
Block a user