[Bugfix] Fix V1 logprobs empty strings for multi-byte UTF-8 tokens when logprobs > 0 (#34875)

Signed-off-by: haosdent <haosdent@gmail.com>
This commit is contained in:
haosdent
2026-04-08 15:30:00 +00:00
committed by GitHub
parent 1a2c17634e
commit 8904fc4d19
2 changed files with 346 additions and 194 deletions
+131 -24
View File
@@ -7,6 +7,7 @@ from dataclasses import dataclass
from vllm.logger import init_logger
from vllm.logprobs import (
FlatLogprobs,
PromptLogprobs,
SampleLogprobs,
append_logprobs_for_next_position,
@@ -96,8 +97,11 @@ class LogprobsProcessor:
decoded_tokens_list = convert_ids_list_to_tokens(
self.tokenizer, token_ids
)
context_token_ids = self._get_sampled_context_ids(self.logprobs)
decoded_tokens = self._verify_tokens(
decoded_tokens_list=decoded_tokens_list, tokens=token_ids
decoded_tokens_list=decoded_tokens_list,
tokens=token_ids,
context_token_ids=context_token_ids,
)
# Sampler puts the sampled logprob in first.
@@ -162,9 +166,14 @@ class LogprobsProcessor:
else:
# Extract decoded tokens for this position
decoded_tokens_slice = all_decoded_tokens[offset:offset_end]
# Context: preceding prompt tokens accumulated in
# self.prompt_logprobs from previous loop iterations.
context_token_ids = self._get_sampled_context_ids(self.prompt_logprobs)
# Apply UTF-8 correction within this position's token boundaries
decoded_tokens_for_pos = self._verify_tokens(
decoded_tokens_list=decoded_tokens_slice, tokens=token_ids_list[pos]
decoded_tokens_list=decoded_tokens_slice,
tokens=token_ids_list[pos],
context_token_ids=context_token_ids,
)
# Update with the Logprob container for this pos.
@@ -196,41 +205,139 @@ class LogprobsProcessor:
self.prompt_logprobs = []
return plp
def _correct_decoded_token(self, idx: int, tokens: list[int]) -> str:
assert self.tokenizer is not None, "self.tokenizer should not be None"
@staticmethod
def _get_sampled_context_ids(
logprobs_source: SampleLogprobs | PromptLogprobs | None,
max_context: int = 4,
) -> list[int]:
"""Extract recent sampled token IDs from a logprobs source.
# try with prev token id in same list
if idx > 0:
possible_decoded_token = self.tokenizer.decode(tokens[idx - 1 : idx + 1])
if not possible_decoded_token.endswith("�"):
return possible_decoded_token
# try with previous logprob token id
if self.logprobs:
latest_token_id = next(iter(self.logprobs[-1]))
The sampled (or prompt) token at each position is the first
entry, since it is always inserted first by
append_logprobs_for_next_position.
decode_ids = [latest_token_id]
if idx > 0:
decode_ids.extend(tokens[idx - 1 : idx + 1])
Args:
logprobs_source: The logprobs container to extract from.
max_context: Maximum number of preceding tokens to return.
4 is sufficient for any UTF-8 multi-byte sequence.
Returns:
List of sampled token IDs, oldest first, most recent last.
"""
if not logprobs_source:
return []
n = len(logprobs_source)
start = max(0, n - max_context)
# Efficient path for FlatLogprobs: access token_ids directly.
if isinstance(logprobs_source, FlatLogprobs):
return [
logprobs_source.token_ids[logprobs_source.start_indices[i]]
for i in range(start, n)
if logprobs_source.start_indices[i] < logprobs_source.end_indices[i]
]
# list[dict] path
result: list[int] = []
for i in range(start, n):
entry = logprobs_source[i]
if entry is not None:
result.append(next(iter(entry)))
return result
def _correct_decoded_token(
self, token_id: int, context_token_ids: list[int]
) -> str:
"""Correct a decoded token that contains the replacement character.
When byte-fallback tokenization splits multi-byte UTF-8
characters across tokens, individual token decoding produces
the replacement character U+FFFD. This method uses preceding
sampled tokens as context to reconstruct the correct text.
Args:
token_id: The single token ID to correct.
context_token_ids: Preceding sampled token IDs in sequential
order (oldest first). These are the actual tokens in
the generated sequence, NOT top-k alternatives.
Returns:
The corrected decoded string, or empty string if the byte
sequence is genuinely incomplete at this point.
"""
assert self.tokenizer is not None
max_ctx = min(len(context_token_ids), 4)
for num_ctx in range(1, max_ctx + 1):
context = context_token_ids[-num_ctx:]
full_decoded = self.tokenizer.decode(context + [token_id])
if full_decoded.endswith("�"):
continue
# Find the boundary between "clean" context tokens and
# byte-fallback tokens that are part of the same incomplete
# sequence. Byte-fallback context tokens returned "" when
# they were processed, so their text must be attributed to
# this completing token.
clean_end = len(context)
for j in range(len(context) - 1, -1, -1):
if self.tokenizer.decode([context[j]]).endswith("�"):
clean_end = j
else:
break
# Decode only the clean (non-byte-fallback) prefix.
if clean_end > 0:
clean_prefix = self.tokenizer.decode(context[:clean_end])
else:
decode_ids.extend(tokens[idx : idx + 1])
clean_prefix = ""
possible_decoded_token = self.tokenizer.decode(decode_ids)
if not possible_decoded_token.endswith("�"):
return possible_decoded_token
if full_decoded.startswith(clean_prefix):
return full_decoded[len(clean_prefix) :]
# Tokenizer normalization may cause prefix mismatch.
# Find the longest common prefix between them.
common_len = 0
for a, b in zip(clean_prefix, full_decoded):
if a != b:
break
common_len += 1
return full_decoded[common_len:]
# by default return empty string
return ""
def _verify_tokens(
self, decoded_tokens_list: list[str], tokens: list[int]
self,
decoded_tokens_list: list[str],
tokens: list[int],
context_token_ids: list[int] | None = None,
) -> list[str]:
"""Verify and correct decoded tokens with replacement characters.
Args:
decoded_tokens_list: Decoded token strings to verify.
tokens: Token IDs corresponding to decoded_tokens_list.
These are alternatives at the SAME position (e.g.
[sampled, top1, top2]), NOT sequential tokens.
context_token_ids: Preceding sampled token IDs providing
sequential context. If None, extracted from
self.logprobs.
"""
if context_token_ids is None:
context_token_ids = self._get_sampled_context_ids(self.logprobs)
corrected_decoded_token_map = dict()
for idx, text in enumerate(decoded_tokens_list):
if text.endswith("�"):
# utf-8 char at the end means it's a potential unfinished byte sequence
# from byte fallback tokenization.
# Replacement char at the end means a potential
# unfinished byte sequence from byte-fallback
# tokenization. Correct each token independently
# using only the sequential context.
corrected_decoded_token_map[idx] = self._correct_decoded_token(
idx, tokens
tokens[idx], context_token_ids
)
for idx, text in corrected_decoded_token_map.items():