[Bugfix] Fix V1 logprobs empty strings for multi-byte UTF-8 tokens when logprobs > 0 (#34875)
Signed-off-by: haosdent <haosdent@gmail.com>
This commit is contained in:
+131
-24
@@ -7,6 +7,7 @@ from dataclasses import dataclass
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import (
|
||||
FlatLogprobs,
|
||||
PromptLogprobs,
|
||||
SampleLogprobs,
|
||||
append_logprobs_for_next_position,
|
||||
@@ -96,8 +97,11 @@ class LogprobsProcessor:
|
||||
decoded_tokens_list = convert_ids_list_to_tokens(
|
||||
self.tokenizer, token_ids
|
||||
)
|
||||
context_token_ids = self._get_sampled_context_ids(self.logprobs)
|
||||
decoded_tokens = self._verify_tokens(
|
||||
decoded_tokens_list=decoded_tokens_list, tokens=token_ids
|
||||
decoded_tokens_list=decoded_tokens_list,
|
||||
tokens=token_ids,
|
||||
context_token_ids=context_token_ids,
|
||||
)
|
||||
|
||||
# Sampler puts the sampled logprob in first.
|
||||
@@ -162,9 +166,14 @@ class LogprobsProcessor:
|
||||
else:
|
||||
# Extract decoded tokens for this position
|
||||
decoded_tokens_slice = all_decoded_tokens[offset:offset_end]
|
||||
# Context: preceding prompt tokens accumulated in
|
||||
# self.prompt_logprobs from previous loop iterations.
|
||||
context_token_ids = self._get_sampled_context_ids(self.prompt_logprobs)
|
||||
# Apply UTF-8 correction within this position's token boundaries
|
||||
decoded_tokens_for_pos = self._verify_tokens(
|
||||
decoded_tokens_list=decoded_tokens_slice, tokens=token_ids_list[pos]
|
||||
decoded_tokens_list=decoded_tokens_slice,
|
||||
tokens=token_ids_list[pos],
|
||||
context_token_ids=context_token_ids,
|
||||
)
|
||||
|
||||
# Update with the Logprob container for this pos.
|
||||
@@ -196,41 +205,139 @@ class LogprobsProcessor:
|
||||
self.prompt_logprobs = []
|
||||
return plp
|
||||
|
||||
def _correct_decoded_token(self, idx: int, tokens: list[int]) -> str:
|
||||
assert self.tokenizer is not None, "self.tokenizer should not be None"
|
||||
@staticmethod
|
||||
def _get_sampled_context_ids(
|
||||
logprobs_source: SampleLogprobs | PromptLogprobs | None,
|
||||
max_context: int = 4,
|
||||
) -> list[int]:
|
||||
"""Extract recent sampled token IDs from a logprobs source.
|
||||
|
||||
# try with prev token id in same list
|
||||
if idx > 0:
|
||||
possible_decoded_token = self.tokenizer.decode(tokens[idx - 1 : idx + 1])
|
||||
if not possible_decoded_token.endswith("�"):
|
||||
return possible_decoded_token
|
||||
# try with previous logprob token id
|
||||
if self.logprobs:
|
||||
latest_token_id = next(iter(self.logprobs[-1]))
|
||||
The sampled (or prompt) token at each position is the first
|
||||
entry, since it is always inserted first by
|
||||
append_logprobs_for_next_position.
|
||||
|
||||
decode_ids = [latest_token_id]
|
||||
if idx > 0:
|
||||
decode_ids.extend(tokens[idx - 1 : idx + 1])
|
||||
Args:
|
||||
logprobs_source: The logprobs container to extract from.
|
||||
max_context: Maximum number of preceding tokens to return.
|
||||
4 is sufficient for any UTF-8 multi-byte sequence.
|
||||
|
||||
Returns:
|
||||
List of sampled token IDs, oldest first, most recent last.
|
||||
"""
|
||||
if not logprobs_source:
|
||||
return []
|
||||
|
||||
n = len(logprobs_source)
|
||||
start = max(0, n - max_context)
|
||||
|
||||
# Efficient path for FlatLogprobs: access token_ids directly.
|
||||
if isinstance(logprobs_source, FlatLogprobs):
|
||||
return [
|
||||
logprobs_source.token_ids[logprobs_source.start_indices[i]]
|
||||
for i in range(start, n)
|
||||
if logprobs_source.start_indices[i] < logprobs_source.end_indices[i]
|
||||
]
|
||||
|
||||
# list[dict] path
|
||||
result: list[int] = []
|
||||
for i in range(start, n):
|
||||
entry = logprobs_source[i]
|
||||
if entry is not None:
|
||||
result.append(next(iter(entry)))
|
||||
return result
|
||||
|
||||
def _correct_decoded_token(
|
||||
self, token_id: int, context_token_ids: list[int]
|
||||
) -> str:
|
||||
"""Correct a decoded token that contains the replacement character.
|
||||
|
||||
When byte-fallback tokenization splits multi-byte UTF-8
|
||||
characters across tokens, individual token decoding produces
|
||||
the replacement character U+FFFD. This method uses preceding
|
||||
sampled tokens as context to reconstruct the correct text.
|
||||
|
||||
Args:
|
||||
token_id: The single token ID to correct.
|
||||
context_token_ids: Preceding sampled token IDs in sequential
|
||||
order (oldest first). These are the actual tokens in
|
||||
the generated sequence, NOT top-k alternatives.
|
||||
|
||||
Returns:
|
||||
The corrected decoded string, or empty string if the byte
|
||||
sequence is genuinely incomplete at this point.
|
||||
"""
|
||||
assert self.tokenizer is not None
|
||||
|
||||
max_ctx = min(len(context_token_ids), 4)
|
||||
|
||||
for num_ctx in range(1, max_ctx + 1):
|
||||
context = context_token_ids[-num_ctx:]
|
||||
full_decoded = self.tokenizer.decode(context + [token_id])
|
||||
|
||||
if full_decoded.endswith("�"):
|
||||
continue
|
||||
|
||||
# Find the boundary between "clean" context tokens and
|
||||
# byte-fallback tokens that are part of the same incomplete
|
||||
# sequence. Byte-fallback context tokens returned "" when
|
||||
# they were processed, so their text must be attributed to
|
||||
# this completing token.
|
||||
clean_end = len(context)
|
||||
for j in range(len(context) - 1, -1, -1):
|
||||
if self.tokenizer.decode([context[j]]).endswith("�"):
|
||||
clean_end = j
|
||||
else:
|
||||
break
|
||||
|
||||
# Decode only the clean (non-byte-fallback) prefix.
|
||||
if clean_end > 0:
|
||||
clean_prefix = self.tokenizer.decode(context[:clean_end])
|
||||
else:
|
||||
decode_ids.extend(tokens[idx : idx + 1])
|
||||
clean_prefix = ""
|
||||
|
||||
possible_decoded_token = self.tokenizer.decode(decode_ids)
|
||||
if not possible_decoded_token.endswith("�"):
|
||||
return possible_decoded_token
|
||||
if full_decoded.startswith(clean_prefix):
|
||||
return full_decoded[len(clean_prefix) :]
|
||||
|
||||
# Tokenizer normalization may cause prefix mismatch.
|
||||
# Find the longest common prefix between them.
|
||||
common_len = 0
|
||||
for a, b in zip(clean_prefix, full_decoded):
|
||||
if a != b:
|
||||
break
|
||||
common_len += 1
|
||||
return full_decoded[common_len:]
|
||||
|
||||
# by default return empty string
|
||||
return ""
|
||||
|
||||
def _verify_tokens(
|
||||
self, decoded_tokens_list: list[str], tokens: list[int]
|
||||
self,
|
||||
decoded_tokens_list: list[str],
|
||||
tokens: list[int],
|
||||
context_token_ids: list[int] | None = None,
|
||||
) -> list[str]:
|
||||
"""Verify and correct decoded tokens with replacement characters.
|
||||
|
||||
Args:
|
||||
decoded_tokens_list: Decoded token strings to verify.
|
||||
tokens: Token IDs corresponding to decoded_tokens_list.
|
||||
These are alternatives at the SAME position (e.g.
|
||||
[sampled, top1, top2]), NOT sequential tokens.
|
||||
context_token_ids: Preceding sampled token IDs providing
|
||||
sequential context. If None, extracted from
|
||||
self.logprobs.
|
||||
"""
|
||||
if context_token_ids is None:
|
||||
context_token_ids = self._get_sampled_context_ids(self.logprobs)
|
||||
|
||||
corrected_decoded_token_map = dict()
|
||||
for idx, text in enumerate(decoded_tokens_list):
|
||||
if text.endswith("�"):
|
||||
# utf-8 char at the end means it's a potential unfinished byte sequence
|
||||
# from byte fallback tokenization.
|
||||
# Replacement char at the end means a potential
|
||||
# unfinished byte sequence from byte-fallback
|
||||
# tokenization. Correct each token independently
|
||||
# using only the sequential context.
|
||||
corrected_decoded_token_map[idx] = self._correct_decoded_token(
|
||||
idx, tokens
|
||||
tokens[idx], context_token_ids
|
||||
)
|
||||
|
||||
for idx, text in corrected_decoded_token_map.items():
|
||||
|
||||
Reference in New Issue
Block a user