[Bugfix] Fix logprobs token-string collision from SentencePiece space… (#48674)

Signed-off-by: Allen Shen <aoshen@inferact.ai>
Co-authored-by: mvanhorn <mvanhorn@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
aoshen02
2026-07-20 12:17:18 +08:00
committed by GitHub
co-authored by mvanhorn Claude Opus 4.6 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 752bd10647
commit dcfebf93f4
2 changed files with 194 additions and 8 deletions
+74 -8
View File
@@ -1,6 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import json
from vllm.tokenizers import TokenizerLike
@@ -56,6 +59,63 @@ def _convert_tokens_to_string_with_added_encoders(
INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5
_CACHED_MARKER_KEY = "_vllm_space_marker_cache"
_NOT_CACHED = "__not_computed__"
def _get_leading_space_marker(tokenizer: TokenizerLike) -> str | None:
"""Read the space marker from the tokenizer's pre_tokenizer config.
Only Metaspace pre_tokenizers (used by SentencePiece-based models like
Llama, Mistral, T5) have a replacement character whose leading instance
gets stripped by decode(). ByteLevel (GPT-2), BertPreTokenizer (BERT),
and others do not have this issue.
Returns the marker character, or None if decode() is safe for single
tokens.
"""
cached = getattr(tokenizer, _CACHED_MARKER_KEY, _NOT_CACHED)
if cached is not _NOT_CACHED:
return cached # type: ignore[return-value]
backend = getattr(tokenizer, "backend_tokenizer", None)
if backend is None:
result = None
else:
result = None
try:
config = json.loads(backend.to_str())
except Exception:
pass
else:
pre = config.get("pre_tokenizer") or {}
pre_type = pre.get("type")
if pre_type == "Metaspace":
result = pre.get("replacement", "▁")
elif pre_type == "Sequence":
for sub in pre.get("pretokenizers", []):
if sub.get("type") == "Metaspace":
result = sub.get("replacement", "▁")
break
setattr(tokenizer, _CACHED_MARKER_KEY, result)
return result
def _restore_leading_spaces(raw_token: str, token_str: str, marker: str) -> str:
"""Restore leading spaces that decode() stripped from a raw vocab piece."""
num_markers = 0
for ch in raw_token:
if ch != marker:
break
num_markers += 1
if num_markers == 0:
return token_str
existing = len(token_str) - len(token_str.lstrip(" "))
missing = num_markers - existing
return " " * missing + token_str if missing > 0 else token_str
def convert_prompt_ids_to_tokens(
tokenizer: TokenizerLike,
prompt_ids: list[int],
@@ -86,6 +146,10 @@ def convert_ids_list_to_tokens(
) -> list[str]:
"""Detokenize the input ids individually.
Uses decode() for human-readable output, then checks the raw vocab
piece via convert_ids_to_tokens() to restore any leading spaces that
decode() stripped (SentencePiece add_dummy_prefix inverse).
Args:
tokenizer: tokenizer used by model under test
token_ids: convert these tokens (Python list form)
@@ -94,14 +158,16 @@ def convert_ids_list_to_tokens(
Python list of token string representations
"""
token_str_lst = []
for token_id in token_ids:
# use default skip_special_tokens.
token_str = tokenizer.decode([token_id])
if token_str is None:
token_str = ""
token_str_lst.append(token_str)
return token_str_lst
if not token_ids:
return []
marker = _get_leading_space_marker(tokenizer)
if marker is None:
return [tokenizer.decode([tid]) or "" for tid in token_ids]
raw_tokens = tokenizer.convert_ids_to_tokens(token_ids)
return [
_restore_leading_spaces(raw, tokenizer.decode([tid]) or "", marker)
for tid, raw in zip(token_ids, raw_tokens)
]
# Based on