[Bugfix] Fix logprobs token-string collision from SentencePiece space… (#48674)
Signed-off-by: Allen Shen <aoshen@inferact.ai> Co-authored-by: mvanhorn <mvanhorn@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mvanhorn
Claude Opus 4.6
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
752bd10647
commit
dcfebf93f4
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
@@ -56,6 +59,63 @@ def _convert_tokens_to_string_with_added_encoders(
|
||||
INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5
|
||||
|
||||
|
||||
_CACHED_MARKER_KEY = "_vllm_space_marker_cache"
|
||||
_NOT_CACHED = "__not_computed__"
|
||||
|
||||
|
||||
def _get_leading_space_marker(tokenizer: TokenizerLike) -> str | None:
|
||||
"""Read the space marker from the tokenizer's pre_tokenizer config.
|
||||
|
||||
Only Metaspace pre_tokenizers (used by SentencePiece-based models like
|
||||
Llama, Mistral, T5) have a replacement character whose leading instance
|
||||
gets stripped by decode(). ByteLevel (GPT-2), BertPreTokenizer (BERT),
|
||||
and others do not have this issue.
|
||||
|
||||
Returns the marker character, or None if decode() is safe for single
|
||||
tokens.
|
||||
"""
|
||||
cached = getattr(tokenizer, _CACHED_MARKER_KEY, _NOT_CACHED)
|
||||
if cached is not _NOT_CACHED:
|
||||
return cached # type: ignore[return-value]
|
||||
|
||||
backend = getattr(tokenizer, "backend_tokenizer", None)
|
||||
if backend is None:
|
||||
result = None
|
||||
else:
|
||||
result = None
|
||||
try:
|
||||
config = json.loads(backend.to_str())
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
pre = config.get("pre_tokenizer") or {}
|
||||
pre_type = pre.get("type")
|
||||
if pre_type == "Metaspace":
|
||||
result = pre.get("replacement", "▁")
|
||||
elif pre_type == "Sequence":
|
||||
for sub in pre.get("pretokenizers", []):
|
||||
if sub.get("type") == "Metaspace":
|
||||
result = sub.get("replacement", "▁")
|
||||
break
|
||||
|
||||
setattr(tokenizer, _CACHED_MARKER_KEY, result)
|
||||
return result
|
||||
|
||||
|
||||
def _restore_leading_spaces(raw_token: str, token_str: str, marker: str) -> str:
|
||||
"""Restore leading spaces that decode() stripped from a raw vocab piece."""
|
||||
num_markers = 0
|
||||
for ch in raw_token:
|
||||
if ch != marker:
|
||||
break
|
||||
num_markers += 1
|
||||
if num_markers == 0:
|
||||
return token_str
|
||||
existing = len(token_str) - len(token_str.lstrip(" "))
|
||||
missing = num_markers - existing
|
||||
return " " * missing + token_str if missing > 0 else token_str
|
||||
|
||||
|
||||
def convert_prompt_ids_to_tokens(
|
||||
tokenizer: TokenizerLike,
|
||||
prompt_ids: list[int],
|
||||
@@ -86,6 +146,10 @@ def convert_ids_list_to_tokens(
|
||||
) -> list[str]:
|
||||
"""Detokenize the input ids individually.
|
||||
|
||||
Uses decode() for human-readable output, then checks the raw vocab
|
||||
piece via convert_ids_to_tokens() to restore any leading spaces that
|
||||
decode() stripped (SentencePiece add_dummy_prefix inverse).
|
||||
|
||||
Args:
|
||||
tokenizer: tokenizer used by model under test
|
||||
token_ids: convert these tokens (Python list form)
|
||||
@@ -94,14 +158,16 @@ def convert_ids_list_to_tokens(
|
||||
Python list of token string representations
|
||||
|
||||
"""
|
||||
token_str_lst = []
|
||||
for token_id in token_ids:
|
||||
# use default skip_special_tokens.
|
||||
token_str = tokenizer.decode([token_id])
|
||||
if token_str is None:
|
||||
token_str = ""
|
||||
token_str_lst.append(token_str)
|
||||
return token_str_lst
|
||||
if not token_ids:
|
||||
return []
|
||||
marker = _get_leading_space_marker(tokenizer)
|
||||
if marker is None:
|
||||
return [tokenizer.decode([tid]) or "" for tid in token_ids]
|
||||
raw_tokens = tokenizer.convert_ids_to_tokens(token_ids)
|
||||
return [
|
||||
_restore_leading_spaces(raw, tokenizer.decode([tid]) or "", marker)
|
||||
for tid, raw in zip(token_ids, raw_tokens)
|
||||
]
|
||||
|
||||
|
||||
# Based on
|
||||
|
||||
Reference in New Issue
Block a user