[Bugfix] Fix logprobs token-string collision from SentencePiece space… (#48674)

Signed-off-by: Allen Shen <aoshen@inferact.ai>
Co-authored-by: mvanhorn <mvanhorn@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
aoshen02
2026-07-20 12:17:18 +08:00
committed by GitHub
co-authored by mvanhorn Claude Opus 4.6 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 752bd10647
commit dcfebf93f4
2 changed files with 194 additions and 8 deletions
+120
View File
@@ -8,6 +8,7 @@ import pytest
from transformers import AutoTokenizer, PythonBackend, TokenizersBackend
from vllm.sampling_params import SamplingParams
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.detokenizer import (
@@ -239,3 +240,122 @@ def test_oov_decode(tokenizer, fast):
assert decoded_text == ""
assert out_ids == [len(tokenizer)]
# ---------- convert_ids_list_to_tokens collision tests ----------
class _MockBackend:
"""Fake backend_tokenizer that exposes pre_tokenizer config."""
def __init__(self, pre_tokenizer_type, replacement=None):
import json
pre: dict = {"type": pre_tokenizer_type}
if replacement is not None:
pre["replacement"] = replacement
self._config = json.dumps({"pre_tokenizer": pre})
def to_str(self):
return self._config
class _MockTokenizer:
"""Minimal tokenizer mock for testing convert_ids_list_to_tokens."""
def __init__(
self,
raw_tokens: dict[int, str],
decoded_tokens: dict[int, str],
pre_tokenizer_type: str = "Metaspace",
replacement: str | None = "▁",
):
self._raw = raw_tokens
self._decoded = decoded_tokens
self.backend_tokenizer = _MockBackend(pre_tokenizer_type, replacement)
def convert_ids_to_tokens(
self, ids: list[int], skip_special_tokens: bool = False
) -> list[str]:
return [self._raw[tid] for tid in ids]
def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
return "".join(self._decoded[tid] for tid in ids)
def test_sentencepiece_leading_space_preserved():
"""▁true and true must produce distinct strings."""
tok = _MockTokenizer(
raw_tokens={0: "▁true", 1: "true", 2: "▁false", 3: "false"},
decoded_tokens={0: "true", 1: "true", 2: "false", 3: "false"},
)
result = convert_ids_list_to_tokens(tok, [0, 1, 2, 3])
assert result == [" true", "true", " false", "false"]
# No dict collision when used as top_logprobs keys
logprobs = dict(zip(result, [-0.1, -0.2, -0.3, -0.4]))
assert len(logprobs) == 4
def test_whitespace_run_tokens_stay_distinct():
"""▁, ▁▁, ▁▁▁ must produce different-length space strings."""
tok = _MockTokenizer(
raw_tokens={0: "▁", 1: "▁▁", 2: "▁▁▁"},
decoded_tokens={0: "", 1: " ", 2: " "},
)
result = convert_ids_list_to_tokens(tok, [0, 1, 2])
assert result == [" ", " ", " "]
def test_bpe_leading_space_already_preserved():
"""GPT-2 BPE: Ġtrue already decodes to ' true', no fix needed."""
tok = _MockTokenizer(
raw_tokens={0: "Ġtrue", 1: "true"},
decoded_tokens={0: " true", 1: "true"},
pre_tokenizer_type="ByteLevel",
replacement=None,
)
result = convert_ids_list_to_tokens(tok, [0, 1])
assert result == [" true", "true"]
def test_logprobs_count_stable_across_k():
"""logprobs=4 and logprobs=10 must return 4 and 10 entries."""
tok = _MockTokenizer(
raw_tokens={
0: "▁true",
1: "a",
2: "b",
3: "c",
4: "true",
5: "d",
6: "e",
7: "f",
8: "g",
9: "h",
},
decoded_tokens={
0: "true",
1: "a",
2: "b",
3: "c",
4: "true",
5: "d",
6: "e",
7: "f",
8: "g",
9: "h",
},
)
ids = list(range(10))
lps = [-0.1 * (i + 1) for i in range(10)]
tokens4 = convert_ids_list_to_tokens(tok, ids[:4])
top4 = dict(zip(tokens4, lps[:4]))
tokens10 = convert_ids_list_to_tokens(tok, ids)
top10 = dict(zip(tokens10, lps))
assert len(top4) == 4
assert len(top10) == 10
assert top4[" true"] == top10[" true"]