[Bugfix] Fix logprobs token-string collision from SentencePiece space… (#48674)
Signed-off-by: Allen Shen <aoshen@inferact.ai> Co-authored-by: mvanhorn <mvanhorn@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mvanhorn
Claude Opus 4.6
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
752bd10647
commit
dcfebf93f4
@@ -8,6 +8,7 @@ import pytest
|
||||
from transformers import AutoTokenizer, PythonBackend, TokenizersBackend
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.detokenizer import (
|
||||
@@ -239,3 +240,122 @@ def test_oov_decode(tokenizer, fast):
|
||||
|
||||
assert decoded_text == ""
|
||||
assert out_ids == [len(tokenizer)]
|
||||
|
||||
|
||||
# ---------- convert_ids_list_to_tokens collision tests ----------
|
||||
|
||||
|
||||
class _MockBackend:
|
||||
"""Fake backend_tokenizer that exposes pre_tokenizer config."""
|
||||
|
||||
def __init__(self, pre_tokenizer_type, replacement=None):
|
||||
import json
|
||||
|
||||
pre: dict = {"type": pre_tokenizer_type}
|
||||
if replacement is not None:
|
||||
pre["replacement"] = replacement
|
||||
self._config = json.dumps({"pre_tokenizer": pre})
|
||||
|
||||
def to_str(self):
|
||||
return self._config
|
||||
|
||||
|
||||
class _MockTokenizer:
|
||||
"""Minimal tokenizer mock for testing convert_ids_list_to_tokens."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
raw_tokens: dict[int, str],
|
||||
decoded_tokens: dict[int, str],
|
||||
pre_tokenizer_type: str = "Metaspace",
|
||||
replacement: str | None = "▁",
|
||||
):
|
||||
self._raw = raw_tokens
|
||||
self._decoded = decoded_tokens
|
||||
self.backend_tokenizer = _MockBackend(pre_tokenizer_type, replacement)
|
||||
|
||||
def convert_ids_to_tokens(
|
||||
self, ids: list[int], skip_special_tokens: bool = False
|
||||
) -> list[str]:
|
||||
return [self._raw[tid] for tid in ids]
|
||||
|
||||
def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
|
||||
return "".join(self._decoded[tid] for tid in ids)
|
||||
|
||||
|
||||
def test_sentencepiece_leading_space_preserved():
|
||||
"""▁true and true must produce distinct strings."""
|
||||
tok = _MockTokenizer(
|
||||
raw_tokens={0: "▁true", 1: "true", 2: "▁false", 3: "false"},
|
||||
decoded_tokens={0: "true", 1: "true", 2: "false", 3: "false"},
|
||||
)
|
||||
result = convert_ids_list_to_tokens(tok, [0, 1, 2, 3])
|
||||
assert result == [" true", "true", " false", "false"]
|
||||
|
||||
# No dict collision when used as top_logprobs keys
|
||||
logprobs = dict(zip(result, [-0.1, -0.2, -0.3, -0.4]))
|
||||
assert len(logprobs) == 4
|
||||
|
||||
|
||||
def test_whitespace_run_tokens_stay_distinct():
|
||||
"""▁, ▁▁, ▁▁▁ must produce different-length space strings."""
|
||||
tok = _MockTokenizer(
|
||||
raw_tokens={0: "▁", 1: "▁▁", 2: "▁▁▁"},
|
||||
decoded_tokens={0: "", 1: " ", 2: " "},
|
||||
)
|
||||
result = convert_ids_list_to_tokens(tok, [0, 1, 2])
|
||||
assert result == [" ", " ", " "]
|
||||
|
||||
|
||||
def test_bpe_leading_space_already_preserved():
|
||||
"""GPT-2 BPE: Ġtrue already decodes to ' true', no fix needed."""
|
||||
tok = _MockTokenizer(
|
||||
raw_tokens={0: "Ġtrue", 1: "true"},
|
||||
decoded_tokens={0: " true", 1: "true"},
|
||||
pre_tokenizer_type="ByteLevel",
|
||||
replacement=None,
|
||||
)
|
||||
result = convert_ids_list_to_tokens(tok, [0, 1])
|
||||
assert result == [" true", "true"]
|
||||
|
||||
|
||||
def test_logprobs_count_stable_across_k():
|
||||
"""logprobs=4 and logprobs=10 must return 4 and 10 entries."""
|
||||
tok = _MockTokenizer(
|
||||
raw_tokens={
|
||||
0: "▁true",
|
||||
1: "a",
|
||||
2: "b",
|
||||
3: "c",
|
||||
4: "true",
|
||||
5: "d",
|
||||
6: "e",
|
||||
7: "f",
|
||||
8: "g",
|
||||
9: "h",
|
||||
},
|
||||
decoded_tokens={
|
||||
0: "true",
|
||||
1: "a",
|
||||
2: "b",
|
||||
3: "c",
|
||||
4: "true",
|
||||
5: "d",
|
||||
6: "e",
|
||||
7: "f",
|
||||
8: "g",
|
||||
9: "h",
|
||||
},
|
||||
)
|
||||
ids = list(range(10))
|
||||
lps = [-0.1 * (i + 1) for i in range(10)]
|
||||
|
||||
tokens4 = convert_ids_list_to_tokens(tok, ids[:4])
|
||||
top4 = dict(zip(tokens4, lps[:4]))
|
||||
|
||||
tokens10 = convert_ids_list_to_tokens(tok, ids)
|
||||
top10 = dict(zip(tokens10, lps))
|
||||
|
||||
assert len(top4) == 4
|
||||
assert len(top10) == 10
|
||||
assert top4[" true"] == top10[" true"]
|
||||
|
||||
Reference in New Issue
Block a user