[Bugfix][Tokenizer] Fix off-by-one max_token_id from vocab_size (#59491)

Signed-off-by: Shijie Lyu <lshjhf@gmail.com>
This commit is contained in:
Shijie Lyu
2026-10-01 05:43:30 +00:00
committed by GitHub
parent b2d3a3b9ad
commit 37cecc8141
2 changed files with 19 additions and 2 deletions
+16
View File
@@ -65,3 +65,19 @@ def test_thread_pool_tokenizer_pickle(model_id: str):
# Idempotence: wrapping an already-pooled tokenizer returns it unchanged.
assert maybe_make_thread_pool(pooled_tokenizer) is pooled_tokenizer # type: ignore[type-var]
@pytest.mark.parametrize("model_id", ["openai-community/gpt2"])
def test_cached_tokenizer_max_token_id_is_a_valid_id(model_id: str):
"""max_token_id is the largest valid id, one less than vocab_size."""
reference_tokenizer = AutoTokenizer.from_pretrained(model_id)
cached_tokenizer = get_cached_tokenizer(deepcopy(reference_tokenizer))
expected = max(
max(reference_tokenizer.get_vocab().values()),
reference_tokenizer.vocab_size - 1,
)
assert cached_tokenizer.max_token_id == expected
assert cached_tokenizer.max_token_id == max(
reference_tokenizer.get_vocab().values()
)