[Bugfix][Tokenizer] Fix off-by-one max_token_id from vocab_size (#59491)

Signed-off-by: Shijie Lyu <lshjhf@gmail.com>
This commit is contained in:
Shijie Lyu
2026-10-01 05:43:30 +00:00
committed by GitHub
parent b2d3a3b9ad
commit 37cecc8141
2 changed files with 19 additions and 2 deletions
+3 -2
View File
@@ -124,10 +124,11 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
# Some tokenizers (e.g., QwenTokenizer) have special tokens that
# are added and included in the implementation of the vocab_size
# property, but not in get_vocab(); if there is an implementation
# of vocab size, we should take the greater value.
# of vocab size, we should take the greater value. vocab_size is a
# count, so the largest id it implies is vocab_size - 1.
if hasattr(tokenizer, "vocab_size"):
with contextlib.suppress(NotImplementedError):
max_token_id = max(max_token_id, tokenizer.vocab_size)
max_token_id = max(max_token_id, tokenizer.vocab_size - 1)
class CachedTokenizer(tokenizer.__class__): # type: ignore
@property