[Bugfix][Tokenizer] Fix off-by-one max_token_id from vocab_size (#59491)
Signed-off-by: Shijie Lyu <lshjhf@gmail.com>
This commit is contained in:
@@ -124,10 +124,11 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
|
||||
# Some tokenizers (e.g., QwenTokenizer) have special tokens that
|
||||
# are added and included in the implementation of the vocab_size
|
||||
# property, but not in get_vocab(); if there is an implementation
|
||||
# of vocab size, we should take the greater value.
|
||||
# of vocab size, we should take the greater value. vocab_size is a
|
||||
# count, so the largest id it implies is vocab_size - 1.
|
||||
if hasattr(tokenizer, "vocab_size"):
|
||||
with contextlib.suppress(NotImplementedError):
|
||||
max_token_id = max(max_token_id, tokenizer.vocab_size)
|
||||
max_token_id = max(max_token_id, tokenizer.vocab_size - 1)
|
||||
|
||||
class CachedTokenizer(tokenizer.__class__): # type: ignore
|
||||
@property
|
||||
|
||||
Reference in New Issue
Block a user