[Bugfix][Tokenizer] Fix off-by-one max_token_id from vocab_size (#59491)

Signed-off-by: Shijie Lyu <lshjhf@gmail.com>

tests/tokenizers_/test_hf.py

@@ -65,3 +65,19 @@ def test_thread_pool_tokenizer_pickle(model_id: str):
 
     # Idempotence: wrapping an already-pooled tokenizer returns it unchanged.
     assert maybe_make_thread_pool(pooled_tokenizer) is pooled_tokenizer  # type: ignore[type-var]
+
+
+@pytest.mark.parametrize("model_id", ["openai-community/gpt2"])
+def test_cached_tokenizer_max_token_id_is_a_valid_id(model_id: str):
+    """max_token_id is the largest valid id, one less than vocab_size."""
+    reference_tokenizer = AutoTokenizer.from_pretrained(model_id)
+    cached_tokenizer = get_cached_tokenizer(deepcopy(reference_tokenizer))
+
+    expected = max(
+        max(reference_tokenizer.get_vocab().values()),
+        reference_tokenizer.vocab_size - 1,
+    )
+    assert cached_tokenizer.max_token_id == expected
+    assert cached_tokenizer.max_token_id == max(
+        reference_tokenizer.get_vocab().values()
+    )

vllm/tokenizers/hf.py

@@ -124,10 +124,11 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
     # Some tokenizers (e.g., QwenTokenizer) have special tokens that
     # are added and included in the implementation of the vocab_size
     # property, but not in get_vocab(); if there is an implementation
-    # of vocab size, we should take the greater value.
+    # of vocab size, we should take the greater value. vocab_size is a
+    # count, so the largest id it implies is vocab_size - 1.
     if hasattr(tokenizer, "vocab_size"):
         with contextlib.suppress(NotImplementedError):
-            max_token_id = max(max_token_id, tokenizer.vocab_size)
+            max_token_id = max(max_token_id, tokenizer.vocab_size - 1)
 
     class CachedTokenizer(tokenizer.__class__):  # type: ignore
         @property