[Performance] Load HF tokenizers and processors from a local snapshot (#60825)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Harry Mellor
2026-10-12 13:45:54 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Kevin H. Luu
parent 092c35dd15
commit 3facf822b7
5 changed files with 146 additions and 4 deletions
+16
View File
@@ -1,11 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import _get_protocol_attrs # type: ignore
from unittest.mock import patch
import pytest
from transformers import (
PreTrainedTokenizerBase,
TokenizersBackend,
tokenization_utils_base,
)
from vllm.tokenizers import TokenizerLike, get_tokenizer
@@ -56,6 +58,20 @@ def test_tokenizer_revision(tokenizer_name: str):
get_tokenizer(tokenizer_name, revision="never")
def test_hub_tokenizer_loads_from_local_snapshot():
"""Transformers makes a Hub request per file when given a repo ID."""
with patch.object(
tokenization_utils_base,
"list_repo_templates",
wraps=tokenization_utils_base.list_repo_templates,
) as list_repo_templates:
tokenizer = get_tokenizer("openai-community/gpt2")
assert isinstance(tokenizer, PreTrainedTokenizerBase)
assert tokenizer.name_or_path == "openai-community/gpt2"
list_repo_templates.assert_not_called()
@pytest.mark.parametrize("tokenizer_name", ["BAAI/bge-base-en"])
@pytest.mark.parametrize("n_tokens", [510])
def test_special_tokens(tokenizer_name: str, n_tokens: int):