[Performance] Load HF tokenizers and processors from a local snapshot (#60825)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Kevin H. Luu
parent
092c35dd15
commit
3facf822b7
@@ -1,11 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import _get_protocol_attrs # type: ignore
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from transformers import (
|
||||
PreTrainedTokenizerBase,
|
||||
TokenizersBackend,
|
||||
tokenization_utils_base,
|
||||
)
|
||||
|
||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||
@@ -56,6 +58,20 @@ def test_tokenizer_revision(tokenizer_name: str):
|
||||
get_tokenizer(tokenizer_name, revision="never")
|
||||
|
||||
|
||||
def test_hub_tokenizer_loads_from_local_snapshot():
|
||||
"""Transformers makes a Hub request per file when given a repo ID."""
|
||||
with patch.object(
|
||||
tokenization_utils_base,
|
||||
"list_repo_templates",
|
||||
wraps=tokenization_utils_base.list_repo_templates,
|
||||
) as list_repo_templates:
|
||||
tokenizer = get_tokenizer("openai-community/gpt2")
|
||||
|
||||
assert isinstance(tokenizer, PreTrainedTokenizerBase)
|
||||
assert tokenizer.name_or_path == "openai-community/gpt2"
|
||||
list_repo_templates.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tokenizer_name", ["BAAI/bge-base-en"])
|
||||
@pytest.mark.parametrize("n_tokens", [510])
|
||||
def test_special_tokens(tokenizer_name: str, n_tokens: int):
|
||||
|
||||
@@ -2,14 +2,19 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib
|
||||
from unittest.mock import patch
|
||||
|
||||
from transformers import processing_utils
|
||||
from transformers.processing_utils import ProcessingKwargs
|
||||
from typing_extensions import Unpack
|
||||
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.transformers_utils.processor import (
|
||||
get_processor,
|
||||
get_processor_kwargs_keys,
|
||||
get_processor_kwargs_type,
|
||||
)
|
||||
from vllm.transformers_utils.repo_utils import _snapshot_non_weight_files
|
||||
|
||||
|
||||
class _FakeProcessorKwargs(ProcessingKwargs, total=False): # type: ignore
|
||||
@@ -65,3 +70,26 @@ def test_get_processor_kwargs_from_processor_module_scan_returns_full_union():
|
||||
proc = _ProcWithoutUnpack()
|
||||
keys = get_processor_kwargs_keys(get_processor_kwargs_type(proc))
|
||||
_assert_has_all_expected(keys)
|
||||
|
||||
|
||||
def test_hub_processor_loads_from_local_snapshot():
|
||||
"""Transformers makes a Hub request per file when given a repo ID."""
|
||||
with patch.object(
|
||||
processing_utils,
|
||||
"list_repo_templates",
|
||||
wraps=processing_utils.list_repo_templates,
|
||||
) as list_repo_templates:
|
||||
processor = get_processor("Qwen/Qwen2-VL-2B-Instruct")
|
||||
|
||||
assert processor.tokenizer.name_or_path == "Qwen/Qwen2-VL-2B-Instruct"
|
||||
assert processor.chat_template is not None
|
||||
list_repo_templates.assert_not_called()
|
||||
|
||||
|
||||
def test_processor_reuses_tokenizer_snapshot():
|
||||
"""The chat template lookup must not snapshot the repo a second time."""
|
||||
model = "Qwen/Qwen2-VL-2B-Instruct"
|
||||
_snapshot_non_weight_files.cache_clear()
|
||||
get_tokenizer(model)
|
||||
get_processor(model)
|
||||
assert _snapshot_non_weight_files.cache_info().misses == 1
|
||||
|
||||
Reference in New Issue
Block a user