[Performance] Load HF tokenizers and processors from a local snapshot (#60825)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Kevin H. Luu
parent
092c35dd15
commit
3facf822b7
@@ -1,11 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import _get_protocol_attrs # type: ignore
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from transformers import (
|
||||
PreTrainedTokenizerBase,
|
||||
TokenizersBackend,
|
||||
tokenization_utils_base,
|
||||
)
|
||||
|
||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||
@@ -56,6 +58,20 @@ def test_tokenizer_revision(tokenizer_name: str):
|
||||
get_tokenizer(tokenizer_name, revision="never")
|
||||
|
||||
|
||||
def test_hub_tokenizer_loads_from_local_snapshot():
|
||||
"""Transformers makes a Hub request per file when given a repo ID."""
|
||||
with patch.object(
|
||||
tokenization_utils_base,
|
||||
"list_repo_templates",
|
||||
wraps=tokenization_utils_base.list_repo_templates,
|
||||
) as list_repo_templates:
|
||||
tokenizer = get_tokenizer("openai-community/gpt2")
|
||||
|
||||
assert isinstance(tokenizer, PreTrainedTokenizerBase)
|
||||
assert tokenizer.name_or_path == "openai-community/gpt2"
|
||||
list_repo_templates.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tokenizer_name", ["BAAI/bge-base-en"])
|
||||
@pytest.mark.parametrize("n_tokens", [510])
|
||||
def test_special_tokens(tokenizer_name: str, n_tokens: int):
|
||||
|
||||
Reference in New Issue
Block a user