[Performance] Load HF tokenizers and processors from a local snapshot (#60825)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Kevin H. Luu
parent
092c35dd15
commit
3facf822b7
@@ -1,11 +1,13 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from typing import _get_protocol_attrs # type: ignore
|
from typing import _get_protocol_attrs # type: ignore
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from transformers import (
|
from transformers import (
|
||||||
PreTrainedTokenizerBase,
|
PreTrainedTokenizerBase,
|
||||||
TokenizersBackend,
|
TokenizersBackend,
|
||||||
|
tokenization_utils_base,
|
||||||
)
|
)
|
||||||
|
|
||||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||||
@@ -56,6 +58,20 @@ def test_tokenizer_revision(tokenizer_name: str):
|
|||||||
get_tokenizer(tokenizer_name, revision="never")
|
get_tokenizer(tokenizer_name, revision="never")
|
||||||
|
|
||||||
|
|
||||||
|
def test_hub_tokenizer_loads_from_local_snapshot():
|
||||||
|
"""Transformers makes a Hub request per file when given a repo ID."""
|
||||||
|
with patch.object(
|
||||||
|
tokenization_utils_base,
|
||||||
|
"list_repo_templates",
|
||||||
|
wraps=tokenization_utils_base.list_repo_templates,
|
||||||
|
) as list_repo_templates:
|
||||||
|
tokenizer = get_tokenizer("openai-community/gpt2")
|
||||||
|
|
||||||
|
assert isinstance(tokenizer, PreTrainedTokenizerBase)
|
||||||
|
assert tokenizer.name_or_path == "openai-community/gpt2"
|
||||||
|
list_repo_templates.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("tokenizer_name", ["BAAI/bge-base-en"])
|
@pytest.mark.parametrize("tokenizer_name", ["BAAI/bge-base-en"])
|
||||||
@pytest.mark.parametrize("n_tokens", [510])
|
@pytest.mark.parametrize("n_tokens", [510])
|
||||||
def test_special_tokens(tokenizer_name: str, n_tokens: int):
|
def test_special_tokens(tokenizer_name: str, n_tokens: int):
|
||||||
|
|||||||
@@ -2,14 +2,19 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import importlib
|
import importlib
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from transformers import processing_utils
|
||||||
from transformers.processing_utils import ProcessingKwargs
|
from transformers.processing_utils import ProcessingKwargs
|
||||||
from typing_extensions import Unpack
|
from typing_extensions import Unpack
|
||||||
|
|
||||||
|
from vllm.tokenizers import get_tokenizer
|
||||||
from vllm.transformers_utils.processor import (
|
from vllm.transformers_utils.processor import (
|
||||||
|
get_processor,
|
||||||
get_processor_kwargs_keys,
|
get_processor_kwargs_keys,
|
||||||
get_processor_kwargs_type,
|
get_processor_kwargs_type,
|
||||||
)
|
)
|
||||||
|
from vllm.transformers_utils.repo_utils import _snapshot_non_weight_files
|
||||||
|
|
||||||
|
|
||||||
class _FakeProcessorKwargs(ProcessingKwargs, total=False): # type: ignore
|
class _FakeProcessorKwargs(ProcessingKwargs, total=False): # type: ignore
|
||||||
@@ -65,3 +70,26 @@ def test_get_processor_kwargs_from_processor_module_scan_returns_full_union():
|
|||||||
proc = _ProcWithoutUnpack()
|
proc = _ProcWithoutUnpack()
|
||||||
keys = get_processor_kwargs_keys(get_processor_kwargs_type(proc))
|
keys = get_processor_kwargs_keys(get_processor_kwargs_type(proc))
|
||||||
_assert_has_all_expected(keys)
|
_assert_has_all_expected(keys)
|
||||||
|
|
||||||
|
|
||||||
|
def test_hub_processor_loads_from_local_snapshot():
|
||||||
|
"""Transformers makes a Hub request per file when given a repo ID."""
|
||||||
|
with patch.object(
|
||||||
|
processing_utils,
|
||||||
|
"list_repo_templates",
|
||||||
|
wraps=processing_utils.list_repo_templates,
|
||||||
|
) as list_repo_templates:
|
||||||
|
processor = get_processor("Qwen/Qwen2-VL-2B-Instruct")
|
||||||
|
|
||||||
|
assert processor.tokenizer.name_or_path == "Qwen/Qwen2-VL-2B-Instruct"
|
||||||
|
assert processor.chat_template is not None
|
||||||
|
list_repo_templates.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_processor_reuses_tokenizer_snapshot():
|
||||||
|
"""The chat template lookup must not snapshot the repo a second time."""
|
||||||
|
model = "Qwen/Qwen2-VL-2B-Instruct"
|
||||||
|
_snapshot_non_weight_files.cache_clear()
|
||||||
|
get_tokenizer(model)
|
||||||
|
get_processor(model)
|
||||||
|
assert _snapshot_non_weight_files.cache_info().misses == 1
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import contextlib
|
|||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING, assert_never
|
from typing import TYPE_CHECKING, assert_never, cast
|
||||||
|
|
||||||
import huggingface_hub
|
import huggingface_hub
|
||||||
from typing_extensions import TypeVar
|
from typing_extensions import TypeVar
|
||||||
@@ -14,11 +14,12 @@ from vllm.logger import init_logger
|
|||||||
from vllm.transformers_utils.config import _maybe_register_hf_config, get_config
|
from vllm.transformers_utils.config import _maybe_register_hf_config, get_config
|
||||||
from vllm.transformers_utils.repo_utils import (
|
from vllm.transformers_utils.repo_utils import (
|
||||||
any_pattern_in_repo_files,
|
any_pattern_in_repo_files,
|
||||||
|
get_non_weight_snapshot_path,
|
||||||
is_mistral_model_repo,
|
is_mistral_model_repo,
|
||||||
)
|
)
|
||||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||||
|
|
||||||
from .hf import CachedHfTokenizer
|
from .hf import CachedHfTokenizer, HfTokenizer
|
||||||
from .protocol import TokenizerLike
|
from .protocol import TokenizerLike
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
@@ -164,6 +165,12 @@ def resolve_tokenizer_args(
|
|||||||
"mistral_format=True is not supported with tokenizer_mode='hf'"
|
"mistral_format=True is not supported with tokenizer_mode='hf'"
|
||||||
)
|
)
|
||||||
kwargs["mistral_format"] = False
|
kwargs["mistral_format"] = False
|
||||||
|
tokenizer_name = get_non_weight_snapshot_path(
|
||||||
|
str(tokenizer_name),
|
||||||
|
revision=revision,
|
||||||
|
cache_dir=download_dir,
|
||||||
|
token=kwargs.get("token"),
|
||||||
|
)
|
||||||
|
|
||||||
return tokenizer_mode, tokenizer_name, args, kwargs
|
return tokenizer_mode, tokenizer_name, args, kwargs
|
||||||
|
|
||||||
@@ -202,6 +209,7 @@ def get_tokenizer(
|
|||||||
|
|
||||||
apply_fastokens_patch()
|
apply_fastokens_patch()
|
||||||
|
|
||||||
|
requested_name = str(tokenizer_name)
|
||||||
tokenizer_mode, tokenizer_name, args, kwargs = cached_resolve_tokenizer_args(
|
tokenizer_mode, tokenizer_name, args, kwargs = cached_resolve_tokenizer_args(
|
||||||
tokenizer_name,
|
tokenizer_name,
|
||||||
*args,
|
*args,
|
||||||
@@ -254,6 +262,9 @@ def get_tokenizer(
|
|||||||
kwargs.setdefault("config", config)
|
kwargs.setdefault("config", config)
|
||||||
|
|
||||||
tokenizer = tokenizer_cls_.from_pretrained(tokenizer_name, *args, **kwargs)
|
tokenizer = tokenizer_cls_.from_pretrained(tokenizer_name, *args, **kwargs)
|
||||||
|
# Report the Hub ID rather than the local snapshot it was loaded from
|
||||||
|
if tokenizer_mode == "hf" and not envs.VLLM_USE_MODELSCOPE:
|
||||||
|
cast(HfTokenizer, tokenizer).name_or_path = requested_name
|
||||||
if model_type in _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS:
|
if model_type in _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS:
|
||||||
from vllm.tokenizers.hf import get_cached_tokenizer
|
from vllm.tokenizers.hf import get_cached_tokenizer
|
||||||
|
|
||||||
|
|||||||
@@ -22,7 +22,10 @@ from typing_extensions import TypeVar
|
|||||||
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.transformers_utils import processors
|
from vllm.transformers_utils import processors
|
||||||
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
|
from vllm.transformers_utils.repo_utils import (
|
||||||
|
get_hf_file_to_dict,
|
||||||
|
get_non_weight_snapshot_path,
|
||||||
|
)
|
||||||
from vllm.transformers_utils.utils import convert_model_repo_to_path
|
from vllm.transformers_utils.utils import convert_model_repo_to_path
|
||||||
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
|
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
|
||||||
|
|
||||||
@@ -214,7 +217,12 @@ def get_processor(
|
|||||||
if revision is None:
|
if revision is None:
|
||||||
revision = "main"
|
revision = "main"
|
||||||
try:
|
try:
|
||||||
processor_name = convert_model_repo_to_path(processor_name)
|
processor_name = requested_name = convert_model_repo_to_path(processor_name)
|
||||||
|
# A snapshot pinned to `revision` would shadow remote code at `code_revision`
|
||||||
|
if kwargs.get("code_revision") in (None, revision):
|
||||||
|
processor_name = get_non_weight_snapshot_path(
|
||||||
|
processor_name, revision=revision, token=kwargs.get("token")
|
||||||
|
)
|
||||||
registered_cls_name = get_processor_cls_name_from_config(
|
registered_cls_name = get_processor_cls_name_from_config(
|
||||||
processor_name, revision=revision
|
processor_name, revision=revision
|
||||||
)
|
)
|
||||||
@@ -269,6 +277,11 @@ def get_processor(
|
|||||||
f"found type: {type(processor)}"
|
f"found type: {type(processor)}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Report the Hub ID rather than the local snapshot it was loaded from
|
||||||
|
tokenizer = getattr(processor, "tokenizer", processor)
|
||||||
|
if hasattr(tokenizer, "name_or_path"):
|
||||||
|
tokenizer.name_or_path = requested_name
|
||||||
|
|
||||||
return processor
|
return processor
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -304,6 +304,80 @@ def get_model_path(model: str | Path, revision: str | None = None):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_WEIGHT_FILE_PATTERNS = [
|
||||||
|
"*.safetensors",
|
||||||
|
"*.bin",
|
||||||
|
"*.pt",
|
||||||
|
"*.pth",
|
||||||
|
"*.ckpt",
|
||||||
|
"*.gguf",
|
||||||
|
"*.h5",
|
||||||
|
"*.msgpack",
|
||||||
|
"*.onnx",
|
||||||
|
"*.onnx_data",
|
||||||
|
# Large non-weight assets that tokenizers and processors never read
|
||||||
|
"*.png",
|
||||||
|
"*.jpg",
|
||||||
|
"*.jpeg",
|
||||||
|
"*.gif",
|
||||||
|
"*.webp",
|
||||||
|
"*.mp3",
|
||||||
|
"*.mp4",
|
||||||
|
"*.wav",
|
||||||
|
"*.flac",
|
||||||
|
"*.parquet",
|
||||||
|
"*.arrow",
|
||||||
|
"*.ipynb",
|
||||||
|
"*.zip",
|
||||||
|
"*.tar*",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@cache
|
||||||
|
def _snapshot_non_weight_files(
|
||||||
|
repo_id: str,
|
||||||
|
revision: str | None,
|
||||||
|
cache_dir: str | None,
|
||||||
|
token: str | bool | None,
|
||||||
|
) -> str:
|
||||||
|
# Avoid circular import
|
||||||
|
from vllm.model_executor.model_loader.weight_utils import get_lock
|
||||||
|
|
||||||
|
with get_lock(repo_id, cache_dir):
|
||||||
|
return hf_api().snapshot_download(
|
||||||
|
repo_id=repo_id,
|
||||||
|
revision=revision,
|
||||||
|
cache_dir=cache_dir,
|
||||||
|
token=token,
|
||||||
|
local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
|
||||||
|
ignore_patterns=_WEIGHT_FILE_PATTERNS,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_non_weight_snapshot_path(
|
||||||
|
repo_id: str,
|
||||||
|
revision: str | None = None,
|
||||||
|
cache_dir: str | None = None,
|
||||||
|
token: str | bool | None = None,
|
||||||
|
) -> str:
|
||||||
|
"""Download all non-weight files of a Hub repo and return the local snapshot.
|
||||||
|
|
||||||
|
Loading tokenizers and processors from a local directory stops Transformers
|
||||||
|
from making a Hub request per file (including the `additional_chat_templates`
|
||||||
|
listing). Returns `repo_id` unchanged if it is local or cannot be fetched, so
|
||||||
|
Transformers handles it as before. Failures are not cached, so they are retried.
|
||||||
|
"""
|
||||||
|
if envs.VLLM_USE_MODELSCOPE or Path(repo_id).exists():
|
||||||
|
return repo_id
|
||||||
|
# Normalise so that `None` and "main" share a cache entry
|
||||||
|
revision = revision or huggingface_hub.constants.DEFAULT_REVISION
|
||||||
|
try:
|
||||||
|
return _snapshot_non_weight_files(repo_id, revision, cache_dir, token)
|
||||||
|
except (HfHubHTTPError, OSError, ValueError):
|
||||||
|
logger.debug("Failed to snapshot %s", repo_id, exc_info=True)
|
||||||
|
return repo_id
|
||||||
|
|
||||||
|
|
||||||
def _try_download_from_hf_hub(
|
def _try_download_from_hf_hub(
|
||||||
model: str | Path, file_name: str, revision: str | None
|
model: str | Path, file_name: str, revision: str | None
|
||||||
) -> Path | None:
|
) -> Path | None:
|
||||||
|
|||||||
Reference in New Issue
Block a user