diff --git a/tests/tokenizers_/test_basic.py b/tests/tokenizers_/test_basic.py index 4a1a2b1560..a46be988e5 100644 --- a/tests/tokenizers_/test_basic.py +++ b/tests/tokenizers_/test_basic.py @@ -1,11 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import _get_protocol_attrs # type: ignore +from unittest.mock import patch import pytest from transformers import ( PreTrainedTokenizerBase, TokenizersBackend, + tokenization_utils_base, ) from vllm.tokenizers import TokenizerLike, get_tokenizer @@ -56,6 +58,20 @@ def test_tokenizer_revision(tokenizer_name: str): get_tokenizer(tokenizer_name, revision="never") +def test_hub_tokenizer_loads_from_local_snapshot(): + """Transformers makes a Hub request per file when given a repo ID.""" + with patch.object( + tokenization_utils_base, + "list_repo_templates", + wraps=tokenization_utils_base.list_repo_templates, + ) as list_repo_templates: + tokenizer = get_tokenizer("openai-community/gpt2") + + assert isinstance(tokenizer, PreTrainedTokenizerBase) + assert tokenizer.name_or_path == "openai-community/gpt2" + list_repo_templates.assert_not_called() + + @pytest.mark.parametrize("tokenizer_name", ["BAAI/bge-base-en"]) @pytest.mark.parametrize("n_tokens", [510]) def test_special_tokens(tokenizer_name: str, n_tokens: int): diff --git a/tests/transformers_utils/test_processor.py b/tests/transformers_utils/test_processor.py index a3a1c78418..93f742ff74 100644 --- a/tests/transformers_utils/test_processor.py +++ b/tests/transformers_utils/test_processor.py @@ -2,14 +2,19 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import importlib +from unittest.mock import patch +from transformers import processing_utils from transformers.processing_utils import ProcessingKwargs from typing_extensions import Unpack +from vllm.tokenizers import get_tokenizer from vllm.transformers_utils.processor import ( + get_processor, get_processor_kwargs_keys, get_processor_kwargs_type, ) +from vllm.transformers_utils.repo_utils import _snapshot_non_weight_files class _FakeProcessorKwargs(ProcessingKwargs, total=False): # type: ignore @@ -65,3 +70,26 @@ def test_get_processor_kwargs_from_processor_module_scan_returns_full_union(): proc = _ProcWithoutUnpack() keys = get_processor_kwargs_keys(get_processor_kwargs_type(proc)) _assert_has_all_expected(keys) + + +def test_hub_processor_loads_from_local_snapshot(): + """Transformers makes a Hub request per file when given a repo ID.""" + with patch.object( + processing_utils, + "list_repo_templates", + wraps=processing_utils.list_repo_templates, + ) as list_repo_templates: + processor = get_processor("Qwen/Qwen2-VL-2B-Instruct") + + assert processor.tokenizer.name_or_path == "Qwen/Qwen2-VL-2B-Instruct" + assert processor.chat_template is not None + list_repo_templates.assert_not_called() + + +def test_processor_reuses_tokenizer_snapshot(): + """The chat template lookup must not snapshot the repo a second time.""" + model = "Qwen/Qwen2-VL-2B-Instruct" + _snapshot_non_weight_files.cache_clear() + get_tokenizer(model) + get_processor(model) + assert _snapshot_non_weight_files.cache_info().misses == 1 diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index ebf85a11fd..a7adf4597f 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -4,7 +4,7 @@ import contextlib from dataclasses import dataclass, field from functools import lru_cache from pathlib import Path -from typing import TYPE_CHECKING, assert_never +from typing import TYPE_CHECKING, assert_never, cast import huggingface_hub from typing_extensions import TypeVar @@ -14,11 +14,12 @@ from vllm.logger import init_logger from vllm.transformers_utils.config import _maybe_register_hf_config, get_config from vllm.transformers_utils.repo_utils import ( any_pattern_in_repo_files, + get_non_weight_snapshot_path, is_mistral_model_repo, ) from vllm.utils.import_utils import resolve_obj_by_qualname -from .hf import CachedHfTokenizer +from .hf import CachedHfTokenizer, HfTokenizer from .protocol import TokenizerLike if TYPE_CHECKING: @@ -164,6 +165,12 @@ def resolve_tokenizer_args( "mistral_format=True is not supported with tokenizer_mode='hf'" ) kwargs["mistral_format"] = False + tokenizer_name = get_non_weight_snapshot_path( + str(tokenizer_name), + revision=revision, + cache_dir=download_dir, + token=kwargs.get("token"), + ) return tokenizer_mode, tokenizer_name, args, kwargs @@ -202,6 +209,7 @@ def get_tokenizer( apply_fastokens_patch() + requested_name = str(tokenizer_name) tokenizer_mode, tokenizer_name, args, kwargs = cached_resolve_tokenizer_args( tokenizer_name, *args, @@ -254,6 +262,9 @@ def get_tokenizer( kwargs.setdefault("config", config) tokenizer = tokenizer_cls_.from_pretrained(tokenizer_name, *args, **kwargs) + # Report the Hub ID rather than the local snapshot it was loaded from + if tokenizer_mode == "hf" and not envs.VLLM_USE_MODELSCOPE: + cast(HfTokenizer, tokenizer).name_or_path = requested_name if model_type in _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS: from vllm.tokenizers.hf import get_cached_tokenizer diff --git a/vllm/transformers_utils/processor.py b/vllm/transformers_utils/processor.py index 9aa5920849..1b9b0bd442 100644 --- a/vllm/transformers_utils/processor.py +++ b/vllm/transformers_utils/processor.py @@ -22,7 +22,10 @@ from typing_extensions import TypeVar from vllm.logger import init_logger from vllm.transformers_utils import processors -from vllm.transformers_utils.repo_utils import get_hf_file_to_dict +from vllm.transformers_utils.repo_utils import ( + get_hf_file_to_dict, + get_non_weight_snapshot_path, +) from vllm.transformers_utils.utils import convert_model_repo_to_path from vllm.utils.func_utils import get_allowed_kwarg_only_overrides @@ -214,7 +217,12 @@ def get_processor( if revision is None: revision = "main" try: - processor_name = convert_model_repo_to_path(processor_name) + processor_name = requested_name = convert_model_repo_to_path(processor_name) + # A snapshot pinned to `revision` would shadow remote code at `code_revision` + if kwargs.get("code_revision") in (None, revision): + processor_name = get_non_weight_snapshot_path( + processor_name, revision=revision, token=kwargs.get("token") + ) registered_cls_name = get_processor_cls_name_from_config( processor_name, revision=revision ) @@ -269,6 +277,11 @@ def get_processor( f"found type: {type(processor)}" ) + # Report the Hub ID rather than the local snapshot it was loaded from + tokenizer = getattr(processor, "tokenizer", processor) + if hasattr(tokenizer, "name_or_path"): + tokenizer.name_or_path = requested_name + return processor diff --git a/vllm/transformers_utils/repo_utils.py b/vllm/transformers_utils/repo_utils.py index b6a23b007a..43ef07676e 100644 --- a/vllm/transformers_utils/repo_utils.py +++ b/vllm/transformers_utils/repo_utils.py @@ -304,6 +304,80 @@ def get_model_path(model: str | Path, revision: str | None = None): ) +_WEIGHT_FILE_PATTERNS = [ + "*.safetensors", + "*.bin", + "*.pt", + "*.pth", + "*.ckpt", + "*.gguf", + "*.h5", + "*.msgpack", + "*.onnx", + "*.onnx_data", + # Large non-weight assets that tokenizers and processors never read + "*.png", + "*.jpg", + "*.jpeg", + "*.gif", + "*.webp", + "*.mp3", + "*.mp4", + "*.wav", + "*.flac", + "*.parquet", + "*.arrow", + "*.ipynb", + "*.zip", + "*.tar*", +] + + +@cache +def _snapshot_non_weight_files( + repo_id: str, + revision: str | None, + cache_dir: str | None, + token: str | bool | None, +) -> str: + # Avoid circular import + from vllm.model_executor.model_loader.weight_utils import get_lock + + with get_lock(repo_id, cache_dir): + return hf_api().snapshot_download( + repo_id=repo_id, + revision=revision, + cache_dir=cache_dir, + token=token, + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + ignore_patterns=_WEIGHT_FILE_PATTERNS, + ) + + +def get_non_weight_snapshot_path( + repo_id: str, + revision: str | None = None, + cache_dir: str | None = None, + token: str | bool | None = None, +) -> str: + """Download all non-weight files of a Hub repo and return the local snapshot. + + Loading tokenizers and processors from a local directory stops Transformers + from making a Hub request per file (including the `additional_chat_templates` + listing). Returns `repo_id` unchanged if it is local or cannot be fetched, so + Transformers handles it as before. Failures are not cached, so they are retried. + """ + if envs.VLLM_USE_MODELSCOPE or Path(repo_id).exists(): + return repo_id + # Normalise so that `None` and "main" share a cache entry + revision = revision or huggingface_hub.constants.DEFAULT_REVISION + try: + return _snapshot_non_weight_files(repo_id, revision, cache_dir, token) + except (HfHubHTTPError, OSError, ValueError): + logger.debug("Failed to snapshot %s", repo_id, exc_info=True) + return repo_id + + def _try_download_from_hf_hub( model: str | Path, file_name: str, revision: str | None ) -> Path | None: