[Performance] Load HF tokenizers and processors from a local snapshot (#60825)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>

tests/tokenizers_/test_basic.py

@@ -1,11 +1,13 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 from typing import _get_protocol_attrs  # type: ignore
+from unittest.mock import patch
 
 import pytest
 from transformers import (
     PreTrainedTokenizerBase,
     TokenizersBackend,
+    tokenization_utils_base,
 )
 
 from vllm.tokenizers import TokenizerLike, get_tokenizer
@@ -56,6 +58,20 @@ def test_tokenizer_revision(tokenizer_name: str):
         get_tokenizer(tokenizer_name, revision="never")
 
 
+def test_hub_tokenizer_loads_from_local_snapshot():
+    """Transformers makes a Hub request per file when given a repo ID."""
+    with patch.object(
+        tokenization_utils_base,
+        "list_repo_templates",
+        wraps=tokenization_utils_base.list_repo_templates,
+    ) as list_repo_templates:
+        tokenizer = get_tokenizer("openai-community/gpt2")
+
+    assert isinstance(tokenizer, PreTrainedTokenizerBase)
+    assert tokenizer.name_or_path == "openai-community/gpt2"
+    list_repo_templates.assert_not_called()
+
+
 @pytest.mark.parametrize("tokenizer_name", ["BAAI/bge-base-en"])
 @pytest.mark.parametrize("n_tokens", [510])
 def test_special_tokens(tokenizer_name: str, n_tokens: int):

tests/transformers_utils/test_processor.py

@@ -2,14 +2,19 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 import importlib
+from unittest.mock import patch
 
+from transformers import processing_utils
 from transformers.processing_utils import ProcessingKwargs
 from typing_extensions import Unpack
 
+from vllm.tokenizers import get_tokenizer
 from vllm.transformers_utils.processor import (
+    get_processor,
     get_processor_kwargs_keys,
     get_processor_kwargs_type,
 )
+from vllm.transformers_utils.repo_utils import _snapshot_non_weight_files
 
 
 class _FakeProcessorKwargs(ProcessingKwargs, total=False):  # type: ignore
@@ -65,3 +70,26 @@ def test_get_processor_kwargs_from_processor_module_scan_returns_full_union():
     proc = _ProcWithoutUnpack()
     keys = get_processor_kwargs_keys(get_processor_kwargs_type(proc))
     _assert_has_all_expected(keys)
+
+
+def test_hub_processor_loads_from_local_snapshot():
+    """Transformers makes a Hub request per file when given a repo ID."""
+    with patch.object(
+        processing_utils,
+        "list_repo_templates",
+        wraps=processing_utils.list_repo_templates,
+    ) as list_repo_templates:
+        processor = get_processor("Qwen/Qwen2-VL-2B-Instruct")
+
+    assert processor.tokenizer.name_or_path == "Qwen/Qwen2-VL-2B-Instruct"
+    assert processor.chat_template is not None
+    list_repo_templates.assert_not_called()
+
+
+def test_processor_reuses_tokenizer_snapshot():
+    """The chat template lookup must not snapshot the repo a second time."""
+    model = "Qwen/Qwen2-VL-2B-Instruct"
+    _snapshot_non_weight_files.cache_clear()
+    get_tokenizer(model)
+    get_processor(model)
+    assert _snapshot_non_weight_files.cache_info().misses == 1

vllm/tokenizers/registry.py

@@ -4,7 +4,7 @@ import contextlib
 from dataclasses import dataclass, field
 from functools import lru_cache
 from pathlib import Path
-from typing import TYPE_CHECKING, assert_never
+from typing import TYPE_CHECKING, assert_never, cast
 
 import huggingface_hub
 from typing_extensions import TypeVar
@@ -14,11 +14,12 @@ from vllm.logger import init_logger
 from vllm.transformers_utils.config import _maybe_register_hf_config, get_config
 from vllm.transformers_utils.repo_utils import (
     any_pattern_in_repo_files,
+    get_non_weight_snapshot_path,
     is_mistral_model_repo,
 )
 from vllm.utils.import_utils import resolve_obj_by_qualname
 
-from .hf import CachedHfTokenizer
+from .hf import CachedHfTokenizer, HfTokenizer
 from .protocol import TokenizerLike
 
 if TYPE_CHECKING:
@@ -164,6 +165,12 @@ def resolve_tokenizer_args(
                 "mistral_format=True is not supported with tokenizer_mode='hf'"
             )
         kwargs["mistral_format"] = False
+        tokenizer_name = get_non_weight_snapshot_path(
+            str(tokenizer_name),
+            revision=revision,
+            cache_dir=download_dir,
+            token=kwargs.get("token"),
+        )
 
     return tokenizer_mode, tokenizer_name, args, kwargs
 
@@ -202,6 +209,7 @@ def get_tokenizer(
 
         apply_fastokens_patch()
 
+    requested_name = str(tokenizer_name)
     tokenizer_mode, tokenizer_name, args, kwargs = cached_resolve_tokenizer_args(
         tokenizer_name,
         *args,
@@ -254,6 +262,9 @@ def get_tokenizer(
         kwargs.setdefault("config", config)
 
     tokenizer = tokenizer_cls_.from_pretrained(tokenizer_name, *args, **kwargs)
+    # Report the Hub ID rather than the local snapshot it was loaded from
+    if tokenizer_mode == "hf" and not envs.VLLM_USE_MODELSCOPE:
+        cast(HfTokenizer, tokenizer).name_or_path = requested_name
     if model_type in _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS:
         from vllm.tokenizers.hf import get_cached_tokenizer
 

vllm/transformers_utils/processor.py

@@ -22,7 +22,10 @@ from typing_extensions import TypeVar
 
 from vllm.logger import init_logger
 from vllm.transformers_utils import processors
-from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
+from vllm.transformers_utils.repo_utils import (
+    get_hf_file_to_dict,
+    get_non_weight_snapshot_path,
+)
 from vllm.transformers_utils.utils import convert_model_repo_to_path
 from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
 
@@ -214,7 +217,12 @@ def get_processor(
     if revision is None:
         revision = "main"
     try:
-        processor_name = convert_model_repo_to_path(processor_name)
+        processor_name = requested_name = convert_model_repo_to_path(processor_name)
+        # A snapshot pinned to `revision` would shadow remote code at `code_revision`
+        if kwargs.get("code_revision") in (None, revision):
+            processor_name = get_non_weight_snapshot_path(
+                processor_name, revision=revision, token=kwargs.get("token")
+            )
         registered_cls_name = get_processor_cls_name_from_config(
             processor_name, revision=revision
         )
@@ -269,6 +277,11 @@ def get_processor(
             f"found type: {type(processor)}"
         )
 
+    # Report the Hub ID rather than the local snapshot it was loaded from
+    tokenizer = getattr(processor, "tokenizer", processor)
+    if hasattr(tokenizer, "name_or_path"):
+        tokenizer.name_or_path = requested_name
+
     return processor
 
 

vllm/transformers_utils/repo_utils.py

@@ -304,6 +304,80 @@ def get_model_path(model: str | Path, revision: str | None = None):
     )
 
 
+_WEIGHT_FILE_PATTERNS = [
+    "*.safetensors",
+    "*.bin",
+    "*.pt",
+    "*.pth",
+    "*.ckpt",
+    "*.gguf",
+    "*.h5",
+    "*.msgpack",
+    "*.onnx",
+    "*.onnx_data",
+    # Large non-weight assets that tokenizers and processors never read
+    "*.png",
+    "*.jpg",
+    "*.jpeg",
+    "*.gif",
+    "*.webp",
+    "*.mp3",
+    "*.mp4",
+    "*.wav",
+    "*.flac",
+    "*.parquet",
+    "*.arrow",
+    "*.ipynb",
+    "*.zip",
+    "*.tar*",
+]
+
+
+@cache
+def _snapshot_non_weight_files(
+    repo_id: str,
+    revision: str | None,
+    cache_dir: str | None,
+    token: str | bool | None,
+) -> str:
+    # Avoid circular import
+    from vllm.model_executor.model_loader.weight_utils import get_lock
+
+    with get_lock(repo_id, cache_dir):
+        return hf_api().snapshot_download(
+            repo_id=repo_id,
+            revision=revision,
+            cache_dir=cache_dir,
+            token=token,
+            local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
+            ignore_patterns=_WEIGHT_FILE_PATTERNS,
+        )
+
+
+def get_non_weight_snapshot_path(
+    repo_id: str,
+    revision: str | None = None,
+    cache_dir: str | None = None,
+    token: str | bool | None = None,
+) -> str:
+    """Download all non-weight files of a Hub repo and return the local snapshot.
+
+    Loading tokenizers and processors from a local directory stops Transformers
+    from making a Hub request per file (including the `additional_chat_templates`
+    listing). Returns `repo_id` unchanged if it is local or cannot be fetched, so
+    Transformers handles it as before. Failures are not cached, so they are retried.
+    """
+    if envs.VLLM_USE_MODELSCOPE or Path(repo_id).exists():
+        return repo_id
+    # Normalise so that `None` and "main" share a cache entry
+    revision = revision or huggingface_hub.constants.DEFAULT_REVISION
+    try:
+        return _snapshot_non_weight_files(repo_id, revision, cache_dir, token)
+    except (HfHubHTTPError, OSError, ValueError):
+        logger.debug("Failed to snapshot %s", repo_id, exc_info=True)
+        return repo_id
+
+
 def _try_download_from_hf_hub(
     model: str | Path, file_name: str, revision: str | None
 ) -> Path | None: