[Performance] Load HF tokenizers and processors from a local snapshot (#60825)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Harry Mellor
2026-10-12 13:45:54 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Kevin H. Luu
parent 092c35dd15
commit 3facf822b7
5 changed files with 146 additions and 4 deletions
+74
View File
@@ -304,6 +304,80 @@ def get_model_path(model: str | Path, revision: str | None = None):
)
_WEIGHT_FILE_PATTERNS = [
"*.safetensors",
"*.bin",
"*.pt",
"*.pth",
"*.ckpt",
"*.gguf",
"*.h5",
"*.msgpack",
"*.onnx",
"*.onnx_data",
# Large non-weight assets that tokenizers and processors never read
"*.png",
"*.jpg",
"*.jpeg",
"*.gif",
"*.webp",
"*.mp3",
"*.mp4",
"*.wav",
"*.flac",
"*.parquet",
"*.arrow",
"*.ipynb",
"*.zip",
"*.tar*",
]
@cache
def _snapshot_non_weight_files(
repo_id: str,
revision: str | None,
cache_dir: str | None,
token: str | bool | None,
) -> str:
# Avoid circular import
from vllm.model_executor.model_loader.weight_utils import get_lock
with get_lock(repo_id, cache_dir):
return hf_api().snapshot_download(
repo_id=repo_id,
revision=revision,
cache_dir=cache_dir,
token=token,
local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
ignore_patterns=_WEIGHT_FILE_PATTERNS,
)
def get_non_weight_snapshot_path(
repo_id: str,
revision: str | None = None,
cache_dir: str | None = None,
token: str | bool | None = None,
) -> str:
"""Download all non-weight files of a Hub repo and return the local snapshot.
Loading tokenizers and processors from a local directory stops Transformers
from making a Hub request per file (including the `additional_chat_templates`
listing). Returns `repo_id` unchanged if it is local or cannot be fetched, so
Transformers handles it as before. Failures are not cached, so they are retried.
"""
if envs.VLLM_USE_MODELSCOPE or Path(repo_id).exists():
return repo_id
# Normalise so that `None` and "main" share a cache entry
revision = revision or huggingface_hub.constants.DEFAULT_REVISION
try:
return _snapshot_non_weight_files(repo_id, revision, cache_dir, token)
except (HfHubHTTPError, OSError, ValueError):
logger.debug("Failed to snapshot %s", repo_id, exc_info=True)
return repo_id
def _try_download_from_hf_hub(
model: str | Path, file_name: str, revision: str | None
) -> Path | None: