[Performance] Load HF tokenizers and processors from a local snapshot (#60825)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Kevin H. Luu
parent
092c35dd15
commit
3facf822b7
@@ -22,7 +22,10 @@ from typing_extensions import TypeVar
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils import processors
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
|
||||
from vllm.transformers_utils.repo_utils import (
|
||||
get_hf_file_to_dict,
|
||||
get_non_weight_snapshot_path,
|
||||
)
|
||||
from vllm.transformers_utils.utils import convert_model_repo_to_path
|
||||
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
|
||||
|
||||
@@ -214,7 +217,12 @@ def get_processor(
|
||||
if revision is None:
|
||||
revision = "main"
|
||||
try:
|
||||
processor_name = convert_model_repo_to_path(processor_name)
|
||||
processor_name = requested_name = convert_model_repo_to_path(processor_name)
|
||||
# A snapshot pinned to `revision` would shadow remote code at `code_revision`
|
||||
if kwargs.get("code_revision") in (None, revision):
|
||||
processor_name = get_non_weight_snapshot_path(
|
||||
processor_name, revision=revision, token=kwargs.get("token")
|
||||
)
|
||||
registered_cls_name = get_processor_cls_name_from_config(
|
||||
processor_name, revision=revision
|
||||
)
|
||||
@@ -269,6 +277,11 @@ def get_processor(
|
||||
f"found type: {type(processor)}"
|
||||
)
|
||||
|
||||
# Report the Hub ID rather than the local snapshot it was loaded from
|
||||
tokenizer = getattr(processor, "tokenizer", processor)
|
||||
if hasattr(tokenizer, "name_or_path"):
|
||||
tokenizer.name_or_path = requested_name
|
||||
|
||||
return processor
|
||||
|
||||
|
||||
|
||||
@@ -304,6 +304,80 @@ def get_model_path(model: str | Path, revision: str | None = None):
|
||||
)
|
||||
|
||||
|
||||
_WEIGHT_FILE_PATTERNS = [
|
||||
"*.safetensors",
|
||||
"*.bin",
|
||||
"*.pt",
|
||||
"*.pth",
|
||||
"*.ckpt",
|
||||
"*.gguf",
|
||||
"*.h5",
|
||||
"*.msgpack",
|
||||
"*.onnx",
|
||||
"*.onnx_data",
|
||||
# Large non-weight assets that tokenizers and processors never read
|
||||
"*.png",
|
||||
"*.jpg",
|
||||
"*.jpeg",
|
||||
"*.gif",
|
||||
"*.webp",
|
||||
"*.mp3",
|
||||
"*.mp4",
|
||||
"*.wav",
|
||||
"*.flac",
|
||||
"*.parquet",
|
||||
"*.arrow",
|
||||
"*.ipynb",
|
||||
"*.zip",
|
||||
"*.tar*",
|
||||
]
|
||||
|
||||
|
||||
@cache
|
||||
def _snapshot_non_weight_files(
|
||||
repo_id: str,
|
||||
revision: str | None,
|
||||
cache_dir: str | None,
|
||||
token: str | bool | None,
|
||||
) -> str:
|
||||
# Avoid circular import
|
||||
from vllm.model_executor.model_loader.weight_utils import get_lock
|
||||
|
||||
with get_lock(repo_id, cache_dir):
|
||||
return hf_api().snapshot_download(
|
||||
repo_id=repo_id,
|
||||
revision=revision,
|
||||
cache_dir=cache_dir,
|
||||
token=token,
|
||||
local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
|
||||
ignore_patterns=_WEIGHT_FILE_PATTERNS,
|
||||
)
|
||||
|
||||
|
||||
def get_non_weight_snapshot_path(
|
||||
repo_id: str,
|
||||
revision: str | None = None,
|
||||
cache_dir: str | None = None,
|
||||
token: str | bool | None = None,
|
||||
) -> str:
|
||||
"""Download all non-weight files of a Hub repo and return the local snapshot.
|
||||
|
||||
Loading tokenizers and processors from a local directory stops Transformers
|
||||
from making a Hub request per file (including the `additional_chat_templates`
|
||||
listing). Returns `repo_id` unchanged if it is local or cannot be fetched, so
|
||||
Transformers handles it as before. Failures are not cached, so they are retried.
|
||||
"""
|
||||
if envs.VLLM_USE_MODELSCOPE or Path(repo_id).exists():
|
||||
return repo_id
|
||||
# Normalise so that `None` and "main" share a cache entry
|
||||
revision = revision or huggingface_hub.constants.DEFAULT_REVISION
|
||||
try:
|
||||
return _snapshot_non_weight_files(repo_id, revision, cache_dir, token)
|
||||
except (HfHubHTTPError, OSError, ValueError):
|
||||
logger.debug("Failed to snapshot %s", repo_id, exc_info=True)
|
||||
return repo_id
|
||||
|
||||
|
||||
def _try_download_from_hf_hub(
|
||||
model: str | Path, file_name: str, revision: str | None
|
||||
) -> Path | None:
|
||||
|
||||
Reference in New Issue
Block a user