[Misc] Refactor tokenizer interface (#29693)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-11-29 04:02:21 -08:00
committed by GitHub
parent f223ed4181
commit 34a984274e
119 changed files with 752 additions and 821 deletions
+33 -27
View File
@@ -5,41 +5,48 @@ import contextlib
import copy
import importlib.util
import os
import warnings
from functools import lru_cache
from pathlib import Path
from typing import TYPE_CHECKING, Any, TypeAlias
from typing import TYPE_CHECKING, Any
import huggingface_hub
from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from typing_extensions import assert_never
from vllm import envs
from vllm.logger import init_logger
from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config
from vllm.transformers_utils.gguf_utils import get_gguf_file_path_from_hf
from vllm.transformers_utils.repo_utils import list_filtered_repo_files
from vllm.transformers_utils.tokenizers import MistralTokenizer
from vllm.transformers_utils.utils import (
check_gguf_file,
is_gguf,
is_remote_gguf,
split_remote_gguf,
)
from vllm.tokenizers import MistralTokenizer, TokenizerLike, TokenizerRegistry
from .config import get_sentence_transformer_tokenizer_config
from .gguf_utils import get_gguf_file_path_from_hf
from .repo_utils import list_filtered_repo_files
from .utils import check_gguf_file, is_gguf, is_remote_gguf, split_remote_gguf
if TYPE_CHECKING:
from vllm.config import ModelConfig
from vllm.transformers_utils.tokenizer_base import TokenizerBase
else:
ModelConfig = Any
TokenizerBase = Any
logger = init_logger(__name__)
AnyTokenizer: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast | TokenizerBase
def __getattr__(name: str):
if name == "AnyTokenizer":
warnings.warn(
"`vllm.transformers_utils.tokenizer.AnyTokenizer` has been moved to "
"`vllm.tokenizers.TokenizerLike`. "
"The old name will be removed in v0.13.",
DeprecationWarning,
stacklevel=2,
)
return TokenizerLike
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
def decode_tokens(
tokenizer: AnyTokenizer,
tokenizer: TokenizerLike,
token_ids: list[int],
*,
skip_special_tokens: bool | None = None,
@@ -58,7 +65,7 @@ def decode_tokens(
def encode_tokens(
tokenizer: AnyTokenizer,
tokenizer: TokenizerLike,
text: str,
*,
truncation: bool | None = None,
@@ -86,7 +93,7 @@ def encode_tokens(
return tokenizer.encode(text, **kw_args)
def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
def get_cached_tokenizer(tokenizer: TokenizerLike) -> TokenizerLike:
"""
By default, transformers will recompute multiple tokenizer properties
each time they are called, leading to a significant slowdown.
@@ -144,7 +151,7 @@ def get_tokenizer(
revision: str | None = None,
download_dir: str | None = None,
**kwargs,
) -> AnyTokenizer:
) -> TokenizerLike:
"""Gets a tokenizer for the given model name via HuggingFace or ModelScope."""
if envs.VLLM_USE_MODELSCOPE:
# download model from ModelScope hub,
@@ -206,15 +213,13 @@ def get_tokenizer(
if len(files_list) > 0:
tokenizer_mode = "mistral"
tokenizer: AnyTokenizer
tokenizer: TokenizerLike
if tokenizer_mode == "mistral":
logger.debug_once(f"Loading MistralTokenizer from {tokenizer_name}")
tokenizer = MistralTokenizer.from_pretrained(
str(tokenizer_name), revision=revision
)
elif tokenizer_mode == "custom":
from vllm.transformers_utils.tokenizer_base import TokenizerRegistry
logger.debug_once(f"Loading CustomTokenizer from {tokenizer_name}")
tokenizer = TokenizerRegistry.get_tokenizer(
str(tokenizer_name),
@@ -260,12 +265,13 @@ def get_tokenizer(
if isinstance(encoder_config, dict) and encoder_config.get(
"do_lower_case", False
):
assert isinstance(tokenizer, PreTrainedTokenizerBase)
special_tokens_map = {
k: v.lower() for k, v in tokenizer.special_tokens_map.items()
}
tokenizer.add_special_tokens(special_tokens_map)
if not isinstance(tokenizer, PreTrainedTokenizerFast):
if not tokenizer.is_fast:
logger.warning(
"Using a slow tokenizer. This might cause a significant "
"slowdown. Consider using a fast tokenizer instead."
@@ -279,7 +285,7 @@ cached_get_tokenizer = lru_cache(get_tokenizer)
def cached_tokenizer_from_config(
model_config: ModelConfig,
model_config: "ModelConfig",
**kwargs: Any,
):
return cached_get_tokenizer(
@@ -291,7 +297,7 @@ def cached_tokenizer_from_config(
)
def init_tokenizer_from_configs(model_config: ModelConfig):
def init_tokenizer_from_configs(model_config: "ModelConfig"):
runner_type = model_config.runner_type
if runner_type == "generate" or runner_type == "draft":
truncation_side = "left"