[Misc] Refactor tokenizer interface (#29693)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -5,41 +5,48 @@ import contextlib
|
||||
import copy
|
||||
import importlib.util
|
||||
import os
|
||||
import warnings
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, TypeAlias
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import huggingface_hub
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config
|
||||
from vllm.transformers_utils.gguf_utils import get_gguf_file_path_from_hf
|
||||
from vllm.transformers_utils.repo_utils import list_filtered_repo_files
|
||||
from vllm.transformers_utils.tokenizers import MistralTokenizer
|
||||
from vllm.transformers_utils.utils import (
|
||||
check_gguf_file,
|
||||
is_gguf,
|
||||
is_remote_gguf,
|
||||
split_remote_gguf,
|
||||
)
|
||||
from vllm.tokenizers import MistralTokenizer, TokenizerLike, TokenizerRegistry
|
||||
|
||||
from .config import get_sentence_transformer_tokenizer_config
|
||||
from .gguf_utils import get_gguf_file_path_from_hf
|
||||
from .repo_utils import list_filtered_repo_files
|
||||
from .utils import check_gguf_file, is_gguf, is_remote_gguf, split_remote_gguf
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.transformers_utils.tokenizer_base import TokenizerBase
|
||||
else:
|
||||
ModelConfig = Any
|
||||
TokenizerBase = Any
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
AnyTokenizer: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast | TokenizerBase
|
||||
|
||||
def __getattr__(name: str):
|
||||
if name == "AnyTokenizer":
|
||||
warnings.warn(
|
||||
"`vllm.transformers_utils.tokenizer.AnyTokenizer` has been moved to "
|
||||
"`vllm.tokenizers.TokenizerLike`. "
|
||||
"The old name will be removed in v0.13.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
return TokenizerLike
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
def decode_tokens(
|
||||
tokenizer: AnyTokenizer,
|
||||
tokenizer: TokenizerLike,
|
||||
token_ids: list[int],
|
||||
*,
|
||||
skip_special_tokens: bool | None = None,
|
||||
@@ -58,7 +65,7 @@ def decode_tokens(
|
||||
|
||||
|
||||
def encode_tokens(
|
||||
tokenizer: AnyTokenizer,
|
||||
tokenizer: TokenizerLike,
|
||||
text: str,
|
||||
*,
|
||||
truncation: bool | None = None,
|
||||
@@ -86,7 +93,7 @@ def encode_tokens(
|
||||
return tokenizer.encode(text, **kw_args)
|
||||
|
||||
|
||||
def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
|
||||
def get_cached_tokenizer(tokenizer: TokenizerLike) -> TokenizerLike:
|
||||
"""
|
||||
By default, transformers will recompute multiple tokenizer properties
|
||||
each time they are called, leading to a significant slowdown.
|
||||
@@ -144,7 +151,7 @@ def get_tokenizer(
|
||||
revision: str | None = None,
|
||||
download_dir: str | None = None,
|
||||
**kwargs,
|
||||
) -> AnyTokenizer:
|
||||
) -> TokenizerLike:
|
||||
"""Gets a tokenizer for the given model name via HuggingFace or ModelScope."""
|
||||
if envs.VLLM_USE_MODELSCOPE:
|
||||
# download model from ModelScope hub,
|
||||
@@ -206,15 +213,13 @@ def get_tokenizer(
|
||||
if len(files_list) > 0:
|
||||
tokenizer_mode = "mistral"
|
||||
|
||||
tokenizer: AnyTokenizer
|
||||
tokenizer: TokenizerLike
|
||||
if tokenizer_mode == "mistral":
|
||||
logger.debug_once(f"Loading MistralTokenizer from {tokenizer_name}")
|
||||
tokenizer = MistralTokenizer.from_pretrained(
|
||||
str(tokenizer_name), revision=revision
|
||||
)
|
||||
elif tokenizer_mode == "custom":
|
||||
from vllm.transformers_utils.tokenizer_base import TokenizerRegistry
|
||||
|
||||
logger.debug_once(f"Loading CustomTokenizer from {tokenizer_name}")
|
||||
tokenizer = TokenizerRegistry.get_tokenizer(
|
||||
str(tokenizer_name),
|
||||
@@ -260,12 +265,13 @@ def get_tokenizer(
|
||||
if isinstance(encoder_config, dict) and encoder_config.get(
|
||||
"do_lower_case", False
|
||||
):
|
||||
assert isinstance(tokenizer, PreTrainedTokenizerBase)
|
||||
special_tokens_map = {
|
||||
k: v.lower() for k, v in tokenizer.special_tokens_map.items()
|
||||
}
|
||||
tokenizer.add_special_tokens(special_tokens_map)
|
||||
|
||||
if not isinstance(tokenizer, PreTrainedTokenizerFast):
|
||||
if not tokenizer.is_fast:
|
||||
logger.warning(
|
||||
"Using a slow tokenizer. This might cause a significant "
|
||||
"slowdown. Consider using a fast tokenizer instead."
|
||||
@@ -279,7 +285,7 @@ cached_get_tokenizer = lru_cache(get_tokenizer)
|
||||
|
||||
|
||||
def cached_tokenizer_from_config(
|
||||
model_config: ModelConfig,
|
||||
model_config: "ModelConfig",
|
||||
**kwargs: Any,
|
||||
):
|
||||
return cached_get_tokenizer(
|
||||
@@ -291,7 +297,7 @@ def cached_tokenizer_from_config(
|
||||
)
|
||||
|
||||
|
||||
def init_tokenizer_from_configs(model_config: ModelConfig):
|
||||
def init_tokenizer_from_configs(model_config: "ModelConfig"):
|
||||
runner_type = model_config.runner_type
|
||||
if runner_type == "generate" or runner_type == "draft":
|
||||
truncation_side = "left"
|
||||
|
||||
Reference in New Issue
Block a user