[Misc] Make cached tokenizer pickle-compatible (#17048)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-04-27 13:05:00 +08:00
committed by GitHub
parent 8e4b351a0c
commit 93a126fbc7
5 changed files with 81 additions and 57 deletions
+17 -17
View File
@@ -2,7 +2,7 @@
import importlib
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from typing import TYPE_CHECKING, Any, Optional, Union
if TYPE_CHECKING:
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
@@ -12,17 +12,17 @@ class TokenizerBase(ABC):
@property
@abstractmethod
def all_special_tokens_extended(self) -> List[str]:
def all_special_tokens_extended(self) -> list[str]:
raise NotImplementedError()
@property
@abstractmethod
def all_special_tokens(self) -> List[str]:
def all_special_tokens(self) -> list[str]:
raise NotImplementedError()
@property
@abstractmethod
def all_special_ids(self) -> List[int]:
def all_special_ids(self) -> list[int]:
raise NotImplementedError()
@property
@@ -66,7 +66,7 @@ class TokenizerBase(ABC):
@abstractmethod
def __call__(
self,
text: Union[str, List[str], List[int]],
text: Union[str, list[str], list[int]],
text_pair: Optional[str] = None,
add_special_tokens: bool = False,
truncation: bool = False,
@@ -75,11 +75,11 @@ class TokenizerBase(ABC):
raise NotImplementedError()
@abstractmethod
def get_vocab(self) -> Dict[str, int]:
def get_vocab(self) -> dict[str, int]:
raise NotImplementedError()
@abstractmethod
def get_added_vocab(self) -> Dict[str, int]:
def get_added_vocab(self) -> dict[str, int]:
raise NotImplementedError()
@abstractmethod
@@ -88,44 +88,44 @@ class TokenizerBase(ABC):
text: str,
truncation: bool = False,
max_length: Optional[int] = None,
) -> List[int]:
) -> list[int]:
raise NotImplementedError()
@abstractmethod
def encode(self,
text: str,
add_special_tokens: Optional[bool] = None) -> List[int]:
add_special_tokens: Optional[bool] = None) -> list[int]:
raise NotImplementedError()
@abstractmethod
def apply_chat_template(self,
messages: List["ChatCompletionMessageParam"],
tools: Optional[List[Dict[str, Any]]] = None,
**kwargs) -> List[int]:
messages: list["ChatCompletionMessageParam"],
tools: Optional[list[dict[str, Any]]] = None,
**kwargs) -> list[int]:
raise NotImplementedError()
@abstractmethod
def convert_tokens_to_string(self, tokens: List[str]) -> str:
def convert_tokens_to_string(self, tokens: list[str]) -> str:
raise NotImplementedError()
@abstractmethod
def decode(self,
ids: Union[List[int], int],
ids: Union[list[int], int],
skip_special_tokens: bool = True) -> str:
raise NotImplementedError()
@abstractmethod
def convert_ids_to_tokens(
self,
ids: List[int],
ids: list[int],
skip_special_tokens: bool = True,
) -> List[str]:
) -> list[str]:
raise NotImplementedError()
class TokenizerRegistry:
# Tokenizer name -> (tokenizer module, tokenizer class)
REGISTRY: Dict[str, Tuple[str, str]] = {}
REGISTRY: dict[str, tuple[str, str]] = {}
@staticmethod
def register(name: str, module: str, class_name: str) -> None: