[Bugfix]: Fix TokenizerLike interface (#30009)

Signed-off-by: Rohan138 <rohanpotdar138@gmail.com>
This commit is contained in:
Rohan Potdar
2025-12-05 20:56:40 -08:00
committed by GitHub
parent e858bc4d14
commit 40a046cd82
8 changed files with 78 additions and 52 deletions
+3
View File
@@ -54,6 +54,9 @@ class DeepseekV32Tokenizer(HfTokenizer):
prompt_str = encode_messages(messages, **encode_config) # type: ignore
return prompt_str
def num_special_tokens_to_add(self) -> int:
return len(self.encode(""))
@property
def all_special_tokens(self) -> list[str]:
return self.tokenizer.all_special_tokens
+5 -1
View File
@@ -309,6 +309,9 @@ class MistralTokenizer(TokenizerLike):
for i in all_special_ids
]
def num_special_tokens_to_add(self) -> int:
return len(self.encode(""))
# the following attributes are set to fit vLLM's design and are used
# by the structured output backends.
@property
@@ -421,6 +424,7 @@ class MistralTokenizer(TokenizerLike):
) -> list[int]:
add_generation_prompt = kwargs.pop("add_generation_prompt", False)
continue_final_message = kwargs.get("continue_final_message", False)
tokenize = kwargs.get("tokenize", True)
padding = kwargs.get("padding", False)
truncation = kwargs.get("truncation", False)
max_length = kwargs.get("max_length")
@@ -433,7 +437,7 @@ class MistralTokenizer(TokenizerLike):
conversation=messages,
tools=tools,
continue_final_message=continue_final_message,
tokenize=True,
tokenize=tokenize,
padding=padding,
truncation=truncation,
max_length=max_length,
+3
View File
@@ -22,6 +22,9 @@ class TokenizerLike(Protocol):
) -> "TokenizerLike":
raise NotImplementedError
def num_special_tokens_to_add(self) -> int:
raise NotImplementedError
@property
def all_special_tokens(self) -> list[str]:
raise NotImplementedError
+1 -1
View File
@@ -183,7 +183,7 @@ def get_tokenizer(
"`tokenizer_mode='custom'` when initializing vLLM.",
tokenizer_args,
str(tokenizer_kwargs),
tokenizer_mode,
tokenizer_name,
)
tokenizer_mode = str(tokenizer_name)