[Bugfix]: Fix TokenizerLike interface (#30009)
Signed-off-by: Rohan138 <rohanpotdar138@gmail.com>
This commit is contained in:
@@ -54,6 +54,9 @@ class DeepseekV32Tokenizer(HfTokenizer):
|
||||
prompt_str = encode_messages(messages, **encode_config) # type: ignore
|
||||
return prompt_str
|
||||
|
||||
def num_special_tokens_to_add(self) -> int:
|
||||
return len(self.encode(""))
|
||||
|
||||
@property
|
||||
def all_special_tokens(self) -> list[str]:
|
||||
return self.tokenizer.all_special_tokens
|
||||
|
||||
@@ -309,6 +309,9 @@ class MistralTokenizer(TokenizerLike):
|
||||
for i in all_special_ids
|
||||
]
|
||||
|
||||
def num_special_tokens_to_add(self) -> int:
|
||||
return len(self.encode(""))
|
||||
|
||||
# the following attributes are set to fit vLLM's design and are used
|
||||
# by the structured output backends.
|
||||
@property
|
||||
@@ -421,6 +424,7 @@ class MistralTokenizer(TokenizerLike):
|
||||
) -> list[int]:
|
||||
add_generation_prompt = kwargs.pop("add_generation_prompt", False)
|
||||
continue_final_message = kwargs.get("continue_final_message", False)
|
||||
tokenize = kwargs.get("tokenize", True)
|
||||
padding = kwargs.get("padding", False)
|
||||
truncation = kwargs.get("truncation", False)
|
||||
max_length = kwargs.get("max_length")
|
||||
@@ -433,7 +437,7 @@ class MistralTokenizer(TokenizerLike):
|
||||
conversation=messages,
|
||||
tools=tools,
|
||||
continue_final_message=continue_final_message,
|
||||
tokenize=True,
|
||||
tokenize=tokenize,
|
||||
padding=padding,
|
||||
truncation=truncation,
|
||||
max_length=max_length,
|
||||
|
||||
@@ -22,6 +22,9 @@ class TokenizerLike(Protocol):
|
||||
) -> "TokenizerLike":
|
||||
raise NotImplementedError
|
||||
|
||||
def num_special_tokens_to_add(self) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def all_special_tokens(self) -> list[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -183,7 +183,7 @@ def get_tokenizer(
|
||||
"`tokenizer_mode='custom'` when initializing vLLM.",
|
||||
tokenizer_args,
|
||||
str(tokenizer_kwargs),
|
||||
tokenizer_mode,
|
||||
tokenizer_name,
|
||||
)
|
||||
|
||||
tokenizer_mode = str(tokenizer_name)
|
||||
|
||||
Reference in New Issue
Block a user