[Model] Add Mistral Tokenization to improve robustness and chat encoding (#7739)
This commit is contained in:
@@ -267,7 +267,7 @@ def apply_chat_template(
|
||||
*,
|
||||
tokenize: bool = False, # Different from HF's default
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
) -> Union[str, List[int]]:
|
||||
if chat_template is None and tokenizer.chat_template is None:
|
||||
raise ValueError(
|
||||
"As of transformers v4.44, default chat template is no longer "
|
||||
@@ -280,6 +280,4 @@ def apply_chat_template(
|
||||
tokenize=tokenize,
|
||||
**kwargs,
|
||||
)
|
||||
assert isinstance(prompt, str)
|
||||
|
||||
return prompt
|
||||
|
||||
@@ -390,15 +390,21 @@ class LLM:
|
||||
conversations, _ = parse_chat_messages(messages, model_config,
|
||||
tokenizer)
|
||||
|
||||
prompts = apply_chat_template(
|
||||
prompt = apply_chat_template(
|
||||
tokenizer,
|
||||
conversations,
|
||||
chat_template=chat_template,
|
||||
add_generation_prompt=add_generation_prompt)
|
||||
|
||||
inputs: PromptInputs
|
||||
if isinstance(prompt, list) and isinstance(prompt[0], int):
|
||||
inputs = TokensPrompt(prompt_token_ids=prompt)
|
||||
else:
|
||||
inputs = TextPrompt(prompt=prompt)
|
||||
|
||||
return self.generate(
|
||||
prompts,
|
||||
sampling_params,
|
||||
inputs,
|
||||
sampling_params=sampling_params,
|
||||
use_tqdm=use_tqdm,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
@@ -22,7 +22,8 @@ from vllm.entrypoints.openai.protocol import (
|
||||
FunctionCall, ToolCall, UsageInfo)
|
||||
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
|
||||
OpenAIServing,
|
||||
PromptAdapterPath)
|
||||
PromptAdapterPath,
|
||||
TextTokensPrompt)
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal import MultiModalDataDict
|
||||
@@ -130,13 +131,22 @@ class OpenAIServingChat(OpenAIServing):
|
||||
guided_decode_logits_processor = (
|
||||
await self._guided_decode_logits_processor(request, tokenizer))
|
||||
|
||||
prompt_inputs = self._tokenize_prompt_input(
|
||||
request,
|
||||
tokenizer,
|
||||
prompt,
|
||||
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||
add_special_tokens=request.add_special_tokens,
|
||||
)
|
||||
if isinstance(prompt, str):
|
||||
prompt_inputs = self._tokenize_prompt_input(
|
||||
request,
|
||||
tokenizer,
|
||||
prompt,
|
||||
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||
add_special_tokens=request.add_special_tokens,
|
||||
)
|
||||
else:
|
||||
assert isinstance(prompt, list) and isinstance(
|
||||
prompt[0], int
|
||||
), "Prompt has to be either a string or a list of token ids"
|
||||
prompt_inputs = TextTokensPrompt(
|
||||
prompt=tokenizer.decode(prompt), prompt_token_ids=prompt)
|
||||
|
||||
assert prompt_inputs is not None
|
||||
|
||||
sampling_params = request.to_sampling_params(
|
||||
tokenizer,
|
||||
|
||||
Reference in New Issue
Block a user