[Model] Add mistral function calling format to all models loaded with "mistral" format (#8515)
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
co-authored by
Cyrus Leung
parent
9855b99502
commit
a54ed80249
@@ -1,5 +1,6 @@
|
||||
from contextlib import contextmanager
|
||||
from typing import ClassVar, List, Optional, Sequence, Union, cast, overload
|
||||
from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Union, cast,
|
||||
overload)
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
@@ -357,6 +358,7 @@ class LLM:
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
chat_template: Optional[str] = None,
|
||||
add_generation_prompt: bool = True,
|
||||
tools: Optional[List[Dict[str, Any]]] = None,
|
||||
) -> List[RequestOutput]:
|
||||
"""
|
||||
Generate responses for a chat conversation.
|
||||
@@ -401,6 +403,7 @@ class LLM:
|
||||
messages=messages,
|
||||
chat_template=chat_template,
|
||||
add_generation_prompt=add_generation_prompt,
|
||||
tools=tools,
|
||||
)
|
||||
else:
|
||||
prompt = apply_hf_chat_template(
|
||||
@@ -408,6 +411,7 @@ class LLM:
|
||||
conversation=conversation,
|
||||
chat_template=chat_template,
|
||||
add_generation_prompt=add_generation_prompt,
|
||||
tools=tools,
|
||||
)
|
||||
|
||||
inputs: PromptInputs
|
||||
|
||||
@@ -123,7 +123,8 @@ class OpenAIServingChat(OpenAIServing):
|
||||
]
|
||||
|
||||
prompt: Union[str, List[int]]
|
||||
if isinstance(tokenizer, MistralTokenizer):
|
||||
is_mistral_tokenizer = isinstance(tokenizer, MistralTokenizer)
|
||||
if is_mistral_tokenizer:
|
||||
prompt = apply_mistral_chat_template(
|
||||
tokenizer,
|
||||
messages=request.messages,
|
||||
@@ -159,10 +160,10 @@ class OpenAIServingChat(OpenAIServing):
|
||||
return self.create_error_response(
|
||||
"tool_choice = \"required\" is not supported!")
|
||||
|
||||
# "auto" tools requires --enable-auto-tool-choice
|
||||
# and --tool-call-parser
|
||||
if request.tool_choice == "auto" and not (
|
||||
if not is_mistral_tokenizer and request.tool_choice == "auto" and not (
|
||||
self.enable_auto_tools and self.tool_parser is not None):
|
||||
# for hf tokenizers, "auto" tools requires
|
||||
# --enable-auto-tool-choice and --tool-call-parser
|
||||
return self.create_error_response(
|
||||
"\"auto\" tool choice requires "
|
||||
"--enable-auto-tool-choice and --tool-call-parser to be set")
|
||||
|
||||
Reference in New Issue
Block a user