[Frontend] Support strict mode for tool calling (#45003)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: cjackal <44624812+cjackal@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
cjackal
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
bd59c913bc
commit
2043258dec
@@ -43,8 +43,7 @@ from vllm.inputs import (
|
||||
tokens_input,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser import ParserManager
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.parser import Parser, ParserManager
|
||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
extract_prompt_components,
|
||||
@@ -52,7 +51,6 @@ from vllm.renderers.inputs.preprocess import (
|
||||
parse_model_prompt,
|
||||
prompt_to_seq,
|
||||
)
|
||||
from vllm.tool_parsers import ToolParser
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
|
||||
from vllm.utils.mistral import mt as _mt
|
||||
@@ -89,16 +87,12 @@ class OpenAIServingRender:
|
||||
self.trust_request_chat_template = trust_request_chat_template
|
||||
self.enable_auto_tools = enable_auto_tools
|
||||
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
|
||||
self.tool_parser: type[ToolParser] | None = ParserManager.get_tool_parser(
|
||||
self.parser: type[Parser] | None = ParserManager.get_parser(
|
||||
tool_parser_name=tool_parser,
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
model_name=model_config.model,
|
||||
)
|
||||
self.reasoning_parser: type[ReasoningParser] | None = (
|
||||
ParserManager.get_reasoning_parser(
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
)
|
||||
)
|
||||
self.default_chat_template_kwargs: dict[str, Any] = (
|
||||
default_chat_template_kwargs or {}
|
||||
)
|
||||
@@ -193,7 +187,7 @@ class OpenAIServingRender:
|
||||
"""
|
||||
tokenizer = self.renderer.tokenizer
|
||||
|
||||
tool_parser = self.tool_parser
|
||||
tool_parser = self.parser.tool_parser_cls if self.parser is not None else None
|
||||
|
||||
if is_mistral_tokenizer(tokenizer):
|
||||
# because of issues with pydantic we need to potentially
|
||||
@@ -252,9 +246,8 @@ class OpenAIServingRender:
|
||||
default_template_content_format=self.chat_template_content_format,
|
||||
default_template_kwargs=self.default_chat_template_kwargs,
|
||||
tool_dicts=tool_dicts,
|
||||
tool_parser=tool_parser,
|
||||
parser=self.parser,
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
reasoning_parser=self.reasoning_parser,
|
||||
)
|
||||
else:
|
||||
# For GPT-OSS.
|
||||
@@ -526,8 +519,7 @@ class OpenAIServingRender:
|
||||
default_template_content_format: ChatTemplateContentFormatOption,
|
||||
default_template_kwargs: dict[str, Any] | None,
|
||||
tool_dicts: list[dict[str, Any]] | None = None,
|
||||
tool_parser: type[ToolParser] | None = None,
|
||||
reasoning_parser: type[ReasoningParser] | None = None,
|
||||
parser: type[Parser] | None = None,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> tuple[list[ConversationMessage], list[EngineInput]]:
|
||||
@@ -567,14 +559,6 @@ class OpenAIServingRender:
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
|
||||
if reasoning_parser is not None:
|
||||
tokenizer = renderer.get_tokenizer()
|
||||
request = reasoning_parser(
|
||||
tokenizer,
|
||||
model_config=self.model_config,
|
||||
chat_template_kwargs=chat_params.chat_template_kwargs,
|
||||
).adjust_request(request=request)
|
||||
|
||||
# tool parsing is done only if a tool_parser has been set and if
|
||||
# tool_choice is not "none" (if tool_choice is "none" but a tool_parser
|
||||
# is set, we want to prevent parsing a tool_call hallucinated by the LLM
|
||||
@@ -582,15 +566,22 @@ class OpenAIServingRender:
|
||||
# Exception: Mistral grammar-capable tokenizers always call
|
||||
# adjust_request — even for tool_choice="none" — so that the grammar
|
||||
# factory can prevent special-token leakage.
|
||||
if tool_parser is not None:
|
||||
tool_choice = getattr(request, "tool_choice", "none")
|
||||
if parser is not None:
|
||||
tokenizer = renderer.get_tokenizer()
|
||||
tool_parser = parser.tool_parser_cls
|
||||
tool_choice = getattr(request, "tool_choice", "none")
|
||||
is_mistral_grammar_eligible = (
|
||||
is_mistral_tool_parser(tool_parser)
|
||||
tool_parser is not None
|
||||
and is_mistral_tool_parser(tool_parser)
|
||||
and is_mistral_tokenizer(tokenizer)
|
||||
and tokenizer.supports_grammar
|
||||
)
|
||||
if tool_choice != "none" or is_mistral_grammar_eligible:
|
||||
should_adjust_request = (
|
||||
parser.reasoning_parser_cls is not None
|
||||
or tool_choice != "none"
|
||||
or is_mistral_grammar_eligible
|
||||
)
|
||||
if should_adjust_request:
|
||||
if not isinstance(request, ChatCompletionRequest | ResponsesRequest):
|
||||
msg = (
|
||||
"Tool usage is only supported "
|
||||
@@ -598,8 +589,13 @@ class OpenAIServingRender:
|
||||
f"but got {type(request).__name__}"
|
||||
)
|
||||
raise NotImplementedError(msg)
|
||||
request = tool_parser(tokenizer, request.tools).adjust_request(
|
||||
request=request
|
||||
request = parser(
|
||||
tokenizer,
|
||||
request.tools,
|
||||
model_config=self.model_config,
|
||||
chat_template_kwargs=chat_params.chat_template_kwargs,
|
||||
).adjust_request(
|
||||
request=request,
|
||||
)
|
||||
|
||||
return conversation, [engine_input]
|
||||
|
||||
Reference in New Issue
Block a user