[Frontend] Support strict mode for tool calling (#45003)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Co-authored-by: cjackal <44624812+cjackal@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Chauncey
2026-06-12 07:51:48 +00:00
committed by GitHub
co-authored by cjackal mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent bd59c913bc
commit 2043258dec
29 changed files with 692 additions and 1956 deletions
-14
View File
@@ -308,20 +308,6 @@ async def init_app_state(
) -> None:
vllm_config = engine_client.vllm_config
# Propagate enable_in_reasoning to the API-server process. The engine core
# runs in a separate process, so the contextvar that backs
# `get_current_vllm_config_or_none()` is None on this stack. Tool parsers
# call `get_enable_structured_outputs_in_reasoning()` during request
# handling and need to see the real flag, otherwise they silently fall
# back to False and mismatch the engine-side bitmask gating.
from vllm.tool_parsers.structural_tag_registry import (
set_enable_structured_outputs_in_reasoning,
)
set_enable_structured_outputs_in_reasoning(
vllm_config.structured_outputs_config.enable_in_reasoning
)
if args.tool_call_parser is not None:
from vllm.parser.metrics import init_parser_metrics
@@ -74,7 +74,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
if error_check_ret is not None:
return error_check_ret
tool_parser = render.tool_parser
parser = render.parser
tool_dicts: list[dict] | None = None
all_conversations: list[list[ConversationMessage]] = []
@@ -94,7 +94,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
default_template_content_format=render.chat_template_content_format,
default_template_kwargs=render.default_chat_template_kwargs,
tool_dicts=tool_dicts,
tool_parser=tool_parser,
parser=parser,
)
all_conversations.append(conversation)
all_engine_prompts.append(engine_prompts[0])
+5 -8
View File
@@ -102,10 +102,9 @@ from vllm.logprobs import Logprob as SampleLogprob
from vllm.logprobs import SampleLogprobs
from vllm.lora.request import LoRARequest
from vllm.outputs import CompletionOutput
from vllm.parser import ParserManager
from vllm.parser import Parser, ParserManager
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers import ToolParser
from vllm.utils import random_uuid
from vllm.utils.collection_utils import as_list
@@ -613,8 +612,7 @@ class OpenAIServingResponses(OpenAIServing):
default_template_content_format=self.chat_template_content_format,
default_template_kwargs=chat_template_kwargs,
tool_dicts=tool_dicts,
tool_parser=self.parser.tool_parser_cls if self.parser else None,
reasoning_parser=self.parser.reasoning_parser_cls if self.parser else None,
parser=self.parser,
)
return messages, engine_inputs
@@ -623,7 +621,7 @@ class OpenAIServingResponses(OpenAIServing):
request: ResponsesRequest,
messages: list[ResponseInputOutputItem],
tool_dicts: list[dict[str, Any]] | None,
tool_parser: type[ToolParser] | None,
parser: type[Parser] | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
):
@@ -638,8 +636,7 @@ class OpenAIServingResponses(OpenAIServing):
default_template_content_format=chat_template_content_format,
default_template_kwargs=chat_template_kwargs,
tool_dicts=tool_dicts,
tool_parser=tool_parser,
reasoning_parser=self.parser.reasoning_parser_cls if self.parser else None,
parser=parser,
)
return engine_inputs
@@ -707,7 +704,7 @@ class OpenAIServingResponses(OpenAIServing):
context.request,
context.parser.response_messages,
context.tool_dicts,
context.parser_cls.tool_parser_cls if context.parser_cls else None,
context.parser_cls,
context.chat_template,
context.chat_template_content_format,
)
+24 -28
View File
@@ -43,8 +43,7 @@ from vllm.inputs import (
tokens_input,
)
from vllm.logger import init_logger
from vllm.parser import ParserManager
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
from vllm.parser import Parser, ParserManager
from vllm.renderers import BaseRenderer, merge_kwargs
from vllm.renderers.inputs.preprocess import (
extract_prompt_components,
@@ -52,7 +51,6 @@ from vllm.renderers.inputs.preprocess import (
parse_model_prompt,
prompt_to_seq,
)
from vllm.tool_parsers import ToolParser
from vllm.utils import random_uuid
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
from vllm.utils.mistral import mt as _mt
@@ -89,16 +87,12 @@ class OpenAIServingRender:
self.trust_request_chat_template = trust_request_chat_template
self.enable_auto_tools = enable_auto_tools
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
self.tool_parser: type[ToolParser] | None = ParserManager.get_tool_parser(
self.parser: type[Parser] | None = ParserManager.get_parser(
tool_parser_name=tool_parser,
reasoning_parser_name=reasoning_parser,
enable_auto_tools=enable_auto_tools,
model_name=model_config.model,
)
self.reasoning_parser: type[ReasoningParser] | None = (
ParserManager.get_reasoning_parser(
reasoning_parser_name=reasoning_parser,
)
)
self.default_chat_template_kwargs: dict[str, Any] = (
default_chat_template_kwargs or {}
)
@@ -193,7 +187,7 @@ class OpenAIServingRender:
"""
tokenizer = self.renderer.tokenizer
tool_parser = self.tool_parser
tool_parser = self.parser.tool_parser_cls if self.parser is not None else None
if is_mistral_tokenizer(tokenizer):
# because of issues with pydantic we need to potentially
@@ -252,9 +246,8 @@ class OpenAIServingRender:
default_template_content_format=self.chat_template_content_format,
default_template_kwargs=self.default_chat_template_kwargs,
tool_dicts=tool_dicts,
tool_parser=tool_parser,
parser=self.parser,
skip_mm_cache=skip_mm_cache,
reasoning_parser=self.reasoning_parser,
)
else:
# For GPT-OSS.
@@ -526,8 +519,7 @@ class OpenAIServingRender:
default_template_content_format: ChatTemplateContentFormatOption,
default_template_kwargs: dict[str, Any] | None,
tool_dicts: list[dict[str, Any]] | None = None,
tool_parser: type[ToolParser] | None = None,
reasoning_parser: type[ReasoningParser] | None = None,
parser: type[Parser] | None = None,
*,
skip_mm_cache: bool = False,
) -> tuple[list[ConversationMessage], list[EngineInput]]:
@@ -567,14 +559,6 @@ class OpenAIServingRender:
skip_mm_cache=skip_mm_cache,
)
if reasoning_parser is not None:
tokenizer = renderer.get_tokenizer()
request = reasoning_parser(
tokenizer,
model_config=self.model_config,
chat_template_kwargs=chat_params.chat_template_kwargs,
).adjust_request(request=request)
# tool parsing is done only if a tool_parser has been set and if
# tool_choice is not "none" (if tool_choice is "none" but a tool_parser
# is set, we want to prevent parsing a tool_call hallucinated by the LLM
@@ -582,15 +566,22 @@ class OpenAIServingRender:
# Exception: Mistral grammar-capable tokenizers always call
# adjust_request — even for tool_choice="none" — so that the grammar
# factory can prevent special-token leakage.
if tool_parser is not None:
tool_choice = getattr(request, "tool_choice", "none")
if parser is not None:
tokenizer = renderer.get_tokenizer()
tool_parser = parser.tool_parser_cls
tool_choice = getattr(request, "tool_choice", "none")
is_mistral_grammar_eligible = (
is_mistral_tool_parser(tool_parser)
tool_parser is not None
and is_mistral_tool_parser(tool_parser)
and is_mistral_tokenizer(tokenizer)
and tokenizer.supports_grammar
)
if tool_choice != "none" or is_mistral_grammar_eligible:
should_adjust_request = (
parser.reasoning_parser_cls is not None
or tool_choice != "none"
or is_mistral_grammar_eligible
)
if should_adjust_request:
if not isinstance(request, ChatCompletionRequest | ResponsesRequest):
msg = (
"Tool usage is only supported "
@@ -598,8 +589,13 @@ class OpenAIServingRender:
f"but got {type(request).__name__}"
)
raise NotImplementedError(msg)
request = tool_parser(tokenizer, request.tools).adjust_request(
request=request
request = parser(
tokenizer,
request.tools,
model_config=self.model_config,
chat_template_kwargs=chat_params.chat_template_kwargs,
).adjust_request(
request=request,
)
return conversation, [engine_input]