[Mistral Grammar] Fix tool and reasoning parsing (#39217)

Signed-off-by: juliendenize <julien.denize@mistral.ai>
This commit is contained in:
Julien Denize
2026-04-15 21:05:04 -07:00
committed by GitHub
parent 951dca8019
commit c0722f22de
10 changed files with 1604 additions and 269 deletions
@@ -11,7 +11,7 @@ from openai.types.chat.chat_completion_audio import (
ChatCompletionAudio as OpenAIChatCompletionAudio,
)
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
from pydantic import Field, model_validator
from pydantic import Field, PrivateAttr, model_validator
from vllm.config import ModelConfig
from vllm.config.utils import replace
@@ -398,6 +398,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
msg["tool_calls"] = list(tool_calls)
return self
_grammar_from_tool_parser: bool = PrivateAttr(default=False)
"""CAUTION: Should only be set by ``ToolParser.adjust_request``."""
def build_chat_params(
self,
default_template: str | None,
@@ -822,13 +825,6 @@ class ChatCompletionRequest(OpenAIBaseModel):
return data
@model_validator(mode="before")
@classmethod
def set_include_reasoning_for_none_effort(cls, data: Any) -> Any:
if data.get("reasoning_effort") == "none":
data["include_reasoning"] = False
return data
class BatchChatCompletionRequest(OpenAIBaseModel):
"""Request model for the /v1/chat/completions/batch endpoint.
@@ -73,7 +73,10 @@ from vllm.reasoning import ReasoningParser
from vllm.renderers import ChatParams
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
from vllm.tool_parsers.mistral_tool_parser import (
MistralToolCall,
MistralToolParser,
)
from vllm.tool_parsers.utils import partial_json_loads
from vllm.utils.collection_utils import as_list
from vllm.utils.mistral import is_mistral_tokenizer
@@ -140,6 +143,12 @@ class OpenAIServingChat(OpenAIServing):
enable_auto_tools=enable_auto_tools,
model_name=self.model_config.model,
)
_is_mistral_tool_parser = self.tool_parser is not None and issubclass(
self.tool_parser, MistralToolParser
)
if _is_mistral_tool_parser and self.reasoning_parser_cls is not None:
MistralToolParser.model_can_reason = True
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
self.enable_prompt_tokens_details = enable_prompt_tokens_details
@@ -310,6 +319,11 @@ class OpenAIServingChat(OpenAIServing):
else:
if not request.include_reasoning:
reasoning_ended = True
elif request._grammar_from_tool_parser:
# The Mistral grammar already includes an optional
# `think?` rule that handles both reasoning and
# non-reasoning outputs.
reasoning_ended = True
elif reasoning_parser:
reasoning_ended = reasoning_parser.is_reasoning_end(
prompt_token_ids or []
@@ -530,6 +544,8 @@ class OpenAIServingChat(OpenAIServing):
harmony_tools_streamed = [False] * num_choices
tools_streamed = [False] * num_choices
is_mistral_grammar_path = request._grammar_from_tool_parser
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
tool_choice_function_name = request.tool_choice.function.name
else:
@@ -553,7 +569,7 @@ class OpenAIServingChat(OpenAIServing):
# Only one of these will be used, thus previous_texts and
# all_previous_token_ids will not be used twice in the same iteration.
if tool_choice_auto or reasoning_parser:
if is_mistral_grammar_path or tool_choice_auto or reasoning_parser:
# These are only required in "auto" tool choice case
all_previous_token_ids = [[] for _ in range(num_choices)]
reasoning_end_arr = [False] * num_choices
@@ -748,7 +764,7 @@ class OpenAIServingChat(OpenAIServing):
delta_message: DeltaMessage | None
# just update previous_texts and previous_token_ids
if tool_choice_auto or reasoning_parser:
if is_mistral_grammar_path or tool_choice_auto or reasoning_parser:
assert previous_texts is not None
assert all_previous_token_ids is not None
previous_text = previous_texts[i]
@@ -772,6 +788,30 @@ class OpenAIServingChat(OpenAIServing):
)
)
harmony_tools_streamed[i] |= tools_streamed_flag
# Mistral grammar path: combined reasoning + tool streaming
elif is_mistral_grammar_path:
assert tool_parser is not None
assert isinstance(tool_parser, MistralToolParser)
assert reasoning_end_arr is not None
output_token_ids = as_list(output.token_ids)
result = tool_parser.extract_maybe_reasoning_and_tool_streaming(
reasoning_parser=reasoning_parser,
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=previous_token_ids,
current_token_ids=current_token_ids,
output_token_ids=output_token_ids,
reasoning_ended=reasoning_end_arr[i],
prompt_is_reasoning_end=(prompt_is_reasoning_end_arr[i]),
request=request,
)
delta_message = result.delta_message
reasoning_end_arr[i] = result.reasoning_ended
current_text = result.current_text
current_token_ids = result.current_token_ids
if result.tools_called:
tools_streamed[i] = True
# handle streaming deltas for tools with named tool_choice
elif tool_choice_function_name:
# When encountering think end id in prompt_token_ids
@@ -925,7 +965,9 @@ class OpenAIServingChat(OpenAIServing):
delta_message = DeltaMessage(content=delta_text)
# update the previous values for the next iteration
if (tool_choice_auto or reasoning_parser) and not self.use_harmony:
if (
is_mistral_grammar_path or tool_choice_auto or reasoning_parser
) and not self.use_harmony:
assert previous_texts is not None
assert all_previous_token_ids is not None
previous_texts[i] = current_text
@@ -1312,7 +1354,24 @@ class OpenAIServingChat(OpenAIServing):
tool_call_class = (
MistralToolCall if is_mistral_tokenizer(tokenizer) else ToolCall
)
if (not self.enable_auto_tools or not self.tool_parser) and (
use_mistral_tool_parser = request._grammar_from_tool_parser
if use_mistral_tool_parser:
tool_call_items = MistralToolParser.build_non_streaming_tool_calls(
tool_calls
)
if tool_call_items:
auto_tools_called = (
request.tool_choice is None or request.tool_choice == "auto"
)
message = ChatMessage(
role=role,
reasoning=reasoning,
content=content,
tool_calls=tool_call_items,
)
elif (not self.enable_auto_tools or not self.tool_parser) and (
not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
and request.tool_choice != "required"
):
+26 -8
View File
@@ -65,6 +65,7 @@ from vllm.renderers.inputs.preprocess import (
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers import ToolParser
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
from vllm.tracing import (
contains_trace_headers,
extract_trace_headers,
@@ -610,16 +611,31 @@ class OpenAIServing:
tool_parser_cls: type[ToolParser] | None,
content: str | None = None,
) -> tuple[list[FunctionCall] | None, str | None]:
# When the Mistral grammar factory injected structured outputs,
# let the parser handle the output.
use_mistral_tool_parser = (
isinstance(request, ChatCompletionRequest)
and tool_parser_cls is not None
and issubclass(tool_parser_cls, MistralToolParser)
and request._grammar_from_tool_parser
)
function_calls = list[FunctionCall]()
if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction):
if (
not use_mistral_tool_parser
and request.tool_choice
and isinstance(request.tool_choice, ToolChoiceFunction)
):
assert content is not None
# Forced Function Call
function_calls.append(
FunctionCall(name=request.tool_choice.name, arguments=content)
)
content = None # Clear content since tool is called.
elif request.tool_choice and isinstance(
request.tool_choice, ChatCompletionNamedToolChoiceParam
elif (
not use_mistral_tool_parser
and request.tool_choice
and isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
):
assert content is not None
# Forced Function Call
@@ -627,7 +643,7 @@ class OpenAIServing:
FunctionCall(name=request.tool_choice.function.name, arguments=content)
)
content = None # Clear content since tool is called.
elif request.tool_choice == "required":
elif not use_mistral_tool_parser and request.tool_choice == "required":
tool_calls = []
with contextlib.suppress(ValidationError):
content = content or ""
@@ -642,10 +658,12 @@ class OpenAIServing:
)
)
content = None # Clear content since tool is called.
elif (
tool_parser_cls
and enable_auto_tools
and (request.tool_choice == "auto" or request.tool_choice is None)
elif tool_parser_cls and (
use_mistral_tool_parser
or (
enable_auto_tools
and (request.tool_choice == "auto" or request.tool_choice is None)
)
):
if tokenizer is None:
raise ValueError(
+12 -2
View File
@@ -53,6 +53,7 @@ from vllm.renderers.inputs.preprocess import (
prompt_to_seq,
)
from vllm.tool_parsers import ToolParser
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
from vllm.utils import random_uuid
from vllm.utils.mistral import is_mistral_tokenizer
from vllm.utils.mistral import mt as _mt
@@ -555,9 +556,19 @@ class OpenAIServingRender:
# tool parsing is done only if a tool_parser has been set and if
# tool_choice is not "none" (if tool_choice is "none" but a tool_parser
# is set, we want to prevent parsing a tool_call hallucinated by the LLM
#
# Exception: Mistral grammar-capable tokenizers always call
# adjust_request — even for tool_choice="none" — so that the grammar
# factory can prevent special-token leakage.
if tool_parser is not None:
tool_choice = getattr(request, "tool_choice", "none")
if tool_choice != "none":
tokenizer = renderer.get_tokenizer()
is_mistral_grammar_eligible = (
issubclass(tool_parser, MistralToolParser)
and is_mistral_tokenizer(tokenizer)
and tokenizer.supports_grammar
)
if tool_choice != "none" or is_mistral_grammar_eligible:
if not isinstance(request, ChatCompletionRequest | ResponsesRequest):
msg = (
"Tool usage is only supported "
@@ -565,7 +576,6 @@ class OpenAIServingRender:
f"but got {type(request).__name__}"
)
raise NotImplementedError(msg)
tokenizer = renderer.get_tokenizer()
request = tool_parser(tokenizer, request.tools).adjust_request(
request=request
)