[Refactor][Mistral] Extract parsing logic into MistralParser (#44596)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-06-10 00:12:23 +00:00
committed by GitHub
parent dac9e9a640
commit d82ac00923
7 changed files with 136 additions and 729 deletions
@@ -3,7 +3,6 @@
import json
from collections.abc import Generator
from typing import Any
from unittest.mock import MagicMock, patch
import partial_json_parser
@@ -29,22 +28,17 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.engine.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
ExtractedToolCallInformation,
StructuralTagResponseFormat,
)
from vllm.entrypoints.openai.engine.protocol import FunctionCall as VllmFunctionCall
from vllm.reasoning.mistral_reasoning_parser import MistralReasoningParser
from vllm.sampling_params import StructuredOutputsParams
from vllm.tokenizers import TokenizerLike, get_tokenizer
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tool_parsers.mistral_tool_parser import (
_DEFAULT_JSON_SCHEMA,
MistralStreamingResult,
MistralToolCall,
MistralToolParser,
)
@@ -1578,382 +1572,3 @@ def test_grammar_from_tool_parser_set_by_adjust_request(
request = _make_request()
result = mistral_tool_parser.adjust_request(request)
assert result._grammar_from_tool_parser is True
@pytest.mark.parametrize(
"tool_calls, expected_len",
[
(None, 0),
([], 0),
([VllmFunctionCall(id="abc123xyz", name="f", arguments="{}")], 1),
([VllmFunctionCall(name="f", arguments="{}")], 1),
(
[
VllmFunctionCall(id="fixed1234", name="a", arguments='{"x": 1}'),
VllmFunctionCall(name="b", arguments='{"y": 2}'),
],
2,
),
],
ids=["none", "empty", "with_id", "without_id", "mixed"],
)
def test_build_non_streaming_tool_calls(
tool_calls: list[VllmFunctionCall] | None,
expected_len: int,
) -> None:
result = MistralToolParser.build_non_streaming_tool_calls(tool_calls)
assert len(result) == expected_len
if tool_calls is None:
return
for i, tc in enumerate(result):
assert isinstance(tc, MistralToolCall)
assert tc.type == "function"
input_tc = tool_calls[i]
if input_tc.id:
assert tc.id == input_tc.id
else:
assert len(tc.id) == 9
assert tc.id.isalnum()
assert tc.function.name == input_tc.name
assert tc.function.arguments == input_tc.arguments
class TestExtractMaybeReasoningAndToolStreaming:
r"""Tests for `MistralToolParser.extract_maybe_reasoning_and_tool_streaming`."""
@pytest.fixture
def parser(self) -> MistralToolParser:
mock_tokenizer = MagicMock()
mock_tokenizer.get_vocab.return_value = {"[TOOL_CALLS]": 1}
return MistralToolParser(mock_tokenizer)
@pytest.fixture
def request_obj(self) -> ChatCompletionRequest:
return _make_request()
@staticmethod
def _call(
parser: MistralToolParser,
request: ChatCompletionRequest,
*,
reasoning_parser: Any = None,
previous_text: str = "",
current_text: str = "hello",
delta_text: str = "hello",
previous_token_ids: list[int] | None = None,
current_token_ids: list[int] | None = None,
output_token_ids: list[int] | None = None,
reasoning_ended: bool = False,
prompt_is_reasoning_end: bool | None = None,
) -> MistralStreamingResult:
return parser.extract_maybe_reasoning_and_tool_streaming(
reasoning_parser=reasoning_parser,
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=previous_token_ids or [],
current_token_ids=current_token_ids or [1, 2, 3],
output_token_ids=output_token_ids or [1, 2, 3],
reasoning_ended=reasoning_ended,
prompt_is_reasoning_end=prompt_is_reasoning_end,
request=request,
)
def test_no_reasoning_tools_called(
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
) -> None:
tool_delta = DeltaMessage(
tool_calls=[
DeltaToolCall(
index=0,
function=DeltaFunctionCall(name="f", arguments="{}"),
)
]
)
with patch.object(
parser, "extract_tool_calls_streaming", return_value=tool_delta
):
result = self._call(parser, request_obj, reasoning_parser=None)
assert result == MistralStreamingResult(
delta_message=tool_delta,
reasoning_ended=False,
tools_called=True,
current_text="hello",
current_token_ids=[1, 2, 3],
)
def test_no_reasoning_no_tools(
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
) -> None:
content_delta = DeltaMessage(content="hello")
with patch.object(
parser, "extract_tool_calls_streaming", return_value=content_delta
):
result = self._call(parser, request_obj, reasoning_parser=None)
assert result == MistralStreamingResult(
delta_message=content_delta,
reasoning_ended=False,
tools_called=False,
current_text="hello",
current_token_ids=[1, 2, 3],
)
def test_mistral_reasoning_parser_no_think_token(
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
) -> None:
mock_rp = MagicMock(spec=MistralReasoningParser)
mock_rp.start_token_id = 999
content_delta = DeltaMessage(content="direct")
with patch.object(
parser, "extract_tool_calls_streaming", return_value=content_delta
):
result = self._call(
parser,
request_obj,
reasoning_parser=mock_rp,
reasoning_ended=False,
current_token_ids=[1, 2, 3],
)
mock_rp.extract_reasoning_streaming.assert_not_called()
assert result == MistralStreamingResult(
delta_message=content_delta,
reasoning_ended=False,
tools_called=False,
current_text="hello",
current_token_ids=[1, 2, 3],
)
def test_mistral_reasoning_parser_with_think_token(
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
) -> None:
mock_rp = MagicMock(spec=MistralReasoningParser)
mock_rp.start_token_id = 999
mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(
reasoning="thinking..."
)
mock_rp.is_reasoning_end_streaming.return_value = False
result = self._call(
parser,
request_obj,
reasoning_parser=mock_rp,
reasoning_ended=False,
current_token_ids=[1, 999, 3],
)
mock_rp.extract_reasoning_streaming.assert_called_once()
assert result == MistralStreamingResult(
delta_message=DeltaMessage(reasoning="thinking..."),
reasoning_ended=False,
tools_called=False,
current_text="hello",
current_token_ids=[1, 999, 3],
)
def test_non_mistral_reasoning_parser_always_expects_thinking(
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
) -> None:
mock_rp = MagicMock()
mock_rp.start_token_id = 999
mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(
reasoning="thinking..."
)
mock_rp.is_reasoning_end_streaming.return_value = False
result = self._call(
parser,
request_obj,
reasoning_parser=mock_rp,
reasoning_ended=False,
current_token_ids=[1, 2, 3],
)
mock_rp.extract_reasoning_streaming.assert_called_once()
assert result == MistralStreamingResult(
delta_message=DeltaMessage(reasoning="thinking..."),
reasoning_ended=False,
tools_called=False,
current_text="hello",
current_token_ids=[1, 2, 3],
)
def test_reasoning_already_ended_no_reset(
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
) -> None:
content_delta = DeltaMessage(content="content")
with patch.object(
parser, "extract_tool_calls_streaming", return_value=content_delta
) as mock_extract:
result = self._call(
parser,
request_obj,
reasoning_parser=MagicMock(),
reasoning_ended=True,
previous_text="prior_tool_text",
previous_token_ids=[10, 20],
current_text="prior_tool_texthello",
current_token_ids=[10, 20, 1, 2, 3],
)
_, call_kwargs = mock_extract.call_args
assert call_kwargs["previous_text"] == "prior_tool_text"
assert call_kwargs["previous_token_ids"] == [10, 20]
assert result == MistralStreamingResult(
delta_message=content_delta,
reasoning_ended=True,
tools_called=False,
current_text="prior_tool_texthello",
current_token_ids=[10, 20, 1, 2, 3],
)
def test_pre_v15_ignores_prompt_reasoning_end(
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
) -> None:
mock_tokenizer = MagicMock(spec=MistralTokenizer)
mock_tokenizer.version = 13
parser.model_tokenizer = mock_tokenizer
mock_rp = MagicMock(spec=MistralReasoningParser)
mock_rp.start_token_id = 999
mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(
reasoning="thinking..."
)
mock_rp.is_reasoning_end_streaming.return_value = False
result = self._call(
parser,
request_obj,
reasoning_parser=mock_rp,
reasoning_ended=False,
prompt_is_reasoning_end=True,
current_token_ids=[999, 1, 2],
)
mock_rp.extract_reasoning_streaming.assert_called_once()
assert result == MistralStreamingResult(
delta_message=DeltaMessage(reasoning="thinking..."),
reasoning_ended=False,
tools_called=False,
current_text="hello",
current_token_ids=[999, 1, 2],
)
def test_non_pre_v15_prompt_reasoning_end(
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
) -> None:
mock_tokenizer = MagicMock(spec=MistralTokenizer)
mock_tokenizer.version = 15
parser.model_tokenizer = mock_tokenizer
mock_rp = MagicMock(spec=MistralReasoningParser)
mock_rp.start_token_id = 999
content_delta = DeltaMessage(content="after reasoning")
with patch.object(
parser, "extract_tool_calls_streaming", return_value=content_delta
):
result = self._call(
parser,
request_obj,
reasoning_parser=mock_rp,
reasoning_ended=False,
prompt_is_reasoning_end=True,
current_token_ids=[999, 1, 2],
output_token_ids=[10, 20, 30],
)
mock_rp.extract_reasoning_streaming.assert_not_called()
assert result == MistralStreamingResult(
delta_message=content_delta,
reasoning_ended=True,
tools_called=False,
current_text="hello",
current_token_ids=[10, 20, 30],
)
def test_reasoning_end_transition_with_content(
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
) -> None:
"""When reasoning ends and the delta has content, that content is
cleared from delta_message and used as current_text for tool parsing."""
mock_rp = MagicMock()
mock_rp.start_token_id = 999
mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(
reasoning="think", content="leftover"
)
mock_rp.is_reasoning_end_streaming.return_value = True
mock_rp.extract_content_ids.return_value = [50, 51]
content_delta = DeltaMessage(content="leftover")
with patch.object(
parser, "extract_tool_calls_streaming", return_value=content_delta
) as mock_extract:
result = self._call(
parser,
request_obj,
reasoning_parser=mock_rp,
reasoning_ended=False,
current_token_ids=[999, 1, 2],
output_token_ids=[10, 20, 30],
)
mock_rp.extract_content_ids.assert_called_once_with([10, 20, 30])
_, call_kwargs = mock_extract.call_args
assert call_kwargs["previous_text"] == ""
assert call_kwargs["previous_token_ids"] == []
assert call_kwargs["delta_text"] == "leftover"
assert call_kwargs["current_token_ids"] == [50, 51]
assert result == MistralStreamingResult(
delta_message=content_delta,
reasoning_ended=True,
tools_called=False,
current_text="leftover",
current_token_ids=[50, 51],
)
def test_reasoning_end_transition_without_content(
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
) -> None:
"""When reasoning ends but the delta has no content, current_text
is set to empty string."""
mock_rp = MagicMock()
mock_rp.start_token_id = 999
mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(
reasoning="think"
)
mock_rp.is_reasoning_end_streaming.return_value = True
mock_rp.extract_content_ids.return_value = [50, 51]
empty_delta = DeltaMessage(content="")
with patch.object(
parser, "extract_tool_calls_streaming", return_value=empty_delta
) as mock_extract:
result = self._call(
parser,
request_obj,
reasoning_parser=mock_rp,
reasoning_ended=False,
current_token_ids=[999, 1, 2],
output_token_ids=[10, 20, 30],
)
_, call_kwargs = mock_extract.call_args
assert call_kwargs["delta_text"] == ""
assert call_kwargs["current_token_ids"] == [50, 51]
assert result == MistralStreamingResult(
delta_message=empty_delta,
reasoning_ended=True,
tools_called=False,
current_text="",
current_token_ids=[50, 51],
)
@@ -72,7 +72,7 @@ from vllm.renderers import ChatParams
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.utils.collection_utils import as_list
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
from vllm.utils.mistral import is_mistral_tool_parser
if TYPE_CHECKING:
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
@@ -425,8 +425,6 @@ class OpenAIServingChat(OpenAIServing):
harmony_tools_streamed = [False] * num_choices
tools_streamed = [False] * num_choices
is_mistral_grammar_path = request._grammar_from_tool_parser
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
tool_choice_function_name = request.tool_choice.function.name
else:
@@ -450,15 +448,12 @@ class OpenAIServingChat(OpenAIServing):
# Only one of these will be used, thus previous_texts and
# all_previous_token_ids will not be used twice in the same iteration.
if (
is_mistral_grammar_path
or tool_choice_auto
tool_choice_auto
or tool_choice_function_name
or request.tool_choice == "required"
or reasoning_parser
):
all_previous_token_ids = [[] for _ in range(num_choices)]
reasoning_end_arr = [False] * num_choices
prompt_is_reasoning_end_arr: list[bool | None] = [None] * num_choices
else:
all_previous_token_ids = None
@@ -592,18 +587,6 @@ class OpenAIServingChat(OpenAIServing):
for output in res.outputs:
i = output.index
parser = parsers[i]
tool_parser = parser.tool_parser if parser is not None else None
if (
reasoning_parser
and res.prompt_token_ids
and prompt_is_reasoning_end_arr[i] is None
):
# only check once per choice, because prompt_token_ids
# are the same for all deltas in that choice
prompt_is_reasoning_end_arr[i] = (
reasoning_parser.is_reasoning_end(res.prompt_token_ids)
)
if finish_reason_sent[i]:
continue
@@ -658,8 +641,7 @@ class OpenAIServingChat(OpenAIServing):
# just update previous_texts and previous_token_ids
if (
is_mistral_grammar_path
or tool_choice_auto
tool_choice_auto
or tool_choice_function_name
or request.tool_choice == "required"
or reasoning_parser
@@ -687,35 +669,6 @@ class OpenAIServingChat(OpenAIServing):
)
)
harmony_tools_streamed[i] |= tools_streamed_flag
# Mistral grammar path: combined reasoning + tool streaming
elif is_mistral_grammar_path:
from vllm.tool_parsers.mistral_tool_parser import (
MistralToolParser,
)
assert tool_parser is not None
assert isinstance(tool_parser, MistralToolParser)
assert reasoning_end_arr is not None
output_token_ids = as_list(output.token_ids)
result = tool_parser.extract_maybe_reasoning_and_tool_streaming(
reasoning_parser=reasoning_parser,
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=previous_token_ids,
current_token_ids=current_token_ids,
output_token_ids=output_token_ids,
reasoning_ended=reasoning_end_arr[i],
prompt_is_reasoning_end=(prompt_is_reasoning_end_arr[i]),
request=request,
)
delta_message = result.delta_message
reasoning_end_arr[i] = result.reasoning_ended
current_text = result.current_text
current_token_ids = result.current_token_ids
if result.tools_called:
tools_streamed[i] = True
elif parser is not None:
delta_message = parser.parse_delta(
delta_text=delta_text,
@@ -732,8 +685,7 @@ class OpenAIServingChat(OpenAIServing):
# update the previous values for the next iteration
if (
is_mistral_grammar_path
or tool_choice_auto
tool_choice_auto
or tool_choice_function_name
or request.tool_choice == "required"
or reasoning_parser
@@ -1067,32 +1019,8 @@ class OpenAIServingChat(OpenAIServing):
tool_calls = []
auto_tools_called = False
if is_mistral_tokenizer(tokenizer):
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
tool_call_class: type[ToolCall] = MistralToolCall
else:
tool_call_class = ToolCall
use_mistral_tool_parser = request._grammar_from_tool_parser
if use_mistral_tool_parser:
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
tool_call_items = MistralToolParser.build_non_streaming_tool_calls(
tool_calls
)
if tool_call_items:
auto_tools_called = (
request.tool_choice is None or request.tool_choice == "auto"
)
message = ChatMessage(
role=role,
reasoning=reasoning,
content=content,
tool_calls=tool_call_items,
)
elif (not self.enable_auto_tools or not self.tool_parser) and (
if (not self.enable_auto_tools or not self.tool_parser) and (
not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
and request.tool_choice != "required"
):
@@ -1102,70 +1030,42 @@ class OpenAIServingChat(OpenAIServing):
request.tool_choice
and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
):
tool_call_class_items = []
tool_call_items = []
tool_calls = tool_calls or []
for idx, tc in enumerate(tool_calls):
# Use native ID if available (e.g., Kimi K2),
# otherwise generate ID with correct id_type
if tc.id:
tool_call_class_items.append(
tool_call_class(id=tc.id, function=tc)
for tc in tool_calls:
if not tc.id:
tc.id = make_tool_call_id(
id_type=self.tool_call_id_type,
func_name=tc.name,
idx=history_tool_call_cnt,
)
else:
# Generate ID using the correct format (kimi_k2 or random),
# but leave it to the class if it's Mistral to preserve
# 9-char IDs
if is_mistral_tokenizer(tokenizer):
tool_call_class_items.append(tool_call_class(function=tc))
else:
generated_id = make_tool_call_id(
id_type=self.tool_call_id_type,
func_name=tc.name,
idx=history_tool_call_cnt,
)
tool_call_class_items.append(
tool_call_class(id=generated_id, function=tc)
)
tool_call_items.append(ToolCall(id=tc.id, function=tc))
history_tool_call_cnt += 1
message = ChatMessage(
role=role,
reasoning=reasoning,
content="",
tool_calls=tool_call_class_items,
content=content or "",
tool_calls=tool_call_items,
)
elif request.tool_choice and request.tool_choice == "required":
tool_call_class_items = []
tool_call_items = []
tool_calls = tool_calls or []
for idx, tool_call in enumerate(tool_calls):
# Use native ID if available,
# otherwise generate ID with correct id_type
if tool_call.id:
tool_call_class_items.append(
tool_call_class(id=tool_call.id, function=tool_call)
for tool_call in tool_calls:
if not tool_call.id:
tool_call.id = make_tool_call_id(
id_type=self.tool_call_id_type,
func_name=tool_call.name,
idx=history_tool_call_cnt,
)
else:
# Generate ID using the correct format (kimi_k2 or random),
# but leave it to the class if it's Mistral to preserve
# 9-char IDs
if is_mistral_tokenizer(tokenizer):
tool_call_class_items.append(
tool_call_class(function=tool_call)
)
else:
generated_id = make_tool_call_id(
id_type=self.tool_call_id_type,
func_name=tool_call.name,
idx=history_tool_call_cnt,
)
tool_call_class_items.append(
tool_call_class(id=generated_id, function=tool_call)
)
tool_call_items.append(
ToolCall(id=tool_call.id, function=tool_call)
)
history_tool_call_cnt += 1
message = ChatMessage(
role=role,
content="",
tool_calls=tool_call_class_items,
content=content or "",
tool_calls=tool_call_items,
reasoning=reasoning,
)
@@ -1181,34 +1081,17 @@ class OpenAIServingChat(OpenAIServing):
and self.enable_auto_tools
and self.tool_parser
):
# In the OpenAI API the finish_reason is "tools_called"
# if the tool choice is auto and the model produced a tool
# call. The same is not true for named function calls
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
if tool_calls:
tool_call_items = []
for idx, tc in enumerate(tool_calls):
# Use native ID if available (e.g., Kimi K2),
# otherwise generate ID with correct id_type
if tc.id:
tool_call_items.append(
tool_call_class(id=tc.id, function=tc)
for tc in tool_calls:
if not tc.id:
tc.id = make_tool_call_id(
id_type=self.tool_call_id_type,
func_name=tc.name,
idx=history_tool_call_cnt,
)
else:
# Generate ID using the correct format (kimi_k2 or random),
# but leave it to the class if it's Mistral to preserve
# 9-char IDs
if is_mistral_tokenizer(tokenizer):
tool_call_items.append(tool_call_class(function=tc))
else:
generated_id = make_tool_call_id(
id_type=self.tool_call_id_type,
func_name=tc.name,
idx=history_tool_call_cnt,
)
tool_call_items.append(
tool_call_class(id=generated_id, function=tc)
)
tool_call_items.append(ToolCall(id=tc.id, function=tc))
history_tool_call_cnt += 1
message = ChatMessage(
role=role,
@@ -1218,18 +1101,10 @@ class OpenAIServingChat(OpenAIServing):
)
else:
# FOR NOW make it a chat message; we will have to detect
# the type to make it later.
ret_content = content
# try to use content return from tool parser first,
# tool parser may do some modify for the content.
if content and len(content) > 0:
ret_content = content
message = ChatMessage(
role=role,
reasoning=reasoning,
content=ret_content,
content=content,
)
# undetermined case that is still important to handle
+3 -20
View File
@@ -43,7 +43,6 @@ from vllm.tool_parsers.streaming import (
extract_required_tool_call_streaming,
)
from vllm.utils import random_uuid
from vllm.utils.mistral import is_mistral_tool_parser
logger = init_logger(__name__)
@@ -546,14 +545,6 @@ class DelegatingParser(Parser):
if tool_parser is None:
return [], content
# When the Mistral grammar factory injected structured outputs,
# let the parser handle the output.
use_mistral_tool_parser = (
is_mistral_tool_parser(type(tool_parser))
and isinstance(request, ChatCompletionRequest)
and request._grammar_from_tool_parser
)
supports_required_and_named = tool_parser.supports_required_and_named
is_named_tool_choice = request.tool_choice and isinstance(
request.tool_choice,
@@ -570,11 +561,7 @@ class DelegatingParser(Parser):
)
tool_calls = list[FunctionCall]()
if (
is_named_tool_choice
and supports_required_and_named
and not use_mistral_tool_parser
):
if is_named_tool_choice and supports_required_and_named:
if content is None:
return [], None
tool_calls.append(
@@ -584,11 +571,7 @@ class DelegatingParser(Parser):
)
)
content = None
elif (
is_required_tool_choice
and supports_required_and_named
and not use_mistral_tool_parser
):
elif is_required_tool_choice and supports_required_and_named:
# "required" with standard JSON-based parsing
parsed_calls = []
with contextlib.suppress(ValidationError):
@@ -604,7 +587,7 @@ class DelegatingParser(Parser):
)
)
content = None
elif is_auto_tool_choice or use_mistral_tool_parser:
elif is_auto_tool_choice:
# Automatic Tool Call Parsing (also used as fallback for
# required/named when supports_required_and_named=False)
tool_call_info = tool_parser.extract_tool_calls(
+77
View File
@@ -0,0 +1,77 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from typing import TYPE_CHECKING
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall
from vllm.parser.abstract_parser import DelegatingParser
if TYPE_CHECKING:
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
class MistralParser(DelegatingParser):
def __init__(self, tokenizer, tools=None, *args, **kwargs):
super().__init__(tokenizer, tools, *args, **kwargs)
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
if not isinstance(self._tool_parser, MistralToolParser):
raise ValueError(
"MistralParser requires --tool-call-parser mistral, "
f"got {self._tool_parser.__class__.__name__}."
)
def _maybe_force_auto_tool_parsing(
self, request: ChatCompletionRequest | ResponsesRequest
) -> None:
# When the Mistral grammar factory injected structured outputs,
# the model emits v11+ format ([TOOL_CALLS]name{args}) that the
# named/required parsers can't handle. Disable them so all
# tool_choice modes fall back to auto tool parsing via
# extract_tool_calls.
if getattr(request, "_grammar_from_tool_parser", False):
assert self._tool_parser is not None
self._tool_parser.supports_required_and_named = False
def parse(
self,
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
enable_auto_tools: bool = False,
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
self._maybe_force_auto_tool_parsing(request)
reasoning, content, tool_calls = super().parse(
model_output, request, enable_auto_tools
)
if tool_calls:
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
# Named/required tool_choice builds FunctionCalls without
# ID, backfill with Mistral-format IDs.
for tc in tool_calls:
if not tc.id:
tc.id = MistralToolCall.generate_random_id()
return reasoning, content, tool_calls
def parse_delta(
self,
delta_text: str,
delta_token_ids: list[int],
request: ChatCompletionRequest | ResponsesRequest,
prompt_token_ids: list[int] | None = None,
*,
finished: bool,
) -> DeltaMessage | None:
self._maybe_force_auto_tool_parsing(request)
return super().parse_delta(
delta_text,
delta_token_ids,
request,
prompt_token_ids,
finished=finished,
)
+9
View File
@@ -106,6 +106,15 @@ class ParserManager:
if reasoning_parser_cls is None and tool_parser_cls is None:
return None
from vllm.utils.mistral import is_mistral_tool_parser
if is_mistral_tool_parser(tool_parser_cls):
from vllm.parser.mistral import MistralParser
MistralParser.reasoning_parser_cls = reasoning_parser_cls
MistralParser.tool_parser_cls = tool_parser_cls
return MistralParser
from vllm.parser.abstract_parser import DelegatingParser
r_cls = reasoning_parser_cls
+10 -1
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from collections.abc import Iterable, Sequence
from functools import cached_property
from typing import TYPE_CHECKING
@@ -76,6 +76,15 @@ class MistralReasoningParser(BaseThinkingReasoningParser):
has_eot_token = True
return False
def is_reasoning_end_streaming(
self, input_ids: Sequence[int], delta_ids: Iterable[int]
) -> bool:
if self.end_token_id in delta_ids:
return True
# Grammar's think? is optional — if [THINK] was never generated,
# reasoning was skipped entirely.
return self.start_token_id not in input_ids
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
"""
Extract the content
+2 -163
View File
@@ -5,11 +5,10 @@ from __future__ import annotations
import json
from collections.abc import Sequence
from dataclasses import dataclass
from enum import Enum, auto
from random import choices
from string import ascii_letters, digits
from typing import TYPE_CHECKING, Any
from typing import Any
import ijson
import regex as re
@@ -40,19 +39,14 @@ from vllm.entrypoints.openai.engine.protocol import (
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.logger import init_logger
from vllm.reasoning.mistral_reasoning_parser import MistralReasoningParser
from vllm.sampling_params import StructuredOutputsParams
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tool_parsers.abstract_tool_parser import (
Tool,
ToolParser,
)
from vllm.utils.mistral import is_mistral_tokenizer
if TYPE_CHECKING:
from vllm.reasoning import ReasoningParser
logger = init_logger(__name__)
ALPHANUMERIC = ascii_letters + digits
@@ -99,19 +93,6 @@ def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool:
return "[ARGS]" not in vocab
@dataclass
class MistralStreamingResult:
r"""Encapsulates the mutable state returned from
`MistralToolParser.extract_maybe_reasoning_and_tool_streaming`.
"""
delta_message: DeltaMessage | None
reasoning_ended: bool
tools_called: bool
current_text: str
current_token_ids: list[int]
class MistralToolParser(ToolParser):
r"""Tool call parser for Mistral models, intended for use with either:
@@ -281,148 +262,6 @@ class MistralToolParser(ToolParser):
request._grammar_from_tool_parser = True
return request
def extract_maybe_reasoning_and_tool_streaming(
self,
*,
reasoning_parser: ReasoningParser | None,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: list[int],
current_token_ids: list[int],
output_token_ids: Sequence[int],
reasoning_ended: bool,
prompt_is_reasoning_end: bool | None,
request: ChatCompletionRequest,
) -> MistralStreamingResult:
r"""Streaming extraction with reasoning followed by tool-call parsing.
This method encapsulates the combined reasoning extraction and
tool-call streaming logic so that the serving layer only needs a
thin routing branch.
The flow is:
1. If a *reasoning_parser* is present and reasoning has **not** ended,
extract reasoning tokens. Pre-v15 models may have pre-filled
`[THINK]...[/THINK]` in system prompts, so we skip the
prompt-level reasoning-end check for those.
2. Once reasoning ends (or if there is no reasoning parser), delegate
to `extract_tool_calls_streaming` and track whether tools were
called.
Args:
reasoning_parser: Optional reasoning parser instance.
previous_text: Accumulated text from prior chunks.
current_text: Full accumulated text including current chunk.
delta_text: New text in this chunk.
previous_token_ids: Token ids from prior chunks.
current_token_ids: Full token ids including current chunk.
output_token_ids: Raw output token ids from the engine.
reasoning_ended: Whether reasoning has already ended.
prompt_is_reasoning_end: Whether the prompt itself ends reasoning.
request: The originating chat completion request.
"""
delta_message: DeltaMessage | None = None
tools_called = False
reasoning_ended_at_entry = reasoning_ended
# For MistralReasoningParser, only enter the reasoning block when
# the model has actually emitted a [THINK] token. Other reasoning
# parsers always expect thinking to be present.
expect_thinking = (
not isinstance(reasoning_parser, MistralReasoningParser)
or reasoning_parser.start_token_id in current_token_ids
)
if reasoning_parser is not None and not reasoning_ended and expect_thinking:
# Pre-v15 models may have pre-filled [THINK]...[/THINK] in
# system prompts, so skip the prompt-level reasoning-end
# check and wait for the output's own end-of-think.
is_pre_v15 = (
isinstance(self.model_tokenizer, MistralTokenizer)
and self.model_tokenizer.version < 15
)
if not is_pre_v15 and prompt_is_reasoning_end:
reasoning_ended = True
current_token_ids = list(output_token_ids)
else:
delta_message = reasoning_parser.extract_reasoning_streaming(
previous_text,
current_text,
delta_text,
previous_token_ids,
current_token_ids,
output_token_ids,
)
if reasoning_parser.is_reasoning_end_streaming(
current_token_ids, output_token_ids
):
reasoning_ended = True
current_token_ids = reasoning_parser.extract_content_ids(
list(output_token_ids)
)
if delta_message and delta_message.content:
current_text = delta_message.content
delta_message.content = None
else:
current_text = ""
if not reasoning_ended:
return MistralStreamingResult(
delta_message=delta_message,
reasoning_ended=False,
tools_called=False,
current_text=current_text,
current_token_ids=current_token_ids,
)
delta_token_ids = list(output_token_ids)
# On the iteration where reasoning just ended, reset the text/token
# state so the tool parser sees a clean history instead of the
# accumulated reasoning text.
if not reasoning_ended_at_entry and reasoning_ended:
previous_text = ""
previous_token_ids = []
delta_text = current_text
delta_token_ids = current_token_ids
delta_message = self.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=previous_token_ids,
current_token_ids=current_token_ids,
delta_token_ids=delta_token_ids,
request=request,
)
if delta_message and delta_message.tool_calls:
tools_called = True
return MistralStreamingResult(
delta_message=delta_message,
reasoning_ended=reasoning_ended,
tools_called=tools_called,
current_text=current_text,
current_token_ids=current_token_ids,
)
@staticmethod
def build_non_streaming_tool_calls(
tool_calls: list[FunctionCall] | None,
) -> list[ToolCall]:
r"""Build `MistralToolCall` items for non-streaming responses."""
if not tool_calls:
return []
return [
MistralToolCall(id=tc.id, function=tc)
if tc.id
else MistralToolCall(function=tc)
for tc in tool_calls
]
def extract_tool_calls(
self,
model_output: str,
@@ -536,7 +375,7 @@ class MistralToolParser(ToolParser):
return ExtractedToolCallInformation(
tools_called=True,
tool_calls=mistral_tool_calls,
content=content if len(content) > 0 else None,
content=content if content.strip() else None,
)
def extract_tool_calls_streaming(