[Refactor][Mistral] Extract parsing logic into MistralParser (#44596)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -3,7 +3,6 @@
|
||||
|
||||
import json
|
||||
from collections.abc import Generator
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import partial_json_parser
|
||||
@@ -29,22 +28,17 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
ExtractedToolCallInformation,
|
||||
StructuralTagResponseFormat,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import FunctionCall as VllmFunctionCall
|
||||
from vllm.reasoning.mistral_reasoning_parser import MistralReasoningParser
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.tool_parsers.mistral_tool_parser import (
|
||||
_DEFAULT_JSON_SCHEMA,
|
||||
MistralStreamingResult,
|
||||
MistralToolCall,
|
||||
MistralToolParser,
|
||||
)
|
||||
|
||||
@@ -1578,382 +1572,3 @@ def test_grammar_from_tool_parser_set_by_adjust_request(
|
||||
request = _make_request()
|
||||
result = mistral_tool_parser.adjust_request(request)
|
||||
assert result._grammar_from_tool_parser is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool_calls, expected_len",
|
||||
[
|
||||
(None, 0),
|
||||
([], 0),
|
||||
([VllmFunctionCall(id="abc123xyz", name="f", arguments="{}")], 1),
|
||||
([VllmFunctionCall(name="f", arguments="{}")], 1),
|
||||
(
|
||||
[
|
||||
VllmFunctionCall(id="fixed1234", name="a", arguments='{"x": 1}'),
|
||||
VllmFunctionCall(name="b", arguments='{"y": 2}'),
|
||||
],
|
||||
2,
|
||||
),
|
||||
],
|
||||
ids=["none", "empty", "with_id", "without_id", "mixed"],
|
||||
)
|
||||
def test_build_non_streaming_tool_calls(
|
||||
tool_calls: list[VllmFunctionCall] | None,
|
||||
expected_len: int,
|
||||
) -> None:
|
||||
result = MistralToolParser.build_non_streaming_tool_calls(tool_calls)
|
||||
assert len(result) == expected_len
|
||||
|
||||
if tool_calls is None:
|
||||
return
|
||||
|
||||
for i, tc in enumerate(result):
|
||||
assert isinstance(tc, MistralToolCall)
|
||||
assert tc.type == "function"
|
||||
|
||||
input_tc = tool_calls[i]
|
||||
if input_tc.id:
|
||||
assert tc.id == input_tc.id
|
||||
else:
|
||||
assert len(tc.id) == 9
|
||||
assert tc.id.isalnum()
|
||||
|
||||
assert tc.function.name == input_tc.name
|
||||
assert tc.function.arguments == input_tc.arguments
|
||||
|
||||
|
||||
class TestExtractMaybeReasoningAndToolStreaming:
|
||||
r"""Tests for `MistralToolParser.extract_maybe_reasoning_and_tool_streaming`."""
|
||||
|
||||
@pytest.fixture
|
||||
def parser(self) -> MistralToolParser:
|
||||
mock_tokenizer = MagicMock()
|
||||
mock_tokenizer.get_vocab.return_value = {"[TOOL_CALLS]": 1}
|
||||
return MistralToolParser(mock_tokenizer)
|
||||
|
||||
@pytest.fixture
|
||||
def request_obj(self) -> ChatCompletionRequest:
|
||||
return _make_request()
|
||||
|
||||
@staticmethod
|
||||
def _call(
|
||||
parser: MistralToolParser,
|
||||
request: ChatCompletionRequest,
|
||||
*,
|
||||
reasoning_parser: Any = None,
|
||||
previous_text: str = "",
|
||||
current_text: str = "hello",
|
||||
delta_text: str = "hello",
|
||||
previous_token_ids: list[int] | None = None,
|
||||
current_token_ids: list[int] | None = None,
|
||||
output_token_ids: list[int] | None = None,
|
||||
reasoning_ended: bool = False,
|
||||
prompt_is_reasoning_end: bool | None = None,
|
||||
) -> MistralStreamingResult:
|
||||
return parser.extract_maybe_reasoning_and_tool_streaming(
|
||||
reasoning_parser=reasoning_parser,
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=previous_token_ids or [],
|
||||
current_token_ids=current_token_ids or [1, 2, 3],
|
||||
output_token_ids=output_token_ids or [1, 2, 3],
|
||||
reasoning_ended=reasoning_ended,
|
||||
prompt_is_reasoning_end=prompt_is_reasoning_end,
|
||||
request=request,
|
||||
)
|
||||
|
||||
def test_no_reasoning_tools_called(
|
||||
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
|
||||
) -> None:
|
||||
tool_delta = DeltaMessage(
|
||||
tool_calls=[
|
||||
DeltaToolCall(
|
||||
index=0,
|
||||
function=DeltaFunctionCall(name="f", arguments="{}"),
|
||||
)
|
||||
]
|
||||
)
|
||||
with patch.object(
|
||||
parser, "extract_tool_calls_streaming", return_value=tool_delta
|
||||
):
|
||||
result = self._call(parser, request_obj, reasoning_parser=None)
|
||||
|
||||
assert result == MistralStreamingResult(
|
||||
delta_message=tool_delta,
|
||||
reasoning_ended=False,
|
||||
tools_called=True,
|
||||
current_text="hello",
|
||||
current_token_ids=[1, 2, 3],
|
||||
)
|
||||
|
||||
def test_no_reasoning_no_tools(
|
||||
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
|
||||
) -> None:
|
||||
content_delta = DeltaMessage(content="hello")
|
||||
with patch.object(
|
||||
parser, "extract_tool_calls_streaming", return_value=content_delta
|
||||
):
|
||||
result = self._call(parser, request_obj, reasoning_parser=None)
|
||||
|
||||
assert result == MistralStreamingResult(
|
||||
delta_message=content_delta,
|
||||
reasoning_ended=False,
|
||||
tools_called=False,
|
||||
current_text="hello",
|
||||
current_token_ids=[1, 2, 3],
|
||||
)
|
||||
|
||||
def test_mistral_reasoning_parser_no_think_token(
|
||||
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
|
||||
) -> None:
|
||||
mock_rp = MagicMock(spec=MistralReasoningParser)
|
||||
mock_rp.start_token_id = 999
|
||||
content_delta = DeltaMessage(content="direct")
|
||||
with patch.object(
|
||||
parser, "extract_tool_calls_streaming", return_value=content_delta
|
||||
):
|
||||
result = self._call(
|
||||
parser,
|
||||
request_obj,
|
||||
reasoning_parser=mock_rp,
|
||||
reasoning_ended=False,
|
||||
current_token_ids=[1, 2, 3],
|
||||
)
|
||||
|
||||
mock_rp.extract_reasoning_streaming.assert_not_called()
|
||||
assert result == MistralStreamingResult(
|
||||
delta_message=content_delta,
|
||||
reasoning_ended=False,
|
||||
tools_called=False,
|
||||
current_text="hello",
|
||||
current_token_ids=[1, 2, 3],
|
||||
)
|
||||
|
||||
def test_mistral_reasoning_parser_with_think_token(
|
||||
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
|
||||
) -> None:
|
||||
mock_rp = MagicMock(spec=MistralReasoningParser)
|
||||
mock_rp.start_token_id = 999
|
||||
mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(
|
||||
reasoning="thinking..."
|
||||
)
|
||||
mock_rp.is_reasoning_end_streaming.return_value = False
|
||||
|
||||
result = self._call(
|
||||
parser,
|
||||
request_obj,
|
||||
reasoning_parser=mock_rp,
|
||||
reasoning_ended=False,
|
||||
current_token_ids=[1, 999, 3],
|
||||
)
|
||||
|
||||
mock_rp.extract_reasoning_streaming.assert_called_once()
|
||||
assert result == MistralStreamingResult(
|
||||
delta_message=DeltaMessage(reasoning="thinking..."),
|
||||
reasoning_ended=False,
|
||||
tools_called=False,
|
||||
current_text="hello",
|
||||
current_token_ids=[1, 999, 3],
|
||||
)
|
||||
|
||||
def test_non_mistral_reasoning_parser_always_expects_thinking(
|
||||
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
|
||||
) -> None:
|
||||
mock_rp = MagicMock()
|
||||
mock_rp.start_token_id = 999
|
||||
mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(
|
||||
reasoning="thinking..."
|
||||
)
|
||||
mock_rp.is_reasoning_end_streaming.return_value = False
|
||||
|
||||
result = self._call(
|
||||
parser,
|
||||
request_obj,
|
||||
reasoning_parser=mock_rp,
|
||||
reasoning_ended=False,
|
||||
current_token_ids=[1, 2, 3],
|
||||
)
|
||||
|
||||
mock_rp.extract_reasoning_streaming.assert_called_once()
|
||||
assert result == MistralStreamingResult(
|
||||
delta_message=DeltaMessage(reasoning="thinking..."),
|
||||
reasoning_ended=False,
|
||||
tools_called=False,
|
||||
current_text="hello",
|
||||
current_token_ids=[1, 2, 3],
|
||||
)
|
||||
|
||||
def test_reasoning_already_ended_no_reset(
|
||||
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
|
||||
) -> None:
|
||||
content_delta = DeltaMessage(content="content")
|
||||
with patch.object(
|
||||
parser, "extract_tool_calls_streaming", return_value=content_delta
|
||||
) as mock_extract:
|
||||
result = self._call(
|
||||
parser,
|
||||
request_obj,
|
||||
reasoning_parser=MagicMock(),
|
||||
reasoning_ended=True,
|
||||
previous_text="prior_tool_text",
|
||||
previous_token_ids=[10, 20],
|
||||
current_text="prior_tool_texthello",
|
||||
current_token_ids=[10, 20, 1, 2, 3],
|
||||
)
|
||||
|
||||
_, call_kwargs = mock_extract.call_args
|
||||
assert call_kwargs["previous_text"] == "prior_tool_text"
|
||||
assert call_kwargs["previous_token_ids"] == [10, 20]
|
||||
|
||||
assert result == MistralStreamingResult(
|
||||
delta_message=content_delta,
|
||||
reasoning_ended=True,
|
||||
tools_called=False,
|
||||
current_text="prior_tool_texthello",
|
||||
current_token_ids=[10, 20, 1, 2, 3],
|
||||
)
|
||||
|
||||
def test_pre_v15_ignores_prompt_reasoning_end(
|
||||
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
|
||||
) -> None:
|
||||
mock_tokenizer = MagicMock(spec=MistralTokenizer)
|
||||
mock_tokenizer.version = 13
|
||||
parser.model_tokenizer = mock_tokenizer
|
||||
|
||||
mock_rp = MagicMock(spec=MistralReasoningParser)
|
||||
mock_rp.start_token_id = 999
|
||||
mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(
|
||||
reasoning="thinking..."
|
||||
)
|
||||
mock_rp.is_reasoning_end_streaming.return_value = False
|
||||
|
||||
result = self._call(
|
||||
parser,
|
||||
request_obj,
|
||||
reasoning_parser=mock_rp,
|
||||
reasoning_ended=False,
|
||||
prompt_is_reasoning_end=True,
|
||||
current_token_ids=[999, 1, 2],
|
||||
)
|
||||
|
||||
mock_rp.extract_reasoning_streaming.assert_called_once()
|
||||
assert result == MistralStreamingResult(
|
||||
delta_message=DeltaMessage(reasoning="thinking..."),
|
||||
reasoning_ended=False,
|
||||
tools_called=False,
|
||||
current_text="hello",
|
||||
current_token_ids=[999, 1, 2],
|
||||
)
|
||||
|
||||
def test_non_pre_v15_prompt_reasoning_end(
|
||||
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
|
||||
) -> None:
|
||||
mock_tokenizer = MagicMock(spec=MistralTokenizer)
|
||||
mock_tokenizer.version = 15
|
||||
parser.model_tokenizer = mock_tokenizer
|
||||
|
||||
mock_rp = MagicMock(spec=MistralReasoningParser)
|
||||
mock_rp.start_token_id = 999
|
||||
|
||||
content_delta = DeltaMessage(content="after reasoning")
|
||||
with patch.object(
|
||||
parser, "extract_tool_calls_streaming", return_value=content_delta
|
||||
):
|
||||
result = self._call(
|
||||
parser,
|
||||
request_obj,
|
||||
reasoning_parser=mock_rp,
|
||||
reasoning_ended=False,
|
||||
prompt_is_reasoning_end=True,
|
||||
current_token_ids=[999, 1, 2],
|
||||
output_token_ids=[10, 20, 30],
|
||||
)
|
||||
|
||||
mock_rp.extract_reasoning_streaming.assert_not_called()
|
||||
assert result == MistralStreamingResult(
|
||||
delta_message=content_delta,
|
||||
reasoning_ended=True,
|
||||
tools_called=False,
|
||||
current_text="hello",
|
||||
current_token_ids=[10, 20, 30],
|
||||
)
|
||||
|
||||
def test_reasoning_end_transition_with_content(
|
||||
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
|
||||
) -> None:
|
||||
"""When reasoning ends and the delta has content, that content is
|
||||
cleared from delta_message and used as current_text for tool parsing."""
|
||||
mock_rp = MagicMock()
|
||||
mock_rp.start_token_id = 999
|
||||
mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(
|
||||
reasoning="think", content="leftover"
|
||||
)
|
||||
mock_rp.is_reasoning_end_streaming.return_value = True
|
||||
mock_rp.extract_content_ids.return_value = [50, 51]
|
||||
|
||||
content_delta = DeltaMessage(content="leftover")
|
||||
with patch.object(
|
||||
parser, "extract_tool_calls_streaming", return_value=content_delta
|
||||
) as mock_extract:
|
||||
result = self._call(
|
||||
parser,
|
||||
request_obj,
|
||||
reasoning_parser=mock_rp,
|
||||
reasoning_ended=False,
|
||||
current_token_ids=[999, 1, 2],
|
||||
output_token_ids=[10, 20, 30],
|
||||
)
|
||||
|
||||
mock_rp.extract_content_ids.assert_called_once_with([10, 20, 30])
|
||||
_, call_kwargs = mock_extract.call_args
|
||||
assert call_kwargs["previous_text"] == ""
|
||||
assert call_kwargs["previous_token_ids"] == []
|
||||
assert call_kwargs["delta_text"] == "leftover"
|
||||
assert call_kwargs["current_token_ids"] == [50, 51]
|
||||
|
||||
assert result == MistralStreamingResult(
|
||||
delta_message=content_delta,
|
||||
reasoning_ended=True,
|
||||
tools_called=False,
|
||||
current_text="leftover",
|
||||
current_token_ids=[50, 51],
|
||||
)
|
||||
|
||||
def test_reasoning_end_transition_without_content(
|
||||
self, parser: MistralToolParser, request_obj: ChatCompletionRequest
|
||||
) -> None:
|
||||
"""When reasoning ends but the delta has no content, current_text
|
||||
is set to empty string."""
|
||||
mock_rp = MagicMock()
|
||||
mock_rp.start_token_id = 999
|
||||
mock_rp.extract_reasoning_streaming.return_value = DeltaMessage(
|
||||
reasoning="think"
|
||||
)
|
||||
mock_rp.is_reasoning_end_streaming.return_value = True
|
||||
mock_rp.extract_content_ids.return_value = [50, 51]
|
||||
|
||||
empty_delta = DeltaMessage(content="")
|
||||
with patch.object(
|
||||
parser, "extract_tool_calls_streaming", return_value=empty_delta
|
||||
) as mock_extract:
|
||||
result = self._call(
|
||||
parser,
|
||||
request_obj,
|
||||
reasoning_parser=mock_rp,
|
||||
reasoning_ended=False,
|
||||
current_token_ids=[999, 1, 2],
|
||||
output_token_ids=[10, 20, 30],
|
||||
)
|
||||
|
||||
_, call_kwargs = mock_extract.call_args
|
||||
assert call_kwargs["delta_text"] == ""
|
||||
assert call_kwargs["current_token_ids"] == [50, 51]
|
||||
|
||||
assert result == MistralStreamingResult(
|
||||
delta_message=empty_delta,
|
||||
reasoning_ended=True,
|
||||
tools_called=False,
|
||||
current_text="",
|
||||
current_token_ids=[50, 51],
|
||||
)
|
||||
|
||||
@@ -72,7 +72,7 @@ from vllm.renderers import ChatParams
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.collection_utils import as_list
|
||||
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
|
||||
from vllm.utils.mistral import is_mistral_tool_parser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
@@ -425,8 +425,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
harmony_tools_streamed = [False] * num_choices
|
||||
tools_streamed = [False] * num_choices
|
||||
|
||||
is_mistral_grammar_path = request._grammar_from_tool_parser
|
||||
|
||||
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
|
||||
tool_choice_function_name = request.tool_choice.function.name
|
||||
else:
|
||||
@@ -450,15 +448,12 @@ class OpenAIServingChat(OpenAIServing):
|
||||
# Only one of these will be used, thus previous_texts and
|
||||
# all_previous_token_ids will not be used twice in the same iteration.
|
||||
if (
|
||||
is_mistral_grammar_path
|
||||
or tool_choice_auto
|
||||
tool_choice_auto
|
||||
or tool_choice_function_name
|
||||
or request.tool_choice == "required"
|
||||
or reasoning_parser
|
||||
):
|
||||
all_previous_token_ids = [[] for _ in range(num_choices)]
|
||||
reasoning_end_arr = [False] * num_choices
|
||||
prompt_is_reasoning_end_arr: list[bool | None] = [None] * num_choices
|
||||
else:
|
||||
all_previous_token_ids = None
|
||||
|
||||
@@ -592,18 +587,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
for output in res.outputs:
|
||||
i = output.index
|
||||
parser = parsers[i]
|
||||
tool_parser = parser.tool_parser if parser is not None else None
|
||||
|
||||
if (
|
||||
reasoning_parser
|
||||
and res.prompt_token_ids
|
||||
and prompt_is_reasoning_end_arr[i] is None
|
||||
):
|
||||
# only check once per choice, because prompt_token_ids
|
||||
# are the same for all deltas in that choice
|
||||
prompt_is_reasoning_end_arr[i] = (
|
||||
reasoning_parser.is_reasoning_end(res.prompt_token_ids)
|
||||
)
|
||||
if finish_reason_sent[i]:
|
||||
continue
|
||||
|
||||
@@ -658,8 +641,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
|
||||
# just update previous_texts and previous_token_ids
|
||||
if (
|
||||
is_mistral_grammar_path
|
||||
or tool_choice_auto
|
||||
tool_choice_auto
|
||||
or tool_choice_function_name
|
||||
or request.tool_choice == "required"
|
||||
or reasoning_parser
|
||||
@@ -687,35 +669,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
)
|
||||
)
|
||||
harmony_tools_streamed[i] |= tools_streamed_flag
|
||||
# Mistral grammar path: combined reasoning + tool streaming
|
||||
elif is_mistral_grammar_path:
|
||||
from vllm.tool_parsers.mistral_tool_parser import (
|
||||
MistralToolParser,
|
||||
)
|
||||
|
||||
assert tool_parser is not None
|
||||
assert isinstance(tool_parser, MistralToolParser)
|
||||
assert reasoning_end_arr is not None
|
||||
output_token_ids = as_list(output.token_ids)
|
||||
result = tool_parser.extract_maybe_reasoning_and_tool_streaming(
|
||||
reasoning_parser=reasoning_parser,
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=previous_token_ids,
|
||||
current_token_ids=current_token_ids,
|
||||
output_token_ids=output_token_ids,
|
||||
reasoning_ended=reasoning_end_arr[i],
|
||||
prompt_is_reasoning_end=(prompt_is_reasoning_end_arr[i]),
|
||||
request=request,
|
||||
)
|
||||
delta_message = result.delta_message
|
||||
reasoning_end_arr[i] = result.reasoning_ended
|
||||
current_text = result.current_text
|
||||
current_token_ids = result.current_token_ids
|
||||
if result.tools_called:
|
||||
tools_streamed[i] = True
|
||||
|
||||
elif parser is not None:
|
||||
delta_message = parser.parse_delta(
|
||||
delta_text=delta_text,
|
||||
@@ -732,8 +685,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
|
||||
# update the previous values for the next iteration
|
||||
if (
|
||||
is_mistral_grammar_path
|
||||
or tool_choice_auto
|
||||
tool_choice_auto
|
||||
or tool_choice_function_name
|
||||
or request.tool_choice == "required"
|
||||
or reasoning_parser
|
||||
@@ -1067,32 +1019,8 @@ class OpenAIServingChat(OpenAIServing):
|
||||
tool_calls = []
|
||||
|
||||
auto_tools_called = False
|
||||
if is_mistral_tokenizer(tokenizer):
|
||||
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
|
||||
|
||||
tool_call_class: type[ToolCall] = MistralToolCall
|
||||
else:
|
||||
tool_call_class = ToolCall
|
||||
|
||||
use_mistral_tool_parser = request._grammar_from_tool_parser
|
||||
if use_mistral_tool_parser:
|
||||
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
|
||||
|
||||
tool_call_items = MistralToolParser.build_non_streaming_tool_calls(
|
||||
tool_calls
|
||||
)
|
||||
if tool_call_items:
|
||||
auto_tools_called = (
|
||||
request.tool_choice is None or request.tool_choice == "auto"
|
||||
)
|
||||
message = ChatMessage(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tool_call_items,
|
||||
)
|
||||
|
||||
elif (not self.enable_auto_tools or not self.tool_parser) and (
|
||||
if (not self.enable_auto_tools or not self.tool_parser) and (
|
||||
not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
|
||||
and request.tool_choice != "required"
|
||||
):
|
||||
@@ -1102,70 +1030,42 @@ class OpenAIServingChat(OpenAIServing):
|
||||
request.tool_choice
|
||||
and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
|
||||
):
|
||||
tool_call_class_items = []
|
||||
tool_call_items = []
|
||||
tool_calls = tool_calls or []
|
||||
for idx, tc in enumerate(tool_calls):
|
||||
# Use native ID if available (e.g., Kimi K2),
|
||||
# otherwise generate ID with correct id_type
|
||||
if tc.id:
|
||||
tool_call_class_items.append(
|
||||
tool_call_class(id=tc.id, function=tc)
|
||||
for tc in tool_calls:
|
||||
if not tc.id:
|
||||
tc.id = make_tool_call_id(
|
||||
id_type=self.tool_call_id_type,
|
||||
func_name=tc.name,
|
||||
idx=history_tool_call_cnt,
|
||||
)
|
||||
else:
|
||||
# Generate ID using the correct format (kimi_k2 or random),
|
||||
# but leave it to the class if it's Mistral to preserve
|
||||
# 9-char IDs
|
||||
if is_mistral_tokenizer(tokenizer):
|
||||
tool_call_class_items.append(tool_call_class(function=tc))
|
||||
else:
|
||||
generated_id = make_tool_call_id(
|
||||
id_type=self.tool_call_id_type,
|
||||
func_name=tc.name,
|
||||
idx=history_tool_call_cnt,
|
||||
)
|
||||
tool_call_class_items.append(
|
||||
tool_call_class(id=generated_id, function=tc)
|
||||
)
|
||||
tool_call_items.append(ToolCall(id=tc.id, function=tc))
|
||||
history_tool_call_cnt += 1
|
||||
message = ChatMessage(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content="",
|
||||
tool_calls=tool_call_class_items,
|
||||
content=content or "",
|
||||
tool_calls=tool_call_items,
|
||||
)
|
||||
|
||||
elif request.tool_choice and request.tool_choice == "required":
|
||||
tool_call_class_items = []
|
||||
tool_call_items = []
|
||||
tool_calls = tool_calls or []
|
||||
for idx, tool_call in enumerate(tool_calls):
|
||||
# Use native ID if available,
|
||||
# otherwise generate ID with correct id_type
|
||||
if tool_call.id:
|
||||
tool_call_class_items.append(
|
||||
tool_call_class(id=tool_call.id, function=tool_call)
|
||||
for tool_call in tool_calls:
|
||||
if not tool_call.id:
|
||||
tool_call.id = make_tool_call_id(
|
||||
id_type=self.tool_call_id_type,
|
||||
func_name=tool_call.name,
|
||||
idx=history_tool_call_cnt,
|
||||
)
|
||||
else:
|
||||
# Generate ID using the correct format (kimi_k2 or random),
|
||||
# but leave it to the class if it's Mistral to preserve
|
||||
# 9-char IDs
|
||||
if is_mistral_tokenizer(tokenizer):
|
||||
tool_call_class_items.append(
|
||||
tool_call_class(function=tool_call)
|
||||
)
|
||||
else:
|
||||
generated_id = make_tool_call_id(
|
||||
id_type=self.tool_call_id_type,
|
||||
func_name=tool_call.name,
|
||||
idx=history_tool_call_cnt,
|
||||
)
|
||||
tool_call_class_items.append(
|
||||
tool_call_class(id=generated_id, function=tool_call)
|
||||
)
|
||||
tool_call_items.append(
|
||||
ToolCall(id=tool_call.id, function=tool_call)
|
||||
)
|
||||
history_tool_call_cnt += 1
|
||||
message = ChatMessage(
|
||||
role=role,
|
||||
content="",
|
||||
tool_calls=tool_call_class_items,
|
||||
content=content or "",
|
||||
tool_calls=tool_call_items,
|
||||
reasoning=reasoning,
|
||||
)
|
||||
|
||||
@@ -1181,34 +1081,17 @@ class OpenAIServingChat(OpenAIServing):
|
||||
and self.enable_auto_tools
|
||||
and self.tool_parser
|
||||
):
|
||||
# In the OpenAI API the finish_reason is "tools_called"
|
||||
# if the tool choice is auto and the model produced a tool
|
||||
# call. The same is not true for named function calls
|
||||
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
|
||||
if tool_calls:
|
||||
tool_call_items = []
|
||||
for idx, tc in enumerate(tool_calls):
|
||||
# Use native ID if available (e.g., Kimi K2),
|
||||
# otherwise generate ID with correct id_type
|
||||
if tc.id:
|
||||
tool_call_items.append(
|
||||
tool_call_class(id=tc.id, function=tc)
|
||||
for tc in tool_calls:
|
||||
if not tc.id:
|
||||
tc.id = make_tool_call_id(
|
||||
id_type=self.tool_call_id_type,
|
||||
func_name=tc.name,
|
||||
idx=history_tool_call_cnt,
|
||||
)
|
||||
else:
|
||||
# Generate ID using the correct format (kimi_k2 or random),
|
||||
# but leave it to the class if it's Mistral to preserve
|
||||
# 9-char IDs
|
||||
if is_mistral_tokenizer(tokenizer):
|
||||
tool_call_items.append(tool_call_class(function=tc))
|
||||
else:
|
||||
generated_id = make_tool_call_id(
|
||||
id_type=self.tool_call_id_type,
|
||||
func_name=tc.name,
|
||||
idx=history_tool_call_cnt,
|
||||
)
|
||||
tool_call_items.append(
|
||||
tool_call_class(id=generated_id, function=tc)
|
||||
)
|
||||
tool_call_items.append(ToolCall(id=tc.id, function=tc))
|
||||
history_tool_call_cnt += 1
|
||||
message = ChatMessage(
|
||||
role=role,
|
||||
@@ -1218,18 +1101,10 @@ class OpenAIServingChat(OpenAIServing):
|
||||
)
|
||||
|
||||
else:
|
||||
# FOR NOW make it a chat message; we will have to detect
|
||||
# the type to make it later.
|
||||
ret_content = content
|
||||
|
||||
# try to use content return from tool parser first,
|
||||
# tool parser may do some modify for the content.
|
||||
if content and len(content) > 0:
|
||||
ret_content = content
|
||||
message = ChatMessage(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=ret_content,
|
||||
content=content,
|
||||
)
|
||||
|
||||
# undetermined case that is still important to handle
|
||||
|
||||
@@ -43,7 +43,6 @@ from vllm.tool_parsers.streaming import (
|
||||
extract_required_tool_call_streaming,
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.mistral import is_mistral_tool_parser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -546,14 +545,6 @@ class DelegatingParser(Parser):
|
||||
if tool_parser is None:
|
||||
return [], content
|
||||
|
||||
# When the Mistral grammar factory injected structured outputs,
|
||||
# let the parser handle the output.
|
||||
use_mistral_tool_parser = (
|
||||
is_mistral_tool_parser(type(tool_parser))
|
||||
and isinstance(request, ChatCompletionRequest)
|
||||
and request._grammar_from_tool_parser
|
||||
)
|
||||
|
||||
supports_required_and_named = tool_parser.supports_required_and_named
|
||||
is_named_tool_choice = request.tool_choice and isinstance(
|
||||
request.tool_choice,
|
||||
@@ -570,11 +561,7 @@ class DelegatingParser(Parser):
|
||||
)
|
||||
|
||||
tool_calls = list[FunctionCall]()
|
||||
if (
|
||||
is_named_tool_choice
|
||||
and supports_required_and_named
|
||||
and not use_mistral_tool_parser
|
||||
):
|
||||
if is_named_tool_choice and supports_required_and_named:
|
||||
if content is None:
|
||||
return [], None
|
||||
tool_calls.append(
|
||||
@@ -584,11 +571,7 @@ class DelegatingParser(Parser):
|
||||
)
|
||||
)
|
||||
content = None
|
||||
elif (
|
||||
is_required_tool_choice
|
||||
and supports_required_and_named
|
||||
and not use_mistral_tool_parser
|
||||
):
|
||||
elif is_required_tool_choice and supports_required_and_named:
|
||||
# "required" with standard JSON-based parsing
|
||||
parsed_calls = []
|
||||
with contextlib.suppress(ValidationError):
|
||||
@@ -604,7 +587,7 @@ class DelegatingParser(Parser):
|
||||
)
|
||||
)
|
||||
content = None
|
||||
elif is_auto_tool_choice or use_mistral_tool_parser:
|
||||
elif is_auto_tool_choice:
|
||||
# Automatic Tool Call Parsing (also used as fallback for
|
||||
# required/named when supports_required_and_named=False)
|
||||
tool_call_info = tool_parser.extract_tool_calls(
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
|
||||
class MistralParser(DelegatingParser):
|
||||
def __init__(self, tokenizer, tools=None, *args, **kwargs):
|
||||
super().__init__(tokenizer, tools, *args, **kwargs)
|
||||
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
|
||||
|
||||
if not isinstance(self._tool_parser, MistralToolParser):
|
||||
raise ValueError(
|
||||
"MistralParser requires --tool-call-parser mistral, "
|
||||
f"got {self._tool_parser.__class__.__name__}."
|
||||
)
|
||||
|
||||
def _maybe_force_auto_tool_parsing(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> None:
|
||||
# When the Mistral grammar factory injected structured outputs,
|
||||
# the model emits v11+ format ([TOOL_CALLS]name{args}) that the
|
||||
# named/required parsers can't handle. Disable them so all
|
||||
# tool_choice modes fall back to auto tool parsing via
|
||||
# extract_tool_calls.
|
||||
if getattr(request, "_grammar_from_tool_parser", False):
|
||||
assert self._tool_parser is not None
|
||||
self._tool_parser.supports_required_and_named = False
|
||||
|
||||
def parse(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
enable_auto_tools: bool = False,
|
||||
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
||||
self._maybe_force_auto_tool_parsing(request)
|
||||
reasoning, content, tool_calls = super().parse(
|
||||
model_output, request, enable_auto_tools
|
||||
)
|
||||
if tool_calls:
|
||||
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
|
||||
|
||||
# Named/required tool_choice builds FunctionCalls without
|
||||
# ID, backfill with Mistral-format IDs.
|
||||
for tc in tool_calls:
|
||||
if not tc.id:
|
||||
tc.id = MistralToolCall.generate_random_id()
|
||||
return reasoning, content, tool_calls
|
||||
|
||||
def parse_delta(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: list[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
self._maybe_force_auto_tool_parsing(request)
|
||||
return super().parse_delta(
|
||||
delta_text,
|
||||
delta_token_ids,
|
||||
request,
|
||||
prompt_token_ids,
|
||||
finished=finished,
|
||||
)
|
||||
@@ -106,6 +106,15 @@ class ParserManager:
|
||||
if reasoning_parser_cls is None and tool_parser_cls is None:
|
||||
return None
|
||||
|
||||
from vllm.utils.mistral import is_mistral_tool_parser
|
||||
|
||||
if is_mistral_tool_parser(tool_parser_cls):
|
||||
from vllm.parser.mistral import MistralParser
|
||||
|
||||
MistralParser.reasoning_parser_cls = reasoning_parser_cls
|
||||
MistralParser.tool_parser_cls = tool_parser_cls
|
||||
return MistralParser
|
||||
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
|
||||
r_cls = reasoning_parser_cls
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Iterable, Sequence
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -76,6 +76,15 @@ class MistralReasoningParser(BaseThinkingReasoningParser):
|
||||
has_eot_token = True
|
||||
return False
|
||||
|
||||
def is_reasoning_end_streaming(
|
||||
self, input_ids: Sequence[int], delta_ids: Iterable[int]
|
||||
) -> bool:
|
||||
if self.end_token_id in delta_ids:
|
||||
return True
|
||||
# Grammar's think? is optional — if [THINK] was never generated,
|
||||
# reasoning was skipped entirely.
|
||||
return self.start_token_id not in input_ids
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
"""
|
||||
Extract the content
|
||||
|
||||
@@ -5,11 +5,10 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from random import choices
|
||||
from string import ascii_letters, digits
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import Any
|
||||
|
||||
import ijson
|
||||
import regex as re
|
||||
@@ -40,19 +39,14 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning.mistral_reasoning_parser import MistralReasoningParser
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.tool_parsers.abstract_tool_parser import (
|
||||
Tool,
|
||||
ToolParser,
|
||||
)
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.reasoning import ReasoningParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
ALPHANUMERIC = ascii_letters + digits
|
||||
@@ -99,19 +93,6 @@ def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool:
|
||||
return "[ARGS]" not in vocab
|
||||
|
||||
|
||||
@dataclass
|
||||
class MistralStreamingResult:
|
||||
r"""Encapsulates the mutable state returned from
|
||||
`MistralToolParser.extract_maybe_reasoning_and_tool_streaming`.
|
||||
"""
|
||||
|
||||
delta_message: DeltaMessage | None
|
||||
reasoning_ended: bool
|
||||
tools_called: bool
|
||||
current_text: str
|
||||
current_token_ids: list[int]
|
||||
|
||||
|
||||
class MistralToolParser(ToolParser):
|
||||
r"""Tool call parser for Mistral models, intended for use with either:
|
||||
|
||||
@@ -281,148 +262,6 @@ class MistralToolParser(ToolParser):
|
||||
request._grammar_from_tool_parser = True
|
||||
return request
|
||||
|
||||
def extract_maybe_reasoning_and_tool_streaming(
|
||||
self,
|
||||
*,
|
||||
reasoning_parser: ReasoningParser | None,
|
||||
previous_text: str,
|
||||
current_text: str,
|
||||
delta_text: str,
|
||||
previous_token_ids: list[int],
|
||||
current_token_ids: list[int],
|
||||
output_token_ids: Sequence[int],
|
||||
reasoning_ended: bool,
|
||||
prompt_is_reasoning_end: bool | None,
|
||||
request: ChatCompletionRequest,
|
||||
) -> MistralStreamingResult:
|
||||
r"""Streaming extraction with reasoning followed by tool-call parsing.
|
||||
|
||||
This method encapsulates the combined reasoning extraction and
|
||||
tool-call streaming logic so that the serving layer only needs a
|
||||
thin routing branch.
|
||||
|
||||
The flow is:
|
||||
|
||||
1. If a *reasoning_parser* is present and reasoning has **not** ended,
|
||||
extract reasoning tokens. Pre-v15 models may have pre-filled
|
||||
`[THINK]...[/THINK]` in system prompts, so we skip the
|
||||
prompt-level reasoning-end check for those.
|
||||
2. Once reasoning ends (or if there is no reasoning parser), delegate
|
||||
to `extract_tool_calls_streaming` and track whether tools were
|
||||
called.
|
||||
|
||||
Args:
|
||||
reasoning_parser: Optional reasoning parser instance.
|
||||
previous_text: Accumulated text from prior chunks.
|
||||
current_text: Full accumulated text including current chunk.
|
||||
delta_text: New text in this chunk.
|
||||
previous_token_ids: Token ids from prior chunks.
|
||||
current_token_ids: Full token ids including current chunk.
|
||||
output_token_ids: Raw output token ids from the engine.
|
||||
reasoning_ended: Whether reasoning has already ended.
|
||||
prompt_is_reasoning_end: Whether the prompt itself ends reasoning.
|
||||
request: The originating chat completion request.
|
||||
"""
|
||||
delta_message: DeltaMessage | None = None
|
||||
tools_called = False
|
||||
reasoning_ended_at_entry = reasoning_ended
|
||||
|
||||
# For MistralReasoningParser, only enter the reasoning block when
|
||||
# the model has actually emitted a [THINK] token. Other reasoning
|
||||
# parsers always expect thinking to be present.
|
||||
expect_thinking = (
|
||||
not isinstance(reasoning_parser, MistralReasoningParser)
|
||||
or reasoning_parser.start_token_id in current_token_ids
|
||||
)
|
||||
if reasoning_parser is not None and not reasoning_ended and expect_thinking:
|
||||
# Pre-v15 models may have pre-filled [THINK]...[/THINK] in
|
||||
# system prompts, so skip the prompt-level reasoning-end
|
||||
# check and wait for the output's own end-of-think.
|
||||
is_pre_v15 = (
|
||||
isinstance(self.model_tokenizer, MistralTokenizer)
|
||||
and self.model_tokenizer.version < 15
|
||||
)
|
||||
|
||||
if not is_pre_v15 and prompt_is_reasoning_end:
|
||||
reasoning_ended = True
|
||||
current_token_ids = list(output_token_ids)
|
||||
else:
|
||||
delta_message = reasoning_parser.extract_reasoning_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
output_token_ids,
|
||||
)
|
||||
if reasoning_parser.is_reasoning_end_streaming(
|
||||
current_token_ids, output_token_ids
|
||||
):
|
||||
reasoning_ended = True
|
||||
current_token_ids = reasoning_parser.extract_content_ids(
|
||||
list(output_token_ids)
|
||||
)
|
||||
if delta_message and delta_message.content:
|
||||
current_text = delta_message.content
|
||||
delta_message.content = None
|
||||
else:
|
||||
current_text = ""
|
||||
|
||||
if not reasoning_ended:
|
||||
return MistralStreamingResult(
|
||||
delta_message=delta_message,
|
||||
reasoning_ended=False,
|
||||
tools_called=False,
|
||||
current_text=current_text,
|
||||
current_token_ids=current_token_ids,
|
||||
)
|
||||
|
||||
delta_token_ids = list(output_token_ids)
|
||||
|
||||
# On the iteration where reasoning just ended, reset the text/token
|
||||
# state so the tool parser sees a clean history instead of the
|
||||
# accumulated reasoning text.
|
||||
if not reasoning_ended_at_entry and reasoning_ended:
|
||||
previous_text = ""
|
||||
previous_token_ids = []
|
||||
delta_text = current_text
|
||||
delta_token_ids = current_token_ids
|
||||
|
||||
delta_message = self.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=previous_token_ids,
|
||||
current_token_ids=current_token_ids,
|
||||
delta_token_ids=delta_token_ids,
|
||||
request=request,
|
||||
)
|
||||
if delta_message and delta_message.tool_calls:
|
||||
tools_called = True
|
||||
|
||||
return MistralStreamingResult(
|
||||
delta_message=delta_message,
|
||||
reasoning_ended=reasoning_ended,
|
||||
tools_called=tools_called,
|
||||
current_text=current_text,
|
||||
current_token_ids=current_token_ids,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def build_non_streaming_tool_calls(
|
||||
tool_calls: list[FunctionCall] | None,
|
||||
) -> list[ToolCall]:
|
||||
r"""Build `MistralToolCall` items for non-streaming responses."""
|
||||
if not tool_calls:
|
||||
return []
|
||||
|
||||
return [
|
||||
MistralToolCall(id=tc.id, function=tc)
|
||||
if tc.id
|
||||
else MistralToolCall(function=tc)
|
||||
for tc in tool_calls
|
||||
]
|
||||
|
||||
def extract_tool_calls(
|
||||
self,
|
||||
model_output: str,
|
||||
@@ -536,7 +375,7 @@ class MistralToolParser(ToolParser):
|
||||
return ExtractedToolCallInformation(
|
||||
tools_called=True,
|
||||
tool_calls=mistral_tool_calls,
|
||||
content=content if len(content) > 0 else None,
|
||||
content=content if content.strip() else None,
|
||||
)
|
||||
|
||||
def extract_tool_calls_streaming(
|
||||
|
||||
Reference in New Issue
Block a user