diff --git a/tests/tool_parsers/test_mistral_tool_parser.py b/tests/tool_parsers/test_mistral_tool_parser.py index c9582159ab..03a10ef099 100644 --- a/tests/tool_parsers/test_mistral_tool_parser.py +++ b/tests/tool_parsers/test_mistral_tool_parser.py @@ -3,7 +3,6 @@ import json from collections.abc import Generator -from typing import Any from unittest.mock import MagicMock, patch import partial_json_parser @@ -29,22 +28,17 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ) from vllm.entrypoints.openai.engine.protocol import ( - DeltaFunctionCall, DeltaMessage, DeltaToolCall, ExtractedToolCallInformation, StructuralTagResponseFormat, ) -from vllm.entrypoints.openai.engine.protocol import FunctionCall as VllmFunctionCall -from vllm.reasoning.mistral_reasoning_parser import MistralReasoningParser from vllm.sampling_params import StructuredOutputsParams from vllm.tokenizers import TokenizerLike, get_tokenizer from vllm.tokenizers.detokenizer_utils import detokenize_incrementally from vllm.tokenizers.mistral import MistralTokenizer from vllm.tool_parsers.mistral_tool_parser import ( _DEFAULT_JSON_SCHEMA, - MistralStreamingResult, - MistralToolCall, MistralToolParser, ) @@ -1578,382 +1572,3 @@ def test_grammar_from_tool_parser_set_by_adjust_request( request = _make_request() result = mistral_tool_parser.adjust_request(request) assert result._grammar_from_tool_parser is True - - -@pytest.mark.parametrize( - "tool_calls, expected_len", - [ - (None, 0), - ([], 0), - ([VllmFunctionCall(id="abc123xyz", name="f", arguments="{}")], 1), - ([VllmFunctionCall(name="f", arguments="{}")], 1), - ( - [ - VllmFunctionCall(id="fixed1234", name="a", arguments='{"x": 1}'), - VllmFunctionCall(name="b", arguments='{"y": 2}'), - ], - 2, - ), - ], - ids=["none", "empty", "with_id", "without_id", "mixed"], -) -def test_build_non_streaming_tool_calls( - tool_calls: list[VllmFunctionCall] | None, - expected_len: int, -) -> None: - result = MistralToolParser.build_non_streaming_tool_calls(tool_calls) - assert len(result) == expected_len - - if tool_calls is None: - return - - for i, tc in enumerate(result): - assert isinstance(tc, MistralToolCall) - assert tc.type == "function" - - input_tc = tool_calls[i] - if input_tc.id: - assert tc.id == input_tc.id - else: - assert len(tc.id) == 9 - assert tc.id.isalnum() - - assert tc.function.name == input_tc.name - assert tc.function.arguments == input_tc.arguments - - -class TestExtractMaybeReasoningAndToolStreaming: - r"""Tests for `MistralToolParser.extract_maybe_reasoning_and_tool_streaming`.""" - - @pytest.fixture - def parser(self) -> MistralToolParser: - mock_tokenizer = MagicMock() - mock_tokenizer.get_vocab.return_value = {"[TOOL_CALLS]": 1} - return MistralToolParser(mock_tokenizer) - - @pytest.fixture - def request_obj(self) -> ChatCompletionRequest: - return _make_request() - - @staticmethod - def _call( - parser: MistralToolParser, - request: ChatCompletionRequest, - *, - reasoning_parser: Any = None, - previous_text: str = "", - current_text: str = "hello", - delta_text: str = "hello", - previous_token_ids: list[int] | None = None, - current_token_ids: list[int] | None = None, - output_token_ids: list[int] | None = None, - reasoning_ended: bool = False, - prompt_is_reasoning_end: bool | None = None, - ) -> MistralStreamingResult: - return parser.extract_maybe_reasoning_and_tool_streaming( - reasoning_parser=reasoning_parser, - previous_text=previous_text, - current_text=current_text, - delta_text=delta_text, - previous_token_ids=previous_token_ids or [], - current_token_ids=current_token_ids or [1, 2, 3], - output_token_ids=output_token_ids or [1, 2, 3], - reasoning_ended=reasoning_ended, - prompt_is_reasoning_end=prompt_is_reasoning_end, - request=request, - ) - - def test_no_reasoning_tools_called( - self, parser: MistralToolParser, request_obj: ChatCompletionRequest - ) -> None: - tool_delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=0, - function=DeltaFunctionCall(name="f", arguments="{}"), - ) - ] - ) - with patch.object( - parser, "extract_tool_calls_streaming", return_value=tool_delta - ): - result = self._call(parser, request_obj, reasoning_parser=None) - - assert result == MistralStreamingResult( - delta_message=tool_delta, - reasoning_ended=False, - tools_called=True, - current_text="hello", - current_token_ids=[1, 2, 3], - ) - - def test_no_reasoning_no_tools( - self, parser: MistralToolParser, request_obj: ChatCompletionRequest - ) -> None: - content_delta = DeltaMessage(content="hello") - with patch.object( - parser, "extract_tool_calls_streaming", return_value=content_delta - ): - result = self._call(parser, request_obj, reasoning_parser=None) - - assert result == MistralStreamingResult( - delta_message=content_delta, - reasoning_ended=False, - tools_called=False, - current_text="hello", - current_token_ids=[1, 2, 3], - ) - - def test_mistral_reasoning_parser_no_think_token( - self, parser: MistralToolParser, request_obj: ChatCompletionRequest - ) -> None: - mock_rp = MagicMock(spec=MistralReasoningParser) - mock_rp.start_token_id = 999 - content_delta = DeltaMessage(content="direct") - with patch.object( - parser, "extract_tool_calls_streaming", return_value=content_delta - ): - result = self._call( - parser, - request_obj, - reasoning_parser=mock_rp, - reasoning_ended=False, - current_token_ids=[1, 2, 3], - ) - - mock_rp.extract_reasoning_streaming.assert_not_called() - assert result == MistralStreamingResult( - delta_message=content_delta, - reasoning_ended=False, - tools_called=False, - current_text="hello", - current_token_ids=[1, 2, 3], - ) - - def test_mistral_reasoning_parser_with_think_token( - self, parser: MistralToolParser, request_obj: ChatCompletionRequest - ) -> None: - mock_rp = MagicMock(spec=MistralReasoningParser) - mock_rp.start_token_id = 999 - mock_rp.extract_reasoning_streaming.return_value = DeltaMessage( - reasoning="thinking..." - ) - mock_rp.is_reasoning_end_streaming.return_value = False - - result = self._call( - parser, - request_obj, - reasoning_parser=mock_rp, - reasoning_ended=False, - current_token_ids=[1, 999, 3], - ) - - mock_rp.extract_reasoning_streaming.assert_called_once() - assert result == MistralStreamingResult( - delta_message=DeltaMessage(reasoning="thinking..."), - reasoning_ended=False, - tools_called=False, - current_text="hello", - current_token_ids=[1, 999, 3], - ) - - def test_non_mistral_reasoning_parser_always_expects_thinking( - self, parser: MistralToolParser, request_obj: ChatCompletionRequest - ) -> None: - mock_rp = MagicMock() - mock_rp.start_token_id = 999 - mock_rp.extract_reasoning_streaming.return_value = DeltaMessage( - reasoning="thinking..." - ) - mock_rp.is_reasoning_end_streaming.return_value = False - - result = self._call( - parser, - request_obj, - reasoning_parser=mock_rp, - reasoning_ended=False, - current_token_ids=[1, 2, 3], - ) - - mock_rp.extract_reasoning_streaming.assert_called_once() - assert result == MistralStreamingResult( - delta_message=DeltaMessage(reasoning="thinking..."), - reasoning_ended=False, - tools_called=False, - current_text="hello", - current_token_ids=[1, 2, 3], - ) - - def test_reasoning_already_ended_no_reset( - self, parser: MistralToolParser, request_obj: ChatCompletionRequest - ) -> None: - content_delta = DeltaMessage(content="content") - with patch.object( - parser, "extract_tool_calls_streaming", return_value=content_delta - ) as mock_extract: - result = self._call( - parser, - request_obj, - reasoning_parser=MagicMock(), - reasoning_ended=True, - previous_text="prior_tool_text", - previous_token_ids=[10, 20], - current_text="prior_tool_texthello", - current_token_ids=[10, 20, 1, 2, 3], - ) - - _, call_kwargs = mock_extract.call_args - assert call_kwargs["previous_text"] == "prior_tool_text" - assert call_kwargs["previous_token_ids"] == [10, 20] - - assert result == MistralStreamingResult( - delta_message=content_delta, - reasoning_ended=True, - tools_called=False, - current_text="prior_tool_texthello", - current_token_ids=[10, 20, 1, 2, 3], - ) - - def test_pre_v15_ignores_prompt_reasoning_end( - self, parser: MistralToolParser, request_obj: ChatCompletionRequest - ) -> None: - mock_tokenizer = MagicMock(spec=MistralTokenizer) - mock_tokenizer.version = 13 - parser.model_tokenizer = mock_tokenizer - - mock_rp = MagicMock(spec=MistralReasoningParser) - mock_rp.start_token_id = 999 - mock_rp.extract_reasoning_streaming.return_value = DeltaMessage( - reasoning="thinking..." - ) - mock_rp.is_reasoning_end_streaming.return_value = False - - result = self._call( - parser, - request_obj, - reasoning_parser=mock_rp, - reasoning_ended=False, - prompt_is_reasoning_end=True, - current_token_ids=[999, 1, 2], - ) - - mock_rp.extract_reasoning_streaming.assert_called_once() - assert result == MistralStreamingResult( - delta_message=DeltaMessage(reasoning="thinking..."), - reasoning_ended=False, - tools_called=False, - current_text="hello", - current_token_ids=[999, 1, 2], - ) - - def test_non_pre_v15_prompt_reasoning_end( - self, parser: MistralToolParser, request_obj: ChatCompletionRequest - ) -> None: - mock_tokenizer = MagicMock(spec=MistralTokenizer) - mock_tokenizer.version = 15 - parser.model_tokenizer = mock_tokenizer - - mock_rp = MagicMock(spec=MistralReasoningParser) - mock_rp.start_token_id = 999 - - content_delta = DeltaMessage(content="after reasoning") - with patch.object( - parser, "extract_tool_calls_streaming", return_value=content_delta - ): - result = self._call( - parser, - request_obj, - reasoning_parser=mock_rp, - reasoning_ended=False, - prompt_is_reasoning_end=True, - current_token_ids=[999, 1, 2], - output_token_ids=[10, 20, 30], - ) - - mock_rp.extract_reasoning_streaming.assert_not_called() - assert result == MistralStreamingResult( - delta_message=content_delta, - reasoning_ended=True, - tools_called=False, - current_text="hello", - current_token_ids=[10, 20, 30], - ) - - def test_reasoning_end_transition_with_content( - self, parser: MistralToolParser, request_obj: ChatCompletionRequest - ) -> None: - """When reasoning ends and the delta has content, that content is - cleared from delta_message and used as current_text for tool parsing.""" - mock_rp = MagicMock() - mock_rp.start_token_id = 999 - mock_rp.extract_reasoning_streaming.return_value = DeltaMessage( - reasoning="think", content="leftover" - ) - mock_rp.is_reasoning_end_streaming.return_value = True - mock_rp.extract_content_ids.return_value = [50, 51] - - content_delta = DeltaMessage(content="leftover") - with patch.object( - parser, "extract_tool_calls_streaming", return_value=content_delta - ) as mock_extract: - result = self._call( - parser, - request_obj, - reasoning_parser=mock_rp, - reasoning_ended=False, - current_token_ids=[999, 1, 2], - output_token_ids=[10, 20, 30], - ) - - mock_rp.extract_content_ids.assert_called_once_with([10, 20, 30]) - _, call_kwargs = mock_extract.call_args - assert call_kwargs["previous_text"] == "" - assert call_kwargs["previous_token_ids"] == [] - assert call_kwargs["delta_text"] == "leftover" - assert call_kwargs["current_token_ids"] == [50, 51] - - assert result == MistralStreamingResult( - delta_message=content_delta, - reasoning_ended=True, - tools_called=False, - current_text="leftover", - current_token_ids=[50, 51], - ) - - def test_reasoning_end_transition_without_content( - self, parser: MistralToolParser, request_obj: ChatCompletionRequest - ) -> None: - """When reasoning ends but the delta has no content, current_text - is set to empty string.""" - mock_rp = MagicMock() - mock_rp.start_token_id = 999 - mock_rp.extract_reasoning_streaming.return_value = DeltaMessage( - reasoning="think" - ) - mock_rp.is_reasoning_end_streaming.return_value = True - mock_rp.extract_content_ids.return_value = [50, 51] - - empty_delta = DeltaMessage(content="") - with patch.object( - parser, "extract_tool_calls_streaming", return_value=empty_delta - ) as mock_extract: - result = self._call( - parser, - request_obj, - reasoning_parser=mock_rp, - reasoning_ended=False, - current_token_ids=[999, 1, 2], - output_token_ids=[10, 20, 30], - ) - - _, call_kwargs = mock_extract.call_args - assert call_kwargs["delta_text"] == "" - assert call_kwargs["current_token_ids"] == [50, 51] - - assert result == MistralStreamingResult( - delta_message=empty_delta, - reasoning_ended=True, - tools_called=False, - current_text="", - current_token_ids=[50, 51], - ) diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 9dd9a34162..12c43bfeda 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -72,7 +72,7 @@ from vllm.renderers import ChatParams from vllm.sampling_params import BeamSearchParams, SamplingParams from vllm.tokenizers import TokenizerLike from vllm.utils.collection_utils import as_list -from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser +from vllm.utils.mistral import is_mistral_tool_parser if TYPE_CHECKING: from vllm.entrypoints.serve.render.serving import OpenAIServingRender @@ -425,8 +425,6 @@ class OpenAIServingChat(OpenAIServing): harmony_tools_streamed = [False] * num_choices tools_streamed = [False] * num_choices - is_mistral_grammar_path = request._grammar_from_tool_parser - if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam): tool_choice_function_name = request.tool_choice.function.name else: @@ -450,15 +448,12 @@ class OpenAIServingChat(OpenAIServing): # Only one of these will be used, thus previous_texts and # all_previous_token_ids will not be used twice in the same iteration. if ( - is_mistral_grammar_path - or tool_choice_auto + tool_choice_auto or tool_choice_function_name or request.tool_choice == "required" or reasoning_parser ): all_previous_token_ids = [[] for _ in range(num_choices)] - reasoning_end_arr = [False] * num_choices - prompt_is_reasoning_end_arr: list[bool | None] = [None] * num_choices else: all_previous_token_ids = None @@ -592,18 +587,6 @@ class OpenAIServingChat(OpenAIServing): for output in res.outputs: i = output.index parser = parsers[i] - tool_parser = parser.tool_parser if parser is not None else None - - if ( - reasoning_parser - and res.prompt_token_ids - and prompt_is_reasoning_end_arr[i] is None - ): - # only check once per choice, because prompt_token_ids - # are the same for all deltas in that choice - prompt_is_reasoning_end_arr[i] = ( - reasoning_parser.is_reasoning_end(res.prompt_token_ids) - ) if finish_reason_sent[i]: continue @@ -658,8 +641,7 @@ class OpenAIServingChat(OpenAIServing): # just update previous_texts and previous_token_ids if ( - is_mistral_grammar_path - or tool_choice_auto + tool_choice_auto or tool_choice_function_name or request.tool_choice == "required" or reasoning_parser @@ -687,35 +669,6 @@ class OpenAIServingChat(OpenAIServing): ) ) harmony_tools_streamed[i] |= tools_streamed_flag - # Mistral grammar path: combined reasoning + tool streaming - elif is_mistral_grammar_path: - from vllm.tool_parsers.mistral_tool_parser import ( - MistralToolParser, - ) - - assert tool_parser is not None - assert isinstance(tool_parser, MistralToolParser) - assert reasoning_end_arr is not None - output_token_ids = as_list(output.token_ids) - result = tool_parser.extract_maybe_reasoning_and_tool_streaming( - reasoning_parser=reasoning_parser, - previous_text=previous_text, - current_text=current_text, - delta_text=delta_text, - previous_token_ids=previous_token_ids, - current_token_ids=current_token_ids, - output_token_ids=output_token_ids, - reasoning_ended=reasoning_end_arr[i], - prompt_is_reasoning_end=(prompt_is_reasoning_end_arr[i]), - request=request, - ) - delta_message = result.delta_message - reasoning_end_arr[i] = result.reasoning_ended - current_text = result.current_text - current_token_ids = result.current_token_ids - if result.tools_called: - tools_streamed[i] = True - elif parser is not None: delta_message = parser.parse_delta( delta_text=delta_text, @@ -732,8 +685,7 @@ class OpenAIServingChat(OpenAIServing): # update the previous values for the next iteration if ( - is_mistral_grammar_path - or tool_choice_auto + tool_choice_auto or tool_choice_function_name or request.tool_choice == "required" or reasoning_parser @@ -1067,32 +1019,8 @@ class OpenAIServingChat(OpenAIServing): tool_calls = [] auto_tools_called = False - if is_mistral_tokenizer(tokenizer): - from vllm.tool_parsers.mistral_tool_parser import MistralToolCall - tool_call_class: type[ToolCall] = MistralToolCall - else: - tool_call_class = ToolCall - - use_mistral_tool_parser = request._grammar_from_tool_parser - if use_mistral_tool_parser: - from vllm.tool_parsers.mistral_tool_parser import MistralToolParser - - tool_call_items = MistralToolParser.build_non_streaming_tool_calls( - tool_calls - ) - if tool_call_items: - auto_tools_called = ( - request.tool_choice is None or request.tool_choice == "auto" - ) - message = ChatMessage( - role=role, - reasoning=reasoning, - content=content, - tool_calls=tool_call_items, - ) - - elif (not self.enable_auto_tools or not self.tool_parser) and ( + if (not self.enable_auto_tools or not self.tool_parser) and ( not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam) and request.tool_choice != "required" ): @@ -1102,70 +1030,42 @@ class OpenAIServingChat(OpenAIServing): request.tool_choice and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam ): - tool_call_class_items = [] + tool_call_items = [] tool_calls = tool_calls or [] - for idx, tc in enumerate(tool_calls): - # Use native ID if available (e.g., Kimi K2), - # otherwise generate ID with correct id_type - if tc.id: - tool_call_class_items.append( - tool_call_class(id=tc.id, function=tc) + for tc in tool_calls: + if not tc.id: + tc.id = make_tool_call_id( + id_type=self.tool_call_id_type, + func_name=tc.name, + idx=history_tool_call_cnt, ) - else: - # Generate ID using the correct format (kimi_k2 or random), - # but leave it to the class if it's Mistral to preserve - # 9-char IDs - if is_mistral_tokenizer(tokenizer): - tool_call_class_items.append(tool_call_class(function=tc)) - else: - generated_id = make_tool_call_id( - id_type=self.tool_call_id_type, - func_name=tc.name, - idx=history_tool_call_cnt, - ) - tool_call_class_items.append( - tool_call_class(id=generated_id, function=tc) - ) + tool_call_items.append(ToolCall(id=tc.id, function=tc)) history_tool_call_cnt += 1 message = ChatMessage( role=role, reasoning=reasoning, - content="", - tool_calls=tool_call_class_items, + content=content or "", + tool_calls=tool_call_items, ) elif request.tool_choice and request.tool_choice == "required": - tool_call_class_items = [] + tool_call_items = [] tool_calls = tool_calls or [] - for idx, tool_call in enumerate(tool_calls): - # Use native ID if available, - # otherwise generate ID with correct id_type - if tool_call.id: - tool_call_class_items.append( - tool_call_class(id=tool_call.id, function=tool_call) + for tool_call in tool_calls: + if not tool_call.id: + tool_call.id = make_tool_call_id( + id_type=self.tool_call_id_type, + func_name=tool_call.name, + idx=history_tool_call_cnt, ) - else: - # Generate ID using the correct format (kimi_k2 or random), - # but leave it to the class if it's Mistral to preserve - # 9-char IDs - if is_mistral_tokenizer(tokenizer): - tool_call_class_items.append( - tool_call_class(function=tool_call) - ) - else: - generated_id = make_tool_call_id( - id_type=self.tool_call_id_type, - func_name=tool_call.name, - idx=history_tool_call_cnt, - ) - tool_call_class_items.append( - tool_call_class(id=generated_id, function=tool_call) - ) + tool_call_items.append( + ToolCall(id=tool_call.id, function=tool_call) + ) history_tool_call_cnt += 1 message = ChatMessage( role=role, - content="", - tool_calls=tool_call_class_items, + content=content or "", + tool_calls=tool_call_items, reasoning=reasoning, ) @@ -1181,34 +1081,17 @@ class OpenAIServingChat(OpenAIServing): and self.enable_auto_tools and self.tool_parser ): - # In the OpenAI API the finish_reason is "tools_called" - # if the tool choice is auto and the model produced a tool - # call. The same is not true for named function calls auto_tools_called = tool_calls is not None and len(tool_calls) > 0 if tool_calls: tool_call_items = [] - for idx, tc in enumerate(tool_calls): - # Use native ID if available (e.g., Kimi K2), - # otherwise generate ID with correct id_type - if tc.id: - tool_call_items.append( - tool_call_class(id=tc.id, function=tc) + for tc in tool_calls: + if not tc.id: + tc.id = make_tool_call_id( + id_type=self.tool_call_id_type, + func_name=tc.name, + idx=history_tool_call_cnt, ) - else: - # Generate ID using the correct format (kimi_k2 or random), - # but leave it to the class if it's Mistral to preserve - # 9-char IDs - if is_mistral_tokenizer(tokenizer): - tool_call_items.append(tool_call_class(function=tc)) - else: - generated_id = make_tool_call_id( - id_type=self.tool_call_id_type, - func_name=tc.name, - idx=history_tool_call_cnt, - ) - tool_call_items.append( - tool_call_class(id=generated_id, function=tc) - ) + tool_call_items.append(ToolCall(id=tc.id, function=tc)) history_tool_call_cnt += 1 message = ChatMessage( role=role, @@ -1218,18 +1101,10 @@ class OpenAIServingChat(OpenAIServing): ) else: - # FOR NOW make it a chat message; we will have to detect - # the type to make it later. - ret_content = content - - # try to use content return from tool parser first, - # tool parser may do some modify for the content. - if content and len(content) > 0: - ret_content = content message = ChatMessage( role=role, reasoning=reasoning, - content=ret_content, + content=content, ) # undetermined case that is still important to handle diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index d5ea574bf7..53809a126c 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -43,7 +43,6 @@ from vllm.tool_parsers.streaming import ( extract_required_tool_call_streaming, ) from vllm.utils import random_uuid -from vllm.utils.mistral import is_mistral_tool_parser logger = init_logger(__name__) @@ -546,14 +545,6 @@ class DelegatingParser(Parser): if tool_parser is None: return [], content - # When the Mistral grammar factory injected structured outputs, - # let the parser handle the output. - use_mistral_tool_parser = ( - is_mistral_tool_parser(type(tool_parser)) - and isinstance(request, ChatCompletionRequest) - and request._grammar_from_tool_parser - ) - supports_required_and_named = tool_parser.supports_required_and_named is_named_tool_choice = request.tool_choice and isinstance( request.tool_choice, @@ -570,11 +561,7 @@ class DelegatingParser(Parser): ) tool_calls = list[FunctionCall]() - if ( - is_named_tool_choice - and supports_required_and_named - and not use_mistral_tool_parser - ): + if is_named_tool_choice and supports_required_and_named: if content is None: return [], None tool_calls.append( @@ -584,11 +571,7 @@ class DelegatingParser(Parser): ) ) content = None - elif ( - is_required_tool_choice - and supports_required_and_named - and not use_mistral_tool_parser - ): + elif is_required_tool_choice and supports_required_and_named: # "required" with standard JSON-based parsing parsed_calls = [] with contextlib.suppress(ValidationError): @@ -604,7 +587,7 @@ class DelegatingParser(Parser): ) ) content = None - elif is_auto_tool_choice or use_mistral_tool_parser: + elif is_auto_tool_choice: # Automatic Tool Call Parsing (also used as fallback for # required/named when supports_required_and_named=False) tool_call_info = tool_parser.extract_tool_calls( diff --git a/vllm/parser/mistral.py b/vllm/parser/mistral.py new file mode 100644 index 0000000000..c7f557a5a9 --- /dev/null +++ b/vllm/parser/mistral.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall +from vllm.parser.abstract_parser import DelegatingParser + +if TYPE_CHECKING: + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ) + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest + + +class MistralParser(DelegatingParser): + def __init__(self, tokenizer, tools=None, *args, **kwargs): + super().__init__(tokenizer, tools, *args, **kwargs) + from vllm.tool_parsers.mistral_tool_parser import MistralToolParser + + if not isinstance(self._tool_parser, MistralToolParser): + raise ValueError( + "MistralParser requires --tool-call-parser mistral, " + f"got {self._tool_parser.__class__.__name__}." + ) + + def _maybe_force_auto_tool_parsing( + self, request: ChatCompletionRequest | ResponsesRequest + ) -> None: + # When the Mistral grammar factory injected structured outputs, + # the model emits v11+ format ([TOOL_CALLS]name{args}) that the + # named/required parsers can't handle. Disable them so all + # tool_choice modes fall back to auto tool parsing via + # extract_tool_calls. + if getattr(request, "_grammar_from_tool_parser", False): + assert self._tool_parser is not None + self._tool_parser.supports_required_and_named = False + + def parse( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + enable_auto_tools: bool = False, + ) -> tuple[str | None, str | None, list[FunctionCall] | None]: + self._maybe_force_auto_tool_parsing(request) + reasoning, content, tool_calls = super().parse( + model_output, request, enable_auto_tools + ) + if tool_calls: + from vllm.tool_parsers.mistral_tool_parser import MistralToolCall + + # Named/required tool_choice builds FunctionCalls without + # ID, backfill with Mistral-format IDs. + for tc in tool_calls: + if not tc.id: + tc.id = MistralToolCall.generate_random_id() + return reasoning, content, tool_calls + + def parse_delta( + self, + delta_text: str, + delta_token_ids: list[int], + request: ChatCompletionRequest | ResponsesRequest, + prompt_token_ids: list[int] | None = None, + *, + finished: bool, + ) -> DeltaMessage | None: + self._maybe_force_auto_tool_parsing(request) + return super().parse_delta( + delta_text, + delta_token_ids, + request, + prompt_token_ids, + finished=finished, + ) diff --git a/vllm/parser/parser_manager.py b/vllm/parser/parser_manager.py index 7afd39d4fe..6c2fdf52dd 100644 --- a/vllm/parser/parser_manager.py +++ b/vllm/parser/parser_manager.py @@ -106,6 +106,15 @@ class ParserManager: if reasoning_parser_cls is None and tool_parser_cls is None: return None + from vllm.utils.mistral import is_mistral_tool_parser + + if is_mistral_tool_parser(tool_parser_cls): + from vllm.parser.mistral import MistralParser + + MistralParser.reasoning_parser_cls = reasoning_parser_cls + MistralParser.tool_parser_cls = tool_parser_cls + return MistralParser + from vllm.parser.abstract_parser import DelegatingParser r_cls = reasoning_parser_cls diff --git a/vllm/reasoning/mistral_reasoning_parser.py b/vllm/reasoning/mistral_reasoning_parser.py index 7117716b6f..74e32cfd16 100644 --- a/vllm/reasoning/mistral_reasoning_parser.py +++ b/vllm/reasoning/mistral_reasoning_parser.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Sequence +from collections.abc import Iterable, Sequence from functools import cached_property from typing import TYPE_CHECKING @@ -76,6 +76,15 @@ class MistralReasoningParser(BaseThinkingReasoningParser): has_eot_token = True return False + def is_reasoning_end_streaming( + self, input_ids: Sequence[int], delta_ids: Iterable[int] + ) -> bool: + if self.end_token_id in delta_ids: + return True + # Grammar's think? is optional — if [THINK] was never generated, + # reasoning was skipped entirely. + return self.start_token_id not in input_ids + def extract_content_ids(self, input_ids: list[int]) -> list[int]: """ Extract the content diff --git a/vllm/tool_parsers/mistral_tool_parser.py b/vllm/tool_parsers/mistral_tool_parser.py index 0a057a3af4..1d605557b1 100644 --- a/vllm/tool_parsers/mistral_tool_parser.py +++ b/vllm/tool_parsers/mistral_tool_parser.py @@ -5,11 +5,10 @@ from __future__ import annotations import json from collections.abc import Sequence -from dataclasses import dataclass from enum import Enum, auto from random import choices from string import ascii_letters, digits -from typing import TYPE_CHECKING, Any +from typing import Any import ijson import regex as re @@ -40,19 +39,14 @@ from vllm.entrypoints.openai.engine.protocol import ( ) from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.logger import init_logger -from vllm.reasoning.mistral_reasoning_parser import MistralReasoningParser from vllm.sampling_params import StructuredOutputsParams from vllm.tokenizers import TokenizerLike -from vllm.tokenizers.mistral import MistralTokenizer from vllm.tool_parsers.abstract_tool_parser import ( Tool, ToolParser, ) from vllm.utils.mistral import is_mistral_tokenizer -if TYPE_CHECKING: - from vllm.reasoning import ReasoningParser - logger = init_logger(__name__) ALPHANUMERIC = ascii_letters + digits @@ -99,19 +93,6 @@ def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool: return "[ARGS]" not in vocab -@dataclass -class MistralStreamingResult: - r"""Encapsulates the mutable state returned from - `MistralToolParser.extract_maybe_reasoning_and_tool_streaming`. - """ - - delta_message: DeltaMessage | None - reasoning_ended: bool - tools_called: bool - current_text: str - current_token_ids: list[int] - - class MistralToolParser(ToolParser): r"""Tool call parser for Mistral models, intended for use with either: @@ -281,148 +262,6 @@ class MistralToolParser(ToolParser): request._grammar_from_tool_parser = True return request - def extract_maybe_reasoning_and_tool_streaming( - self, - *, - reasoning_parser: ReasoningParser | None, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: list[int], - current_token_ids: list[int], - output_token_ids: Sequence[int], - reasoning_ended: bool, - prompt_is_reasoning_end: bool | None, - request: ChatCompletionRequest, - ) -> MistralStreamingResult: - r"""Streaming extraction with reasoning followed by tool-call parsing. - - This method encapsulates the combined reasoning extraction and - tool-call streaming logic so that the serving layer only needs a - thin routing branch. - - The flow is: - - 1. If a *reasoning_parser* is present and reasoning has **not** ended, - extract reasoning tokens. Pre-v15 models may have pre-filled - `[THINK]...[/THINK]` in system prompts, so we skip the - prompt-level reasoning-end check for those. - 2. Once reasoning ends (or if there is no reasoning parser), delegate - to `extract_tool_calls_streaming` and track whether tools were - called. - - Args: - reasoning_parser: Optional reasoning parser instance. - previous_text: Accumulated text from prior chunks. - current_text: Full accumulated text including current chunk. - delta_text: New text in this chunk. - previous_token_ids: Token ids from prior chunks. - current_token_ids: Full token ids including current chunk. - output_token_ids: Raw output token ids from the engine. - reasoning_ended: Whether reasoning has already ended. - prompt_is_reasoning_end: Whether the prompt itself ends reasoning. - request: The originating chat completion request. - """ - delta_message: DeltaMessage | None = None - tools_called = False - reasoning_ended_at_entry = reasoning_ended - - # For MistralReasoningParser, only enter the reasoning block when - # the model has actually emitted a [THINK] token. Other reasoning - # parsers always expect thinking to be present. - expect_thinking = ( - not isinstance(reasoning_parser, MistralReasoningParser) - or reasoning_parser.start_token_id in current_token_ids - ) - if reasoning_parser is not None and not reasoning_ended and expect_thinking: - # Pre-v15 models may have pre-filled [THINK]...[/THINK] in - # system prompts, so skip the prompt-level reasoning-end - # check and wait for the output's own end-of-think. - is_pre_v15 = ( - isinstance(self.model_tokenizer, MistralTokenizer) - and self.model_tokenizer.version < 15 - ) - - if not is_pre_v15 and prompt_is_reasoning_end: - reasoning_ended = True - current_token_ids = list(output_token_ids) - else: - delta_message = reasoning_parser.extract_reasoning_streaming( - previous_text, - current_text, - delta_text, - previous_token_ids, - current_token_ids, - output_token_ids, - ) - if reasoning_parser.is_reasoning_end_streaming( - current_token_ids, output_token_ids - ): - reasoning_ended = True - current_token_ids = reasoning_parser.extract_content_ids( - list(output_token_ids) - ) - if delta_message and delta_message.content: - current_text = delta_message.content - delta_message.content = None - else: - current_text = "" - - if not reasoning_ended: - return MistralStreamingResult( - delta_message=delta_message, - reasoning_ended=False, - tools_called=False, - current_text=current_text, - current_token_ids=current_token_ids, - ) - - delta_token_ids = list(output_token_ids) - - # On the iteration where reasoning just ended, reset the text/token - # state so the tool parser sees a clean history instead of the - # accumulated reasoning text. - if not reasoning_ended_at_entry and reasoning_ended: - previous_text = "" - previous_token_ids = [] - delta_text = current_text - delta_token_ids = current_token_ids - - delta_message = self.extract_tool_calls_streaming( - previous_text=previous_text, - current_text=current_text, - delta_text=delta_text, - previous_token_ids=previous_token_ids, - current_token_ids=current_token_ids, - delta_token_ids=delta_token_ids, - request=request, - ) - if delta_message and delta_message.tool_calls: - tools_called = True - - return MistralStreamingResult( - delta_message=delta_message, - reasoning_ended=reasoning_ended, - tools_called=tools_called, - current_text=current_text, - current_token_ids=current_token_ids, - ) - - @staticmethod - def build_non_streaming_tool_calls( - tool_calls: list[FunctionCall] | None, - ) -> list[ToolCall]: - r"""Build `MistralToolCall` items for non-streaming responses.""" - if not tool_calls: - return [] - - return [ - MistralToolCall(id=tc.id, function=tc) - if tc.id - else MistralToolCall(function=tc) - for tc in tool_calls - ] - def extract_tool_calls( self, model_output: str, @@ -536,7 +375,7 @@ class MistralToolParser(ToolParser): return ExtractedToolCallInformation( tools_called=True, tool_calls=mistral_tool_calls, - content=content if len(content) > 0 else None, + content=content if content.strip() else None, ) def extract_tool_calls_streaming(