From c6235ed1803e31bafedd90d53766e5705c570780 Mon Sep 17 00:00:00 2001 From: Julien Denize <40604584+juliendenize@users.noreply.github.com> Date: Tue, 5 May 2026 19:48:53 +0200 Subject: [PATCH] [BUGFIX] Support streamed_args_for_tool in MistralToolParser (#41730) Signed-off-by: juliendenize --- .../tool_parsers/test_mistral_tool_parser.py | 29 +++++++++++++ vllm/tool_parsers/mistral_tool_parser.py | 41 ++++++++++++------- 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/tests/tool_parsers/test_mistral_tool_parser.py b/tests/tool_parsers/test_mistral_tool_parser.py index 42e8cf138b..f6a5c6bfb2 100644 --- a/tests/tool_parsers/test_mistral_tool_parser.py +++ b/tests/tool_parsers/test_mistral_tool_parser.py @@ -590,6 +590,33 @@ def _test_extract_tool_calls_streaming( ] assert_tool_calls(actual_tool_calls, expected_tool_calls) + if expected_tool_calls: + assert len(tool_parser.streamed_args_for_tool) == len(expected_tool_calls) + assert len(tool_parser.prev_tool_call_arr) == len(expected_tool_calls) + for i in range(len(expected_tool_calls)): + assert ( + tool_parser.prev_tool_call_arr[i]["arguments"] + == tool_parser.streamed_args_for_tool[i] + ) + assert tool_parser.streamed_args_for_tool[i] == function_args_strs[i] + assert ( + tool_parser.prev_tool_call_arr[i]["name"] + == expected_tool_calls[i].function.name + ) + + # Simulate the serving layer's unstreamed-args check + index = len(tool_parser.prev_tool_call_arr) - 1 + args = tool_parser.prev_tool_call_arr[index].get("arguments", {}) + expected_call = ( + args if isinstance(args, str) else json.dumps(args, ensure_ascii=False) + ) + actual_call = tool_parser.streamed_args_for_tool[index] + remaining_call = expected_call.replace(actual_call, "", 1) + assert remaining_call == "" + else: + assert len(tool_parser.streamed_args_for_tool) == 0 + assert len(tool_parser.prev_tool_call_arr) == 0 + @pytest.mark.parametrize( ids=[ @@ -855,6 +882,8 @@ def test_extract_tool_calls_streaming_v11_no_tools( previous_text = current_text assert collected_content == model_output + assert len(mistral_tool_parser.streamed_args_for_tool) == 0 + assert len(mistral_tool_parser.prev_tool_call_arr) == 0 @pytest.mark.parametrize( diff --git a/vllm/tool_parsers/mistral_tool_parser.py b/vllm/tool_parsers/mistral_tool_parser.py index 94b7b67897..945820ed9a 100644 --- a/vllm/tool_parsers/mistral_tool_parser.py +++ b/vllm/tool_parsers/mistral_tool_parser.py @@ -623,13 +623,6 @@ class MistralToolParser(ToolParser): if len(delta_tool_calls) > 0: delta.tool_calls = delta_tool_calls - # HACK: serving_chat.py inspects the internal state of tool parsers - # when determining its final streaming delta, automatically - # adding autocompleted JSON. - # These two lines avoid that nonsense while ensuring finish_reason - # is set to tool_calls when at least one tool is called. - if delta_tool_calls and not self.prev_tool_call_arr: - self.prev_tool_call_arr = [{"arguments": {}}] return delta def _generate_delta_tool_call(self, delta_text: str) -> list[DeltaToolCall]: @@ -642,6 +635,8 @@ class MistralToolParser(ToolParser): StreamingState.PARSING_ARGUMENTS, ] and delta_text.startswith(self.bot_token): self.current_tool_id += 1 + self.streamed_args_for_tool.append("") + self.prev_tool_call_arr.append({}) self.streaming_state = StreamingState.PARSING_NAME delta_text = delta_text.replace(self.bot_token, "", 1) if self.streaming_state == StreamingState.PARSING_NAME: @@ -655,6 +650,9 @@ class MistralToolParser(ToolParser): self.current_tool_name += delta_function_name # HF tokenizers may include [ARGS] in the text self.current_tool_name = self.current_tool_name.replace("[ARGS]", "") + self.prev_tool_call_arr[self.current_tool_id]["name"] = ( + self.current_tool_name + ) delta_text = delta_text[len(delta_function_name) :] self.streaming_state = StreamingState.PARSING_ARGUMENTS else: @@ -671,6 +669,10 @@ class MistralToolParser(ToolParser): self.streaming_state = StreamingState.TOOL_COMPLETE else: delta_arguments = delta_text + self.streamed_args_for_tool[self.current_tool_id] += delta_arguments + self.prev_tool_call_arr[self.current_tool_id]["arguments"] = ( + self.streamed_args_for_tool[self.current_tool_id] + ) ret = [] if self.current_tool_name or delta_arguments: ret += [ @@ -820,9 +822,12 @@ class MistralToolParser(ToolParser): if self.current_tool_mistral_id is not None: current_tool_call.id = self.current_tool_mistral_id self.current_tool_mistral_id = None + self._track_streamed_args_pre_v11(current_tool_call) delta_tool_calls.append(current_tool_call) current_tool_call_modified = False self.current_tool_id += 1 + self.streamed_args_for_tool.append("") + self.prev_tool_call_arr.append({}) self.current_tool_mistral_id = MistralToolCall.generate_random_id() current_tool_call = DeltaToolCall( index=self.current_tool_id, @@ -835,6 +840,9 @@ class MistralToolParser(ToolParser): # we have the complete tool name current_tool_call_modified = True current_tool_call.function.name = self.current_tool_name + self.prev_tool_call_arr[self.current_tool_id]["name"] = ( + self.current_tool_name + ) self.current_tool_name = None if self.streaming_state == StreamingState.PARSING_NAME_COMPLETED: self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY @@ -860,16 +868,9 @@ class MistralToolParser(ToolParser): if self.current_tool_mistral_id is not None: current_tool_call.id = self.current_tool_mistral_id self.current_tool_mistral_id = None + self._track_streamed_args_pre_v11(current_tool_call) delta_tool_calls.append(current_tool_call) - # HACK: serving_chat.py inspects the internal state of tool parsers - # when determining it's final streaming delta, automatically - # adding autocompleted JSON. - # These two lines avoid that nonsense while ensuring finish_reason - # is set to tool_calls when at least one tool is called. - if delta_tool_calls and not self.prev_tool_call_arr: - self.prev_tool_call_arr = [{"arguments": {}}] - if content or len(delta_tool_calls) > 0: delta_message = DeltaMessage() if content: @@ -883,6 +884,16 @@ class MistralToolParser(ToolParser): else: return None + def _track_streamed_args_pre_v11(self, tool_call: DeltaToolCall) -> None: + r"""Accumulate `tool_call` arguments into the streaming state.""" + if tool_call.function is not None and tool_call.function.arguments is not None: + self.streamed_args_for_tool[self.current_tool_id] += ( + tool_call.function.arguments + ) + self.prev_tool_call_arr[self.current_tool_id]["arguments"] = ( + self.streamed_args_for_tool[self.current_tool_id] + ) + def _split_delta( self, delta_text: str,