[BUGFIX] Support streamed_args_for_tool in MistralToolParser (#41730)

Signed-off-by: juliendenize <julien.denize@mistral.ai>
This commit is contained in:
Julien Denize
2026-05-05 17:48:53 +00:00
committed by GitHub
parent 628c436301
commit c6235ed180
2 changed files with 55 additions and 15 deletions
@@ -590,6 +590,33 @@ def _test_extract_tool_calls_streaming(
]
assert_tool_calls(actual_tool_calls, expected_tool_calls)
if expected_tool_calls:
assert len(tool_parser.streamed_args_for_tool) == len(expected_tool_calls)
assert len(tool_parser.prev_tool_call_arr) == len(expected_tool_calls)
for i in range(len(expected_tool_calls)):
assert (
tool_parser.prev_tool_call_arr[i]["arguments"]
== tool_parser.streamed_args_for_tool[i]
)
assert tool_parser.streamed_args_for_tool[i] == function_args_strs[i]
assert (
tool_parser.prev_tool_call_arr[i]["name"]
== expected_tool_calls[i].function.name
)
# Simulate the serving layer's unstreamed-args check
index = len(tool_parser.prev_tool_call_arr) - 1
args = tool_parser.prev_tool_call_arr[index].get("arguments", {})
expected_call = (
args if isinstance(args, str) else json.dumps(args, ensure_ascii=False)
)
actual_call = tool_parser.streamed_args_for_tool[index]
remaining_call = expected_call.replace(actual_call, "", 1)
assert remaining_call == ""
else:
assert len(tool_parser.streamed_args_for_tool) == 0
assert len(tool_parser.prev_tool_call_arr) == 0
@pytest.mark.parametrize(
ids=[
@@ -855,6 +882,8 @@ def test_extract_tool_calls_streaming_v11_no_tools(
previous_text = current_text
assert collected_content == model_output
assert len(mistral_tool_parser.streamed_args_for_tool) == 0
assert len(mistral_tool_parser.prev_tool_call_arr) == 0
@pytest.mark.parametrize(
+26 -15
View File
@@ -623,13 +623,6 @@ class MistralToolParser(ToolParser):
if len(delta_tool_calls) > 0:
delta.tool_calls = delta_tool_calls
# HACK: serving_chat.py inspects the internal state of tool parsers
# when determining its final streaming delta, automatically
# adding autocompleted JSON.
# These two lines avoid that nonsense while ensuring finish_reason
# is set to tool_calls when at least one tool is called.
if delta_tool_calls and not self.prev_tool_call_arr:
self.prev_tool_call_arr = [{"arguments": {}}]
return delta
def _generate_delta_tool_call(self, delta_text: str) -> list[DeltaToolCall]:
@@ -642,6 +635,8 @@ class MistralToolParser(ToolParser):
StreamingState.PARSING_ARGUMENTS,
] and delta_text.startswith(self.bot_token):
self.current_tool_id += 1
self.streamed_args_for_tool.append("")
self.prev_tool_call_arr.append({})
self.streaming_state = StreamingState.PARSING_NAME
delta_text = delta_text.replace(self.bot_token, "", 1)
if self.streaming_state == StreamingState.PARSING_NAME:
@@ -655,6 +650,9 @@ class MistralToolParser(ToolParser):
self.current_tool_name += delta_function_name
# HF tokenizers may include [ARGS] in the text
self.current_tool_name = self.current_tool_name.replace("[ARGS]", "")
self.prev_tool_call_arr[self.current_tool_id]["name"] = (
self.current_tool_name
)
delta_text = delta_text[len(delta_function_name) :]
self.streaming_state = StreamingState.PARSING_ARGUMENTS
else:
@@ -671,6 +669,10 @@ class MistralToolParser(ToolParser):
self.streaming_state = StreamingState.TOOL_COMPLETE
else:
delta_arguments = delta_text
self.streamed_args_for_tool[self.current_tool_id] += delta_arguments
self.prev_tool_call_arr[self.current_tool_id]["arguments"] = (
self.streamed_args_for_tool[self.current_tool_id]
)
ret = []
if self.current_tool_name or delta_arguments:
ret += [
@@ -820,9 +822,12 @@ class MistralToolParser(ToolParser):
if self.current_tool_mistral_id is not None:
current_tool_call.id = self.current_tool_mistral_id
self.current_tool_mistral_id = None
self._track_streamed_args_pre_v11(current_tool_call)
delta_tool_calls.append(current_tool_call)
current_tool_call_modified = False
self.current_tool_id += 1
self.streamed_args_for_tool.append("")
self.prev_tool_call_arr.append({})
self.current_tool_mistral_id = MistralToolCall.generate_random_id()
current_tool_call = DeltaToolCall(
index=self.current_tool_id,
@@ -835,6 +840,9 @@ class MistralToolParser(ToolParser):
# we have the complete tool name
current_tool_call_modified = True
current_tool_call.function.name = self.current_tool_name
self.prev_tool_call_arr[self.current_tool_id]["name"] = (
self.current_tool_name
)
self.current_tool_name = None
if self.streaming_state == StreamingState.PARSING_NAME_COMPLETED:
self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
@@ -860,16 +868,9 @@ class MistralToolParser(ToolParser):
if self.current_tool_mistral_id is not None:
current_tool_call.id = self.current_tool_mistral_id
self.current_tool_mistral_id = None
self._track_streamed_args_pre_v11(current_tool_call)
delta_tool_calls.append(current_tool_call)
# HACK: serving_chat.py inspects the internal state of tool parsers
# when determining it's final streaming delta, automatically
# adding autocompleted JSON.
# These two lines avoid that nonsense while ensuring finish_reason
# is set to tool_calls when at least one tool is called.
if delta_tool_calls and not self.prev_tool_call_arr:
self.prev_tool_call_arr = [{"arguments": {}}]
if content or len(delta_tool_calls) > 0:
delta_message = DeltaMessage()
if content:
@@ -883,6 +884,16 @@ class MistralToolParser(ToolParser):
else:
return None
def _track_streamed_args_pre_v11(self, tool_call: DeltaToolCall) -> None:
r"""Accumulate `tool_call` arguments into the streaming state."""
if tool_call.function is not None and tool_call.function.arguments is not None:
self.streamed_args_for_tool[self.current_tool_id] += (
tool_call.function.arguments
)
self.prev_tool_call_arr[self.current_tool_id]["arguments"] = (
self.streamed_args_for_tool[self.current_tool_id]
)
def _split_delta(
self,
delta_text: str,