[BUGFIX] Support streamed_args_for_tool in MistralToolParser (#41730)
Signed-off-by: juliendenize <julien.denize@mistral.ai>
This commit is contained in:
@@ -590,6 +590,33 @@ def _test_extract_tool_calls_streaming(
|
||||
]
|
||||
assert_tool_calls(actual_tool_calls, expected_tool_calls)
|
||||
|
||||
if expected_tool_calls:
|
||||
assert len(tool_parser.streamed_args_for_tool) == len(expected_tool_calls)
|
||||
assert len(tool_parser.prev_tool_call_arr) == len(expected_tool_calls)
|
||||
for i in range(len(expected_tool_calls)):
|
||||
assert (
|
||||
tool_parser.prev_tool_call_arr[i]["arguments"]
|
||||
== tool_parser.streamed_args_for_tool[i]
|
||||
)
|
||||
assert tool_parser.streamed_args_for_tool[i] == function_args_strs[i]
|
||||
assert (
|
||||
tool_parser.prev_tool_call_arr[i]["name"]
|
||||
== expected_tool_calls[i].function.name
|
||||
)
|
||||
|
||||
# Simulate the serving layer's unstreamed-args check
|
||||
index = len(tool_parser.prev_tool_call_arr) - 1
|
||||
args = tool_parser.prev_tool_call_arr[index].get("arguments", {})
|
||||
expected_call = (
|
||||
args if isinstance(args, str) else json.dumps(args, ensure_ascii=False)
|
||||
)
|
||||
actual_call = tool_parser.streamed_args_for_tool[index]
|
||||
remaining_call = expected_call.replace(actual_call, "", 1)
|
||||
assert remaining_call == ""
|
||||
else:
|
||||
assert len(tool_parser.streamed_args_for_tool) == 0
|
||||
assert len(tool_parser.prev_tool_call_arr) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
ids=[
|
||||
@@ -855,6 +882,8 @@ def test_extract_tool_calls_streaming_v11_no_tools(
|
||||
previous_text = current_text
|
||||
|
||||
assert collected_content == model_output
|
||||
assert len(mistral_tool_parser.streamed_args_for_tool) == 0
|
||||
assert len(mistral_tool_parser.prev_tool_call_arr) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -623,13 +623,6 @@ class MistralToolParser(ToolParser):
|
||||
if len(delta_tool_calls) > 0:
|
||||
delta.tool_calls = delta_tool_calls
|
||||
|
||||
# HACK: serving_chat.py inspects the internal state of tool parsers
|
||||
# when determining its final streaming delta, automatically
|
||||
# adding autocompleted JSON.
|
||||
# These two lines avoid that nonsense while ensuring finish_reason
|
||||
# is set to tool_calls when at least one tool is called.
|
||||
if delta_tool_calls and not self.prev_tool_call_arr:
|
||||
self.prev_tool_call_arr = [{"arguments": {}}]
|
||||
return delta
|
||||
|
||||
def _generate_delta_tool_call(self, delta_text: str) -> list[DeltaToolCall]:
|
||||
@@ -642,6 +635,8 @@ class MistralToolParser(ToolParser):
|
||||
StreamingState.PARSING_ARGUMENTS,
|
||||
] and delta_text.startswith(self.bot_token):
|
||||
self.current_tool_id += 1
|
||||
self.streamed_args_for_tool.append("")
|
||||
self.prev_tool_call_arr.append({})
|
||||
self.streaming_state = StreamingState.PARSING_NAME
|
||||
delta_text = delta_text.replace(self.bot_token, "", 1)
|
||||
if self.streaming_state == StreamingState.PARSING_NAME:
|
||||
@@ -655,6 +650,9 @@ class MistralToolParser(ToolParser):
|
||||
self.current_tool_name += delta_function_name
|
||||
# HF tokenizers may include [ARGS] in the text
|
||||
self.current_tool_name = self.current_tool_name.replace("[ARGS]", "")
|
||||
self.prev_tool_call_arr[self.current_tool_id]["name"] = (
|
||||
self.current_tool_name
|
||||
)
|
||||
delta_text = delta_text[len(delta_function_name) :]
|
||||
self.streaming_state = StreamingState.PARSING_ARGUMENTS
|
||||
else:
|
||||
@@ -671,6 +669,10 @@ class MistralToolParser(ToolParser):
|
||||
self.streaming_state = StreamingState.TOOL_COMPLETE
|
||||
else:
|
||||
delta_arguments = delta_text
|
||||
self.streamed_args_for_tool[self.current_tool_id] += delta_arguments
|
||||
self.prev_tool_call_arr[self.current_tool_id]["arguments"] = (
|
||||
self.streamed_args_for_tool[self.current_tool_id]
|
||||
)
|
||||
ret = []
|
||||
if self.current_tool_name or delta_arguments:
|
||||
ret += [
|
||||
@@ -820,9 +822,12 @@ class MistralToolParser(ToolParser):
|
||||
if self.current_tool_mistral_id is not None:
|
||||
current_tool_call.id = self.current_tool_mistral_id
|
||||
self.current_tool_mistral_id = None
|
||||
self._track_streamed_args_pre_v11(current_tool_call)
|
||||
delta_tool_calls.append(current_tool_call)
|
||||
current_tool_call_modified = False
|
||||
self.current_tool_id += 1
|
||||
self.streamed_args_for_tool.append("")
|
||||
self.prev_tool_call_arr.append({})
|
||||
self.current_tool_mistral_id = MistralToolCall.generate_random_id()
|
||||
current_tool_call = DeltaToolCall(
|
||||
index=self.current_tool_id,
|
||||
@@ -835,6 +840,9 @@ class MistralToolParser(ToolParser):
|
||||
# we have the complete tool name
|
||||
current_tool_call_modified = True
|
||||
current_tool_call.function.name = self.current_tool_name
|
||||
self.prev_tool_call_arr[self.current_tool_id]["name"] = (
|
||||
self.current_tool_name
|
||||
)
|
||||
self.current_tool_name = None
|
||||
if self.streaming_state == StreamingState.PARSING_NAME_COMPLETED:
|
||||
self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
|
||||
@@ -860,16 +868,9 @@ class MistralToolParser(ToolParser):
|
||||
if self.current_tool_mistral_id is not None:
|
||||
current_tool_call.id = self.current_tool_mistral_id
|
||||
self.current_tool_mistral_id = None
|
||||
self._track_streamed_args_pre_v11(current_tool_call)
|
||||
delta_tool_calls.append(current_tool_call)
|
||||
|
||||
# HACK: serving_chat.py inspects the internal state of tool parsers
|
||||
# when determining it's final streaming delta, automatically
|
||||
# adding autocompleted JSON.
|
||||
# These two lines avoid that nonsense while ensuring finish_reason
|
||||
# is set to tool_calls when at least one tool is called.
|
||||
if delta_tool_calls and not self.prev_tool_call_arr:
|
||||
self.prev_tool_call_arr = [{"arguments": {}}]
|
||||
|
||||
if content or len(delta_tool_calls) > 0:
|
||||
delta_message = DeltaMessage()
|
||||
if content:
|
||||
@@ -883,6 +884,16 @@ class MistralToolParser(ToolParser):
|
||||
else:
|
||||
return None
|
||||
|
||||
def _track_streamed_args_pre_v11(self, tool_call: DeltaToolCall) -> None:
|
||||
r"""Accumulate `tool_call` arguments into the streaming state."""
|
||||
if tool_call.function is not None and tool_call.function.arguments is not None:
|
||||
self.streamed_args_for_tool[self.current_tool_id] += (
|
||||
tool_call.function.arguments
|
||||
)
|
||||
self.prev_tool_call_arr[self.current_tool_id]["arguments"] = (
|
||||
self.streamed_args_for_tool[self.current_tool_id]
|
||||
)
|
||||
|
||||
def _split_delta(
|
||||
self,
|
||||
delta_text: str,
|
||||
|
||||
Reference in New Issue
Block a user