[Bugfix][Parser] Mistral pre-v11: don't fail requests on unexpected tool call JSON (#54844)

Signed-off-by: lzhan011 <35493221+lzhan011@users.noreply.github.com>
Signed-off-by: Flora Feng <4florafeng@gmail.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
lzhan011
2026-10-06 03:37:30 +00:00
committed by GitHub
co-authored by Flora Feng
parent 8c417a2dae
commit a3e0243b1c
2 changed files with 208 additions and 56 deletions
+128 -11
View File
@@ -475,17 +475,22 @@ def test_extract_tool_calls_pre_v11_tokenizer(
assert extracted_tool_calls.content == expected_content
def test_extract_tool_calls_pre_v11_multiple_bot_tokens_raises(
def test_extract_tool_calls_pre_v11_multiple_bot_tokens(
mistral_pre_v11_tool_parser,
):
"""A second BOT token must not fail the request; like streaming, only the
calls after the first BOT token are extracted."""
model_output = (
'[TOOL_CALLS] [{"name": "add", "arguments":{"a": 1}}]'
'[TOOL_CALLS] [{"name": "sub", "arguments":{"b": 2}}]'
)
with pytest.raises(ValueError, match="Only one BOT token"):
mistral_pre_v11_tool_parser.extract_tool_calls(
model_output, request=_DUMMY_REQUEST
)
result = mistral_pre_v11_tool_parser.extract_tool_calls(
model_output, request=_DUMMY_REQUEST
)
assert result.tools_called
assert [(tc.function.name, tc.function.arguments) for tc in result.tool_calls] == [
("add", '{"a": 1}')
]
def test_extract_tool_calls_pre_v11_regex_fallback(
@@ -506,15 +511,126 @@ def test_extract_tool_calls_pre_v11_regex_fallback(
assert result.tool_calls[0].function.arguments == json.dumps({"a": 1, "b": 2})
def test_extract_tool_calls_pre_v11_regex_fallback_fails(
mistral_pre_v11_tool_parser,
@pytest.mark.parametrize(
"stringified_tool_calls",
["not json at all", "42", '["add"]', '{"foo": 1}'],
ids=["invalid_json", "scalar", "list_of_str", "not_a_call"],
)
def test_extract_tool_calls_pre_v11_no_tool_call_returns_content(
mistral_pre_v11_tool_parser, stringified_tool_calls
):
"""Output that holds no tool call is returned as content instead of failing
the request."""
result = mistral_pre_v11_tool_parser.extract_tool_calls(
f"[TOOL_CALLS] {stringified_tool_calls}", request=_DUMMY_REQUEST
)
assert result == ExtractedToolCallInformation(
tools_called=False, tool_calls=[], content=stringified_tool_calls
)
# (model_output, expected (name, arguments), streamed content). Non-streaming
# drops the text after the calls; streaming returns it as content.
_PRE_V11_UNUSUAL_CALLS = [
pytest.param(
'[TOOL_CALLS] {"name": "add", "arguments": {"a": 1}}',
[("add", '{"a": 1}')],
"",
id="bare_object",
),
# The next three end the bare object and start the trailing text in the
# same token, in each state the object can close from.
pytest.param(
'[TOOL_CALLS] {"name": "add", "arguments": {"a": 1}}, '
'{"name": "sub", "arguments": {"b": 2}}',
[("add", '{"a": 1}')],
', {"name": "sub", "arguments": {"b": 2}}',
id="bare_object_then_text",
),
pytest.param(
'[TOOL_CALLS] {"arguments": {"a": 1}, "name": "add"}, '
'{"name": "sub", "arguments": {"b": 2}}',
[("add", '{"a": 1}')],
', {"name": "sub", "arguments": {"b": 2}}',
id="bare_object_name_last_then_text",
),
pytest.param(
'[TOOL_CALLS] {"name": "add", "arguments": "{\\"a\\": 1}"}, '
'{"name": "sub", "arguments": {"b": 2}}',
[("add", '{"a": 1}')],
', {"name": "sub", "arguments": {"b": 2}}',
id="bare_object_string_arguments_then_text",
),
pytest.param(
'[TOOL_CALLS] [{"arguments": {"a": 1}}]',
[("", '{"a": 1}')],
"",
id="missing_name",
),
pytest.param(
'[TOOL_CALLS] [{"name": "add", "arguments": "{\\"a\\": 1}"}, '
'{"name": "sub", "arguments": {"b": 2}}]',
[("add", '{"a": 1}'), ("sub", '{"b": 2}')],
"",
id="string_arguments",
),
pytest.param(
'[TOOL_CALLS] [{"name": "add", "arguments": {"a": 1}}, "junk"]',
[("add", '{"a": 1}')],
"",
id="entry_not_a_call",
),
# Mistral-7B-Instruct-v0.3 emits repeated BOT tokens at temperature 0.
pytest.param(
'[TOOL_CALLS] [TOOL_CALLS] [{"name": "add", "arguments": {"a": 1}}]',
[("add", '{"a": 1}')],
"",
id="consecutive_bot_tokens",
),
pytest.param(
'[TOOL_CALLS] [TOOL_CALLS] {"name": "add", "arguments": {"a": 1}}',
[("add", '{"a": 1}')],
"",
id="consecutive_bot_tokens_bare_object",
),
]
@pytest.mark.parametrize("model_output,expected,_", _PRE_V11_UNUSUAL_CALLS)
def test_extract_tool_calls_pre_v11_unusual_calls(
mistral_pre_v11_tool_parser, model_output, expected, _
):
model_output = "[TOOL_CALLS] not json at all"
result = mistral_pre_v11_tool_parser.extract_tool_calls(
model_output, request=_DUMMY_REQUEST
)
assert result == ExtractedToolCallInformation(
tools_called=False, tool_calls=[], content="not json at all"
assert result.tools_called
assert [
(tc.function.name, tc.function.arguments) for tc in result.tool_calls
] == expected
assert result.content is None
@pytest.mark.parametrize(
"model_output,expected,expected_content", _PRE_V11_UNUSUAL_CALLS
)
def test_extract_tool_calls_streaming_pre_v11_unusual_calls(
mistral_pre_v11_tool_parser,
mistral_pre_v11_tokenizer,
model_output,
expected,
expected_content,
):
"""Streaming extracts the same calls as the non-streaming path."""
_test_extract_tool_calls_streaming(
mistral_pre_v11_tool_parser,
mistral_pre_v11_tokenizer,
model_output,
None,
[
ToolCall(function=FunctionCall(name=name, arguments=arguments))
for name, arguments in expected
],
expected_content,
)
@@ -882,6 +998,7 @@ def _test_extract_tool_calls_streaming(
# if a new tool is being called, set up empty arguments
if tool_call.index != tool_call_idx:
tool_call_idx = tool_call.index
function_names.append("")
function_args_strs.append("")
tool_call_ids.append(None)
@@ -895,7 +1012,7 @@ def _test_extract_tool_calls_streaming(
# IN ENTIRETY, exactly one time.
if tool_call.function.name:
assert isinstance(tool_call.function.name, str)
function_names.append(tool_call.function.name)
function_names[tool_call.index] = tool_call.function.name
if tool_call.function.arguments:
# make sure they're a string and then add them to the list
+80 -45
View File
@@ -117,6 +117,37 @@ def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool:
return _ARGS not in vocab
def _pre_v11_function_calls(parsed: Any) -> list[FunctionCall]:
"""Decode the JSON after a pre-v11 ``[TOOL_CALLS]`` marker.
Mirrors the streaming path: a bare object is a single call, entries that
are not call objects are skipped, a missing or non-string name becomes
``""`` (as in the v11+ path) and string arguments are kept as-is.
"""
if isinstance(parsed, dict):
parsed = [parsed]
if not isinstance(parsed, list):
return []
function_calls: list[FunctionCall] = []
for tool_call in parsed:
if not isinstance(tool_call, dict) or not (
"name" in tool_call or "arguments" in tool_call
):
continue
name = tool_call.get("name")
arguments = tool_call.get("arguments")
if arguments is None:
arguments = "{}"
elif not isinstance(arguments, str):
arguments = json.dumps(arguments, ensure_ascii=False)
function_calls.append(
FunctionCall(
name=name if isinstance(name, str) else "", arguments=arguments
)
)
return function_calls
@functools.cache
def mistral_config(
*,
@@ -301,6 +332,7 @@ class MistralParser(ParserEngine):
self.starting_new_tool: bool = False
self.streamed_args_for_tool: list[str] = []
self._is_pre_v11: bool = _is_pre_v11_tokeniser(tokenizer)
self.current_tool_string_arguments: str | None = None
self.parse_coro = None
if self._is_pre_v11:
self.parse_coro = ijson.parse_coro(
@@ -551,16 +583,13 @@ class MistralParser(ParserEngine):
content: str | None = None
if self.bot_token in model_output:
content_and_raw_tool_calls = model_output.split(self.bot_token)
content = content_and_raw_tool_calls[0]
raw_tool_calls = content_and_raw_tool_calls[1:]
# pre-v11: content[BOT] [{tool_call1},{tool_call2}]
if len(raw_tool_calls) != 1:
raise ValueError(
"Only one BOT token should have been outputted, "
f"but got {model_output}."
)
stringified_tool_calls = raw_tool_calls[0].strip()
# Repeated BOT tokens with nothing between them count as one, and
# output after a later BOT token is trailing text, as in streaming.
content, *segments = model_output.split(self.bot_token)
stringified_tool_calls = next(
(segment.strip() for segment in segments if segment.strip()), ""
)
elif tool_choice == "required" or isinstance(
tool_choice, ChatCompletionNamedToolChoiceParam
):
@@ -575,49 +604,31 @@ class MistralParser(ParserEngine):
# Use raw_decode to parse the first valid JSON value,
# ignoring trailing tokens the model may emit after
# the tool call array.
tool_calls, _ = json.JSONDecoder().raw_decode(stringified_tool_calls)
parsed, _ = json.JSONDecoder().raw_decode(stringified_tool_calls)
except json.JSONDecodeError:
try:
raw_tool_call = self.tool_call_regex.findall(stringified_tool_calls)[0]
tool_calls = json.loads(raw_tool_call)
tool_calls = [
{
"name": tool_call["name"],
"arguments": json.dumps(
tool_call.get("arguments", {}),
ensure_ascii=False,
),
}
for tool_call in tool_calls
]
parsed = json.loads(raw_tool_call)
except (IndexError, json.JSONDecodeError):
logger.exception("Error in extracting tool call from response.")
return ExtractedToolCallInformation(
tools_called=False,
tool_calls=[],
content=stringified_tool_calls,
parsed = None
function_calls = _pre_v11_function_calls(parsed)
if not function_calls:
if parsed is not None:
logger.warning(
"No tool call found in the JSON after %s; returning it as content.",
self.bot_token,
)
else:
tool_calls = [
{
"name": tool_call["name"],
"arguments": json.dumps(
tool_call.get("arguments", {}),
ensure_ascii=False,
),
}
for tool_call in tool_calls
]
return ExtractedToolCallInformation(
tools_called=False,
tool_calls=[],
content=stringified_tool_calls,
)
mistral_tool_calls: list[MistralToolCall] = [
MistralToolCall(
type="function",
function=FunctionCall(
name=tool_call["name"],
arguments=tool_call.get("arguments", "{}"),
),
)
for tool_call in tool_calls
MistralToolCall(type="function", function=function_call)
for function_call in function_calls
]
return ExtractedToolCallInformation(
@@ -669,9 +680,16 @@ class MistralParser(ParserEngine):
@ijson.coroutine
def update_stream_state_pre_v11_tokenizer(self):
bare_object = False
while True:
(prefix, event, value) = yield
# A bare top-level object is a single call: parse it as an item.
if prefix == "" and event == "start_map":
bare_object = True
if bare_object:
prefix = f"item.{prefix}" if prefix else "item"
if prefix == "item" and event == "start_map":
self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
self.starting_new_tool = True
@@ -686,8 +704,14 @@ class MistralParser(ParserEngine):
self.streaming_state = StreamingState.PARSING_ARGUMENTS
if prefix == "item.arguments" and event == "end_map":
self.streaming_state = StreamingState.PARSING_ARGUMENTS_COMPLETED
if prefix == "item.arguments" and event == "string":
self.current_tool_string_arguments = value
if prefix == "item" and event == "end_map":
self.streaming_state = StreamingState.TOOL_COMPLETE
self.streaming_state = (
StreamingState.ALL_TOOLS_COMPLETE
if bare_object
else StreamingState.TOOL_COMPLETE
)
if prefix == "" and event == "end_array":
self.streaming_state = StreamingState.ALL_TOOLS_COMPLETE
@@ -723,22 +747,27 @@ class MistralParser(ParserEngine):
delta_text=delta_text,
stop_after_opening_curly_braces=1,
)
# Every state in which the call object can close stops after "}",
# so text following a bare object is never sent to ijson.
elif self.streaming_state == StreamingState.WAITING_FOR_TOOL_KEY:
delta_to_be_parsed, delta_text = self._split_delta(
delta_text=delta_text,
stop_after_colon=1,
stop_after_opening_curly_braces=1,
stop_after_closing_curly_braces=1,
)
elif self.streaming_state == StreamingState.PARSING_NAME:
delta_to_be_parsed, delta_text = self._split_delta(
delta_text=delta_text,
stop_after_comma=1,
stop_after_closing_brackets=1,
stop_after_closing_curly_braces=1,
)
elif self.streaming_state == StreamingState.WAITING_FOR_ARGUMENTS_START:
delta_to_be_parsed, delta_text = self._split_delta(
delta_text=delta_text,
stop_after_opening_curly_braces=1,
stop_after_closing_curly_braces=1,
)
elif self.streaming_state == StreamingState.PARSING_ARGUMENTS:
delta_to_be_parsed, delta_text = self._split_delta(
@@ -799,6 +828,12 @@ class MistralParser(ParserEngine):
self.current_tool_name
)
self.current_tool_name = None
if self.current_tool_string_arguments is not None:
current_tool_call_modified = True
current_tool_call.function.arguments = (
self.current_tool_string_arguments
)
self.current_tool_string_arguments = None
if self.streaming_state == StreamingState.PARSING_NAME_COMPLETED:
self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
if self.streaming_state in [