[Bugfix][Parser] Mistral pre-v11: don't fail requests on unexpected tool call JSON (#54844)
Signed-off-by: lzhan011 <35493221+lzhan011@users.noreply.github.com> Signed-off-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -475,17 +475,22 @@ def test_extract_tool_calls_pre_v11_tokenizer(
|
||||
assert extracted_tool_calls.content == expected_content
|
||||
|
||||
|
||||
def test_extract_tool_calls_pre_v11_multiple_bot_tokens_raises(
|
||||
def test_extract_tool_calls_pre_v11_multiple_bot_tokens(
|
||||
mistral_pre_v11_tool_parser,
|
||||
):
|
||||
"""A second BOT token must not fail the request; like streaming, only the
|
||||
calls after the first BOT token are extracted."""
|
||||
model_output = (
|
||||
'[TOOL_CALLS] [{"name": "add", "arguments":{"a": 1}}]'
|
||||
'[TOOL_CALLS] [{"name": "sub", "arguments":{"b": 2}}]'
|
||||
)
|
||||
with pytest.raises(ValueError, match="Only one BOT token"):
|
||||
mistral_pre_v11_tool_parser.extract_tool_calls(
|
||||
model_output, request=_DUMMY_REQUEST
|
||||
)
|
||||
result = mistral_pre_v11_tool_parser.extract_tool_calls(
|
||||
model_output, request=_DUMMY_REQUEST
|
||||
)
|
||||
assert result.tools_called
|
||||
assert [(tc.function.name, tc.function.arguments) for tc in result.tool_calls] == [
|
||||
("add", '{"a": 1}')
|
||||
]
|
||||
|
||||
|
||||
def test_extract_tool_calls_pre_v11_regex_fallback(
|
||||
@@ -506,15 +511,126 @@ def test_extract_tool_calls_pre_v11_regex_fallback(
|
||||
assert result.tool_calls[0].function.arguments == json.dumps({"a": 1, "b": 2})
|
||||
|
||||
|
||||
def test_extract_tool_calls_pre_v11_regex_fallback_fails(
|
||||
mistral_pre_v11_tool_parser,
|
||||
@pytest.mark.parametrize(
|
||||
"stringified_tool_calls",
|
||||
["not json at all", "42", '["add"]', '{"foo": 1}'],
|
||||
ids=["invalid_json", "scalar", "list_of_str", "not_a_call"],
|
||||
)
|
||||
def test_extract_tool_calls_pre_v11_no_tool_call_returns_content(
|
||||
mistral_pre_v11_tool_parser, stringified_tool_calls
|
||||
):
|
||||
"""Output that holds no tool call is returned as content instead of failing
|
||||
the request."""
|
||||
result = mistral_pre_v11_tool_parser.extract_tool_calls(
|
||||
f"[TOOL_CALLS] {stringified_tool_calls}", request=_DUMMY_REQUEST
|
||||
)
|
||||
assert result == ExtractedToolCallInformation(
|
||||
tools_called=False, tool_calls=[], content=stringified_tool_calls
|
||||
)
|
||||
|
||||
|
||||
# (model_output, expected (name, arguments), streamed content). Non-streaming
|
||||
# drops the text after the calls; streaming returns it as content.
|
||||
_PRE_V11_UNUSUAL_CALLS = [
|
||||
pytest.param(
|
||||
'[TOOL_CALLS] {"name": "add", "arguments": {"a": 1}}',
|
||||
[("add", '{"a": 1}')],
|
||||
"",
|
||||
id="bare_object",
|
||||
),
|
||||
# The next three end the bare object and start the trailing text in the
|
||||
# same token, in each state the object can close from.
|
||||
pytest.param(
|
||||
'[TOOL_CALLS] {"name": "add", "arguments": {"a": 1}}, '
|
||||
'{"name": "sub", "arguments": {"b": 2}}',
|
||||
[("add", '{"a": 1}')],
|
||||
', {"name": "sub", "arguments": {"b": 2}}',
|
||||
id="bare_object_then_text",
|
||||
),
|
||||
pytest.param(
|
||||
'[TOOL_CALLS] {"arguments": {"a": 1}, "name": "add"}, '
|
||||
'{"name": "sub", "arguments": {"b": 2}}',
|
||||
[("add", '{"a": 1}')],
|
||||
', {"name": "sub", "arguments": {"b": 2}}',
|
||||
id="bare_object_name_last_then_text",
|
||||
),
|
||||
pytest.param(
|
||||
'[TOOL_CALLS] {"name": "add", "arguments": "{\\"a\\": 1}"}, '
|
||||
'{"name": "sub", "arguments": {"b": 2}}',
|
||||
[("add", '{"a": 1}')],
|
||||
', {"name": "sub", "arguments": {"b": 2}}',
|
||||
id="bare_object_string_arguments_then_text",
|
||||
),
|
||||
pytest.param(
|
||||
'[TOOL_CALLS] [{"arguments": {"a": 1}}]',
|
||||
[("", '{"a": 1}')],
|
||||
"",
|
||||
id="missing_name",
|
||||
),
|
||||
pytest.param(
|
||||
'[TOOL_CALLS] [{"name": "add", "arguments": "{\\"a\\": 1}"}, '
|
||||
'{"name": "sub", "arguments": {"b": 2}}]',
|
||||
[("add", '{"a": 1}'), ("sub", '{"b": 2}')],
|
||||
"",
|
||||
id="string_arguments",
|
||||
),
|
||||
pytest.param(
|
||||
'[TOOL_CALLS] [{"name": "add", "arguments": {"a": 1}}, "junk"]',
|
||||
[("add", '{"a": 1}')],
|
||||
"",
|
||||
id="entry_not_a_call",
|
||||
),
|
||||
# Mistral-7B-Instruct-v0.3 emits repeated BOT tokens at temperature 0.
|
||||
pytest.param(
|
||||
'[TOOL_CALLS] [TOOL_CALLS] [{"name": "add", "arguments": {"a": 1}}]',
|
||||
[("add", '{"a": 1}')],
|
||||
"",
|
||||
id="consecutive_bot_tokens",
|
||||
),
|
||||
pytest.param(
|
||||
'[TOOL_CALLS] [TOOL_CALLS] {"name": "add", "arguments": {"a": 1}}',
|
||||
[("add", '{"a": 1}')],
|
||||
"",
|
||||
id="consecutive_bot_tokens_bare_object",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_output,expected,_", _PRE_V11_UNUSUAL_CALLS)
|
||||
def test_extract_tool_calls_pre_v11_unusual_calls(
|
||||
mistral_pre_v11_tool_parser, model_output, expected, _
|
||||
):
|
||||
model_output = "[TOOL_CALLS] not json at all"
|
||||
result = mistral_pre_v11_tool_parser.extract_tool_calls(
|
||||
model_output, request=_DUMMY_REQUEST
|
||||
)
|
||||
assert result == ExtractedToolCallInformation(
|
||||
tools_called=False, tool_calls=[], content="not json at all"
|
||||
assert result.tools_called
|
||||
assert [
|
||||
(tc.function.name, tc.function.arguments) for tc in result.tool_calls
|
||||
] == expected
|
||||
assert result.content is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_output,expected,expected_content", _PRE_V11_UNUSUAL_CALLS
|
||||
)
|
||||
def test_extract_tool_calls_streaming_pre_v11_unusual_calls(
|
||||
mistral_pre_v11_tool_parser,
|
||||
mistral_pre_v11_tokenizer,
|
||||
model_output,
|
||||
expected,
|
||||
expected_content,
|
||||
):
|
||||
"""Streaming extracts the same calls as the non-streaming path."""
|
||||
_test_extract_tool_calls_streaming(
|
||||
mistral_pre_v11_tool_parser,
|
||||
mistral_pre_v11_tokenizer,
|
||||
model_output,
|
||||
None,
|
||||
[
|
||||
ToolCall(function=FunctionCall(name=name, arguments=arguments))
|
||||
for name, arguments in expected
|
||||
],
|
||||
expected_content,
|
||||
)
|
||||
|
||||
|
||||
@@ -882,6 +998,7 @@ def _test_extract_tool_calls_streaming(
|
||||
# if a new tool is being called, set up empty arguments
|
||||
if tool_call.index != tool_call_idx:
|
||||
tool_call_idx = tool_call.index
|
||||
function_names.append("")
|
||||
function_args_strs.append("")
|
||||
tool_call_ids.append(None)
|
||||
|
||||
@@ -895,7 +1012,7 @@ def _test_extract_tool_calls_streaming(
|
||||
# IN ENTIRETY, exactly one time.
|
||||
if tool_call.function.name:
|
||||
assert isinstance(tool_call.function.name, str)
|
||||
function_names.append(tool_call.function.name)
|
||||
function_names[tool_call.index] = tool_call.function.name
|
||||
|
||||
if tool_call.function.arguments:
|
||||
# make sure they're a string and then add them to the list
|
||||
|
||||
+80
-45
@@ -117,6 +117,37 @@ def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool:
|
||||
return _ARGS not in vocab
|
||||
|
||||
|
||||
def _pre_v11_function_calls(parsed: Any) -> list[FunctionCall]:
|
||||
"""Decode the JSON after a pre-v11 ``[TOOL_CALLS]`` marker.
|
||||
|
||||
Mirrors the streaming path: a bare object is a single call, entries that
|
||||
are not call objects are skipped, a missing or non-string name becomes
|
||||
``""`` (as in the v11+ path) and string arguments are kept as-is.
|
||||
"""
|
||||
if isinstance(parsed, dict):
|
||||
parsed = [parsed]
|
||||
if not isinstance(parsed, list):
|
||||
return []
|
||||
function_calls: list[FunctionCall] = []
|
||||
for tool_call in parsed:
|
||||
if not isinstance(tool_call, dict) or not (
|
||||
"name" in tool_call or "arguments" in tool_call
|
||||
):
|
||||
continue
|
||||
name = tool_call.get("name")
|
||||
arguments = tool_call.get("arguments")
|
||||
if arguments is None:
|
||||
arguments = "{}"
|
||||
elif not isinstance(arguments, str):
|
||||
arguments = json.dumps(arguments, ensure_ascii=False)
|
||||
function_calls.append(
|
||||
FunctionCall(
|
||||
name=name if isinstance(name, str) else "", arguments=arguments
|
||||
)
|
||||
)
|
||||
return function_calls
|
||||
|
||||
|
||||
@functools.cache
|
||||
def mistral_config(
|
||||
*,
|
||||
@@ -301,6 +332,7 @@ class MistralParser(ParserEngine):
|
||||
self.starting_new_tool: bool = False
|
||||
self.streamed_args_for_tool: list[str] = []
|
||||
self._is_pre_v11: bool = _is_pre_v11_tokeniser(tokenizer)
|
||||
self.current_tool_string_arguments: str | None = None
|
||||
self.parse_coro = None
|
||||
if self._is_pre_v11:
|
||||
self.parse_coro = ijson.parse_coro(
|
||||
@@ -551,16 +583,13 @@ class MistralParser(ParserEngine):
|
||||
|
||||
content: str | None = None
|
||||
if self.bot_token in model_output:
|
||||
content_and_raw_tool_calls = model_output.split(self.bot_token)
|
||||
content = content_and_raw_tool_calls[0]
|
||||
raw_tool_calls = content_and_raw_tool_calls[1:]
|
||||
# pre-v11: content[BOT] [{tool_call1},{tool_call2}]
|
||||
if len(raw_tool_calls) != 1:
|
||||
raise ValueError(
|
||||
"Only one BOT token should have been outputted, "
|
||||
f"but got {model_output}."
|
||||
)
|
||||
stringified_tool_calls = raw_tool_calls[0].strip()
|
||||
# Repeated BOT tokens with nothing between them count as one, and
|
||||
# output after a later BOT token is trailing text, as in streaming.
|
||||
content, *segments = model_output.split(self.bot_token)
|
||||
stringified_tool_calls = next(
|
||||
(segment.strip() for segment in segments if segment.strip()), ""
|
||||
)
|
||||
elif tool_choice == "required" or isinstance(
|
||||
tool_choice, ChatCompletionNamedToolChoiceParam
|
||||
):
|
||||
@@ -575,49 +604,31 @@ class MistralParser(ParserEngine):
|
||||
# Use raw_decode to parse the first valid JSON value,
|
||||
# ignoring trailing tokens the model may emit after
|
||||
# the tool call array.
|
||||
tool_calls, _ = json.JSONDecoder().raw_decode(stringified_tool_calls)
|
||||
parsed, _ = json.JSONDecoder().raw_decode(stringified_tool_calls)
|
||||
except json.JSONDecodeError:
|
||||
try:
|
||||
raw_tool_call = self.tool_call_regex.findall(stringified_tool_calls)[0]
|
||||
tool_calls = json.loads(raw_tool_call)
|
||||
tool_calls = [
|
||||
{
|
||||
"name": tool_call["name"],
|
||||
"arguments": json.dumps(
|
||||
tool_call.get("arguments", {}),
|
||||
ensure_ascii=False,
|
||||
),
|
||||
}
|
||||
for tool_call in tool_calls
|
||||
]
|
||||
parsed = json.loads(raw_tool_call)
|
||||
except (IndexError, json.JSONDecodeError):
|
||||
logger.exception("Error in extracting tool call from response.")
|
||||
return ExtractedToolCallInformation(
|
||||
tools_called=False,
|
||||
tool_calls=[],
|
||||
content=stringified_tool_calls,
|
||||
parsed = None
|
||||
|
||||
function_calls = _pre_v11_function_calls(parsed)
|
||||
if not function_calls:
|
||||
if parsed is not None:
|
||||
logger.warning(
|
||||
"No tool call found in the JSON after %s; returning it as content.",
|
||||
self.bot_token,
|
||||
)
|
||||
else:
|
||||
tool_calls = [
|
||||
{
|
||||
"name": tool_call["name"],
|
||||
"arguments": json.dumps(
|
||||
tool_call.get("arguments", {}),
|
||||
ensure_ascii=False,
|
||||
),
|
||||
}
|
||||
for tool_call in tool_calls
|
||||
]
|
||||
return ExtractedToolCallInformation(
|
||||
tools_called=False,
|
||||
tool_calls=[],
|
||||
content=stringified_tool_calls,
|
||||
)
|
||||
|
||||
mistral_tool_calls: list[MistralToolCall] = [
|
||||
MistralToolCall(
|
||||
type="function",
|
||||
function=FunctionCall(
|
||||
name=tool_call["name"],
|
||||
arguments=tool_call.get("arguments", "{}"),
|
||||
),
|
||||
)
|
||||
for tool_call in tool_calls
|
||||
MistralToolCall(type="function", function=function_call)
|
||||
for function_call in function_calls
|
||||
]
|
||||
|
||||
return ExtractedToolCallInformation(
|
||||
@@ -669,9 +680,16 @@ class MistralParser(ParserEngine):
|
||||
|
||||
@ijson.coroutine
|
||||
def update_stream_state_pre_v11_tokenizer(self):
|
||||
bare_object = False
|
||||
while True:
|
||||
(prefix, event, value) = yield
|
||||
|
||||
# A bare top-level object is a single call: parse it as an item.
|
||||
if prefix == "" and event == "start_map":
|
||||
bare_object = True
|
||||
if bare_object:
|
||||
prefix = f"item.{prefix}" if prefix else "item"
|
||||
|
||||
if prefix == "item" and event == "start_map":
|
||||
self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
|
||||
self.starting_new_tool = True
|
||||
@@ -686,8 +704,14 @@ class MistralParser(ParserEngine):
|
||||
self.streaming_state = StreamingState.PARSING_ARGUMENTS
|
||||
if prefix == "item.arguments" and event == "end_map":
|
||||
self.streaming_state = StreamingState.PARSING_ARGUMENTS_COMPLETED
|
||||
if prefix == "item.arguments" and event == "string":
|
||||
self.current_tool_string_arguments = value
|
||||
if prefix == "item" and event == "end_map":
|
||||
self.streaming_state = StreamingState.TOOL_COMPLETE
|
||||
self.streaming_state = (
|
||||
StreamingState.ALL_TOOLS_COMPLETE
|
||||
if bare_object
|
||||
else StreamingState.TOOL_COMPLETE
|
||||
)
|
||||
if prefix == "" and event == "end_array":
|
||||
self.streaming_state = StreamingState.ALL_TOOLS_COMPLETE
|
||||
|
||||
@@ -723,22 +747,27 @@ class MistralParser(ParserEngine):
|
||||
delta_text=delta_text,
|
||||
stop_after_opening_curly_braces=1,
|
||||
)
|
||||
# Every state in which the call object can close stops after "}",
|
||||
# so text following a bare object is never sent to ijson.
|
||||
elif self.streaming_state == StreamingState.WAITING_FOR_TOOL_KEY:
|
||||
delta_to_be_parsed, delta_text = self._split_delta(
|
||||
delta_text=delta_text,
|
||||
stop_after_colon=1,
|
||||
stop_after_opening_curly_braces=1,
|
||||
stop_after_closing_curly_braces=1,
|
||||
)
|
||||
elif self.streaming_state == StreamingState.PARSING_NAME:
|
||||
delta_to_be_parsed, delta_text = self._split_delta(
|
||||
delta_text=delta_text,
|
||||
stop_after_comma=1,
|
||||
stop_after_closing_brackets=1,
|
||||
stop_after_closing_curly_braces=1,
|
||||
)
|
||||
elif self.streaming_state == StreamingState.WAITING_FOR_ARGUMENTS_START:
|
||||
delta_to_be_parsed, delta_text = self._split_delta(
|
||||
delta_text=delta_text,
|
||||
stop_after_opening_curly_braces=1,
|
||||
stop_after_closing_curly_braces=1,
|
||||
)
|
||||
elif self.streaming_state == StreamingState.PARSING_ARGUMENTS:
|
||||
delta_to_be_parsed, delta_text = self._split_delta(
|
||||
@@ -799,6 +828,12 @@ class MistralParser(ParserEngine):
|
||||
self.current_tool_name
|
||||
)
|
||||
self.current_tool_name = None
|
||||
if self.current_tool_string_arguments is not None:
|
||||
current_tool_call_modified = True
|
||||
current_tool_call.function.arguments = (
|
||||
self.current_tool_string_arguments
|
||||
)
|
||||
self.current_tool_string_arguments = None
|
||||
if self.streaming_state == StreamingState.PARSING_NAME_COMPLETED:
|
||||
self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
|
||||
if self.streaming_state in [
|
||||
|
||||
Reference in New Issue
Block a user