From a3e0243b1c3ea313b902a43b0f6559fb4148ff81 Mon Sep 17 00:00:00 2001 From: lzhan011 <35493221+lzhan011@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:37:30 -0500 Subject: [PATCH] [Bugfix][Parser] Mistral pre-v11: don't fail requests on unexpected tool call JSON (#54844) Signed-off-by: lzhan011 <35493221+lzhan011@users.noreply.github.com> Signed-off-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> --- tests/parser/mistral/test_tool_calls.py | 139 ++++++++++++++++++++++-- vllm/parser/mistral.py | 125 +++++++++++++-------- 2 files changed, 208 insertions(+), 56 deletions(-) diff --git a/tests/parser/mistral/test_tool_calls.py b/tests/parser/mistral/test_tool_calls.py index a3359dccc6..c17f618ba8 100644 --- a/tests/parser/mistral/test_tool_calls.py +++ b/tests/parser/mistral/test_tool_calls.py @@ -475,17 +475,22 @@ def test_extract_tool_calls_pre_v11_tokenizer( assert extracted_tool_calls.content == expected_content -def test_extract_tool_calls_pre_v11_multiple_bot_tokens_raises( +def test_extract_tool_calls_pre_v11_multiple_bot_tokens( mistral_pre_v11_tool_parser, ): + """A second BOT token must not fail the request; like streaming, only the + calls after the first BOT token are extracted.""" model_output = ( '[TOOL_CALLS] [{"name": "add", "arguments":{"a": 1}}]' '[TOOL_CALLS] [{"name": "sub", "arguments":{"b": 2}}]' ) - with pytest.raises(ValueError, match="Only one BOT token"): - mistral_pre_v11_tool_parser.extract_tool_calls( - model_output, request=_DUMMY_REQUEST - ) + result = mistral_pre_v11_tool_parser.extract_tool_calls( + model_output, request=_DUMMY_REQUEST + ) + assert result.tools_called + assert [(tc.function.name, tc.function.arguments) for tc in result.tool_calls] == [ + ("add", '{"a": 1}') + ] def test_extract_tool_calls_pre_v11_regex_fallback( @@ -506,15 +511,126 @@ def test_extract_tool_calls_pre_v11_regex_fallback( assert result.tool_calls[0].function.arguments == json.dumps({"a": 1, "b": 2}) -def test_extract_tool_calls_pre_v11_regex_fallback_fails( - mistral_pre_v11_tool_parser, +@pytest.mark.parametrize( + "stringified_tool_calls", + ["not json at all", "42", '["add"]', '{"foo": 1}'], + ids=["invalid_json", "scalar", "list_of_str", "not_a_call"], +) +def test_extract_tool_calls_pre_v11_no_tool_call_returns_content( + mistral_pre_v11_tool_parser, stringified_tool_calls +): + """Output that holds no tool call is returned as content instead of failing + the request.""" + result = mistral_pre_v11_tool_parser.extract_tool_calls( + f"[TOOL_CALLS] {stringified_tool_calls}", request=_DUMMY_REQUEST + ) + assert result == ExtractedToolCallInformation( + tools_called=False, tool_calls=[], content=stringified_tool_calls + ) + + +# (model_output, expected (name, arguments), streamed content). Non-streaming +# drops the text after the calls; streaming returns it as content. +_PRE_V11_UNUSUAL_CALLS = [ + pytest.param( + '[TOOL_CALLS] {"name": "add", "arguments": {"a": 1}}', + [("add", '{"a": 1}')], + "", + id="bare_object", + ), + # The next three end the bare object and start the trailing text in the + # same token, in each state the object can close from. + pytest.param( + '[TOOL_CALLS] {"name": "add", "arguments": {"a": 1}}, ' + '{"name": "sub", "arguments": {"b": 2}}', + [("add", '{"a": 1}')], + ', {"name": "sub", "arguments": {"b": 2}}', + id="bare_object_then_text", + ), + pytest.param( + '[TOOL_CALLS] {"arguments": {"a": 1}, "name": "add"}, ' + '{"name": "sub", "arguments": {"b": 2}}', + [("add", '{"a": 1}')], + ', {"name": "sub", "arguments": {"b": 2}}', + id="bare_object_name_last_then_text", + ), + pytest.param( + '[TOOL_CALLS] {"name": "add", "arguments": "{\\"a\\": 1}"}, ' + '{"name": "sub", "arguments": {"b": 2}}', + [("add", '{"a": 1}')], + ', {"name": "sub", "arguments": {"b": 2}}', + id="bare_object_string_arguments_then_text", + ), + pytest.param( + '[TOOL_CALLS] [{"arguments": {"a": 1}}]', + [("", '{"a": 1}')], + "", + id="missing_name", + ), + pytest.param( + '[TOOL_CALLS] [{"name": "add", "arguments": "{\\"a\\": 1}"}, ' + '{"name": "sub", "arguments": {"b": 2}}]', + [("add", '{"a": 1}'), ("sub", '{"b": 2}')], + "", + id="string_arguments", + ), + pytest.param( + '[TOOL_CALLS] [{"name": "add", "arguments": {"a": 1}}, "junk"]', + [("add", '{"a": 1}')], + "", + id="entry_not_a_call", + ), + # Mistral-7B-Instruct-v0.3 emits repeated BOT tokens at temperature 0. + pytest.param( + '[TOOL_CALLS] [TOOL_CALLS] [{"name": "add", "arguments": {"a": 1}}]', + [("add", '{"a": 1}')], + "", + id="consecutive_bot_tokens", + ), + pytest.param( + '[TOOL_CALLS] [TOOL_CALLS] {"name": "add", "arguments": {"a": 1}}', + [("add", '{"a": 1}')], + "", + id="consecutive_bot_tokens_bare_object", + ), +] + + +@pytest.mark.parametrize("model_output,expected,_", _PRE_V11_UNUSUAL_CALLS) +def test_extract_tool_calls_pre_v11_unusual_calls( + mistral_pre_v11_tool_parser, model_output, expected, _ ): - model_output = "[TOOL_CALLS] not json at all" result = mistral_pre_v11_tool_parser.extract_tool_calls( model_output, request=_DUMMY_REQUEST ) - assert result == ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content="not json at all" + assert result.tools_called + assert [ + (tc.function.name, tc.function.arguments) for tc in result.tool_calls + ] == expected + assert result.content is None + + +@pytest.mark.parametrize( + "model_output,expected,expected_content", _PRE_V11_UNUSUAL_CALLS +) +def test_extract_tool_calls_streaming_pre_v11_unusual_calls( + mistral_pre_v11_tool_parser, + mistral_pre_v11_tokenizer, + model_output, + expected, + expected_content, +): + """Streaming extracts the same calls as the non-streaming path.""" + _test_extract_tool_calls_streaming( + mistral_pre_v11_tool_parser, + mistral_pre_v11_tokenizer, + model_output, + None, + [ + ToolCall(function=FunctionCall(name=name, arguments=arguments)) + for name, arguments in expected + ], + expected_content, ) @@ -882,6 +998,7 @@ def _test_extract_tool_calls_streaming( # if a new tool is being called, set up empty arguments if tool_call.index != tool_call_idx: tool_call_idx = tool_call.index + function_names.append("") function_args_strs.append("") tool_call_ids.append(None) @@ -895,7 +1012,7 @@ def _test_extract_tool_calls_streaming( # IN ENTIRETY, exactly one time. if tool_call.function.name: assert isinstance(tool_call.function.name, str) - function_names.append(tool_call.function.name) + function_names[tool_call.index] = tool_call.function.name if tool_call.function.arguments: # make sure they're a string and then add them to the list diff --git a/vllm/parser/mistral.py b/vllm/parser/mistral.py index 61f133598c..f6d3435bd7 100644 --- a/vllm/parser/mistral.py +++ b/vllm/parser/mistral.py @@ -117,6 +117,37 @@ def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool: return _ARGS not in vocab +def _pre_v11_function_calls(parsed: Any) -> list[FunctionCall]: + """Decode the JSON after a pre-v11 ``[TOOL_CALLS]`` marker. + + Mirrors the streaming path: a bare object is a single call, entries that + are not call objects are skipped, a missing or non-string name becomes + ``""`` (as in the v11+ path) and string arguments are kept as-is. + """ + if isinstance(parsed, dict): + parsed = [parsed] + if not isinstance(parsed, list): + return [] + function_calls: list[FunctionCall] = [] + for tool_call in parsed: + if not isinstance(tool_call, dict) or not ( + "name" in tool_call or "arguments" in tool_call + ): + continue + name = tool_call.get("name") + arguments = tool_call.get("arguments") + if arguments is None: + arguments = "{}" + elif not isinstance(arguments, str): + arguments = json.dumps(arguments, ensure_ascii=False) + function_calls.append( + FunctionCall( + name=name if isinstance(name, str) else "", arguments=arguments + ) + ) + return function_calls + + @functools.cache def mistral_config( *, @@ -301,6 +332,7 @@ class MistralParser(ParserEngine): self.starting_new_tool: bool = False self.streamed_args_for_tool: list[str] = [] self._is_pre_v11: bool = _is_pre_v11_tokeniser(tokenizer) + self.current_tool_string_arguments: str | None = None self.parse_coro = None if self._is_pre_v11: self.parse_coro = ijson.parse_coro( @@ -551,16 +583,13 @@ class MistralParser(ParserEngine): content: str | None = None if self.bot_token in model_output: - content_and_raw_tool_calls = model_output.split(self.bot_token) - content = content_and_raw_tool_calls[0] - raw_tool_calls = content_and_raw_tool_calls[1:] # pre-v11: content[BOT] [{tool_call1},{tool_call2}] - if len(raw_tool_calls) != 1: - raise ValueError( - "Only one BOT token should have been outputted, " - f"but got {model_output}." - ) - stringified_tool_calls = raw_tool_calls[0].strip() + # Repeated BOT tokens with nothing between them count as one, and + # output after a later BOT token is trailing text, as in streaming. + content, *segments = model_output.split(self.bot_token) + stringified_tool_calls = next( + (segment.strip() for segment in segments if segment.strip()), "" + ) elif tool_choice == "required" or isinstance( tool_choice, ChatCompletionNamedToolChoiceParam ): @@ -575,49 +604,31 @@ class MistralParser(ParserEngine): # Use raw_decode to parse the first valid JSON value, # ignoring trailing tokens the model may emit after # the tool call array. - tool_calls, _ = json.JSONDecoder().raw_decode(stringified_tool_calls) + parsed, _ = json.JSONDecoder().raw_decode(stringified_tool_calls) except json.JSONDecodeError: try: raw_tool_call = self.tool_call_regex.findall(stringified_tool_calls)[0] - tool_calls = json.loads(raw_tool_call) - tool_calls = [ - { - "name": tool_call["name"], - "arguments": json.dumps( - tool_call.get("arguments", {}), - ensure_ascii=False, - ), - } - for tool_call in tool_calls - ] + parsed = json.loads(raw_tool_call) except (IndexError, json.JSONDecodeError): logger.exception("Error in extracting tool call from response.") - return ExtractedToolCallInformation( - tools_called=False, - tool_calls=[], - content=stringified_tool_calls, + parsed = None + + function_calls = _pre_v11_function_calls(parsed) + if not function_calls: + if parsed is not None: + logger.warning( + "No tool call found in the JSON after %s; returning it as content.", + self.bot_token, ) - else: - tool_calls = [ - { - "name": tool_call["name"], - "arguments": json.dumps( - tool_call.get("arguments", {}), - ensure_ascii=False, - ), - } - for tool_call in tool_calls - ] + return ExtractedToolCallInformation( + tools_called=False, + tool_calls=[], + content=stringified_tool_calls, + ) mistral_tool_calls: list[MistralToolCall] = [ - MistralToolCall( - type="function", - function=FunctionCall( - name=tool_call["name"], - arguments=tool_call.get("arguments", "{}"), - ), - ) - for tool_call in tool_calls + MistralToolCall(type="function", function=function_call) + for function_call in function_calls ] return ExtractedToolCallInformation( @@ -669,9 +680,16 @@ class MistralParser(ParserEngine): @ijson.coroutine def update_stream_state_pre_v11_tokenizer(self): + bare_object = False while True: (prefix, event, value) = yield + # A bare top-level object is a single call: parse it as an item. + if prefix == "" and event == "start_map": + bare_object = True + if bare_object: + prefix = f"item.{prefix}" if prefix else "item" + if prefix == "item" and event == "start_map": self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY self.starting_new_tool = True @@ -686,8 +704,14 @@ class MistralParser(ParserEngine): self.streaming_state = StreamingState.PARSING_ARGUMENTS if prefix == "item.arguments" and event == "end_map": self.streaming_state = StreamingState.PARSING_ARGUMENTS_COMPLETED + if prefix == "item.arguments" and event == "string": + self.current_tool_string_arguments = value if prefix == "item" and event == "end_map": - self.streaming_state = StreamingState.TOOL_COMPLETE + self.streaming_state = ( + StreamingState.ALL_TOOLS_COMPLETE + if bare_object + else StreamingState.TOOL_COMPLETE + ) if prefix == "" and event == "end_array": self.streaming_state = StreamingState.ALL_TOOLS_COMPLETE @@ -723,22 +747,27 @@ class MistralParser(ParserEngine): delta_text=delta_text, stop_after_opening_curly_braces=1, ) + # Every state in which the call object can close stops after "}", + # so text following a bare object is never sent to ijson. elif self.streaming_state == StreamingState.WAITING_FOR_TOOL_KEY: delta_to_be_parsed, delta_text = self._split_delta( delta_text=delta_text, stop_after_colon=1, stop_after_opening_curly_braces=1, + stop_after_closing_curly_braces=1, ) elif self.streaming_state == StreamingState.PARSING_NAME: delta_to_be_parsed, delta_text = self._split_delta( delta_text=delta_text, stop_after_comma=1, stop_after_closing_brackets=1, + stop_after_closing_curly_braces=1, ) elif self.streaming_state == StreamingState.WAITING_FOR_ARGUMENTS_START: delta_to_be_parsed, delta_text = self._split_delta( delta_text=delta_text, stop_after_opening_curly_braces=1, + stop_after_closing_curly_braces=1, ) elif self.streaming_state == StreamingState.PARSING_ARGUMENTS: delta_to_be_parsed, delta_text = self._split_delta( @@ -799,6 +828,12 @@ class MistralParser(ParserEngine): self.current_tool_name ) self.current_tool_name = None + if self.current_tool_string_arguments is not None: + current_tool_call_modified = True + current_tool_call.function.arguments = ( + self.current_tool_string_arguments + ) + self.current_tool_string_arguments = None if self.streaming_state == StreamingState.PARSING_NAME_COMPLETED: self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY if self.streaming_state in [