diff --git a/requirements/common.txt b/requirements/common.txt index ec4386baa9..748ffe88f4 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -31,7 +31,7 @@ partial-json-parser # used for parsing partial JSON outputs jsonschema >= 4.23.0 # required for MiniMax M3 tool schema validation pyzmq >= 25.0.0 msgspec -mistral_common[image] >= 1.11.5 +mistral_common[image] >= 1.11.6 opencv-python-headless >= 4.13.0 # required for video IO pyyaml six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12 diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index 923b54bb14..c6f90df7d6 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -476,7 +476,7 @@ mcp==1.28.1 # via -r requirements/test/../common.txt mdurl==0.1.2 # via markdown-it-py -mistral-common==1.11.5 +mistral-common==1.11.6 # via # -r requirements/test/../common.txt # -r requirements/test/cuda.in diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index 377224eac3..51225d922b 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -33,7 +33,7 @@ torchaudio==2.11.0 torchvision==0.28.0 transformers_stream_generator # required for qwen-vl test matplotlib # required for qwen-vl test -mistral_common[image,audio] >= 1.11.5 # required for voxtral test +mistral_common[image,audio] >= 1.11.6 # required for voxtral test num2words # required for smolvlm test open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py datamodel_code_generator # required for minicpm3 test diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index 4d1734e15f..192f3ac5a6 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -502,7 +502,7 @@ mcp==1.28.1 # via -r requirements/test/../common.txt mdurl==0.1.2 # via markdown-it-py -mistral-common==1.11.5 +mistral-common==1.11.6 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/nightly-torch.txt b/requirements/test/nightly-torch.txt index d2436f8f63..1beb4294d0 100644 --- a/requirements/test/nightly-torch.txt +++ b/requirements/test/nightly-torch.txt @@ -23,7 +23,7 @@ jiwer # required for audio tests timm # required for internvl test transformers_stream_generator # required for qwen-vl test matplotlib # required for qwen-vl test -mistral_common[image,audio] >= 1.11.5 # required for voxtral test +mistral_common[image,audio] >= 1.11.6 # required for voxtral test num2words # required for smolvlm test opencv-python-headless >= 4.13.0 # required for video test datamodel_code_generator # required for minicpm3 test diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in index e61ce724e3..b30e47cecb 100644 --- a/requirements/test/rocm.in +++ b/requirements/test/rocm.in @@ -29,7 +29,7 @@ tblib # for pickling test exceptions timm>=1.0.17 # required for internvl and gemma3n-mm test transformers_stream_generator # required for qwen-vl test matplotlib # required for qwen-vl test -mistral_common[image,audio]>=1.11.5 # required for voxtral test +mistral_common[image,audio]>=1.11.6 # required for voxtral test num2words # required for smolvlm test open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py datamodel_code_generator # required for minicpm3 test diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 094d0e2f90..f1f611f919 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -497,7 +497,7 @@ mcp==1.27.0 # via -r requirements/test/../common.txt mdurl==0.1.2 # via markdown-it-py -mistral-common==1.11.5 +mistral-common==1.11.6 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index 30fdee5025..4ecf0bede9 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -373,7 +373,7 @@ mcp==1.28.1 # via -r requirements/test/../common.txt mdurl==0.1.2 # via markdown-it-py -mistral-common==1.11.5 +mistral-common==1.11.6 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/tests/models/language/generation/test_mistral.py b/tests/models/language/generation/test_mistral.py index 2bc9bc350f..fc774bc6e8 100644 --- a/tests/models/language/generation/test_mistral.py +++ b/tests/models/language/generation/test_mistral.py @@ -5,12 +5,10 @@ import json import pytest +from vllm.parser.mistral import MistralToolCall from vllm.sampling_params import SamplingParams from vllm.tokenizers.mistral import MistralTokenizer -from vllm.tool_parsers.mistral_tool_parser import ( - MistralToolCall, - MistralToolParser, -) +from vllm.tool_parsers.mistral_tool_parser import MistralToolParser from ...utils import check_logprobs_close diff --git a/tests/parser/engine/test_delegating_replay.py b/tests/parser/engine/test_delegating_replay.py index b922e71e39..50c41a8ea6 100644 --- a/tests/parser/engine/test_delegating_replay.py +++ b/tests/parser/engine/test_delegating_replay.py @@ -40,6 +40,7 @@ from vllm.parser.engine.adapters import ( ParserEngineReasoningAdapter, ParserEngineToolAdapter, ) +from vllm.parser.mistral import MistralParser _TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam]) @@ -84,6 +85,11 @@ def _discover_pairings() -> list[_PairingInfo]: for engine_cls, adapters in engines.items(): if "tool" not in adapters or "reasoning" not in adapters: continue + if engine_cls is MistralParser: + # Mistral uses brace-balanced JSON tool args with no TOOL_END + # token, so it does not fit this TOOL_END-based replay harness. + # It is covered by tests/parser/mistral/ instead. + continue cfg = engine_cls(bare_tok, None).parser_engine_config if cfg.name not in _BUILDERS: missing_builders.append(f"{engine_cls.__name__} (config.name={cfg.name!r})") diff --git a/tests/parser/engine/test_replay.py b/tests/parser/engine/test_replay.py index 064e1b6236..5bba7f7eaf 100644 --- a/tests/parser/engine/test_replay.py +++ b/tests/parser/engine/test_replay.py @@ -34,6 +34,7 @@ from tests.parser.engine.trace_builder import _BUILDERS, build_samples from vllm.parser.engine import registered_adapters as _adapters_mod from vllm.parser.engine.parser_engine import ParserEngine from vllm.parser.engine.parser_engine_config import ParserState +from vllm.parser.mistral import MistralParser # ── Parser discovery ───────────────────────────────────────────────── @@ -64,6 +65,11 @@ def _discover_parsers() -> list[_ParserInfo]: and obj is not ParserEngine ): continue + if obj is MistralParser: + # Mistral uses brace-balanced JSON tool args with no TOOL_END + # token, so it does not fit this TOOL_END-based replay harness. + # It is covered by tests/parser/mistral/ instead. + continue cfg = obj(bare_tok, None).parser_engine_config if cfg.name not in _BUILDERS: missing_builders.append(f"{obj.__name__} (config.name={cfg.name!r})") diff --git a/tests/parser/mistral/__init__.py b/tests/parser/mistral/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/parser/mistral/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/parser/mistral/test_reasoning.py b/tests/parser/mistral/test_reasoning.py new file mode 100644 index 0000000000..137da64853 --- /dev/null +++ b/tests/parser/mistral/test_reasoning.py @@ -0,0 +1,599 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import json + +import pytest +from mistral_common.protocol.instruct.messages import ( + AssistantMessage, + ThinkChunk, + ToolMessage, + UserMessage, +) +from mistral_common.protocol.instruct.request import InstructRequest +from mistral_common.protocol.instruct.tool_calls import FunctionCall, ToolCall + +from tests.reasoning.utils import run_reasoning_extraction_mistral +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.parser.parser_manager import ParserManager +from vllm.reasoning import ReasoningParser, ReasoningParserManager +from vllm.tokenizers.detokenizer_utils import detokenize_incrementally +from vllm.tokenizers.mistral import MistralTokenizer + +_PARSER_NAME = "mistral" +_MODEL_V13 = "mistralai/Magistral-Small-2509" +_MODEL_V11 = "mistralai/Magistral-Small-2506" + +_SAMPLE_TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } +] + + +@pytest.fixture(scope="module") +def mistral_tokenizer() -> MistralTokenizer: + """v13 Magistral tokenizer with `[THINK]`/`[/THINK]` special tokens.""" + return MistralTokenizer.from_pretrained(_MODEL_V13) + + +@pytest.fixture(scope="module") +def mistral_v11_tokenizer() -> MistralTokenizer: + """v11 Magistral tokenizer using plain-text ``/`` tags.""" + return MistralTokenizer.from_pretrained(_MODEL_V11) + + +def _encode_v13(tokenizer: MistralTokenizer, output: str) -> list[int]: + """Encode output string with `[THINK]`/`[/THINK]` markers into token IDs. + + `[THINK]` and `[/THINK]` placeholders in `output` are replaced by their + special-token IDs; all surrounding text is encoded with the base tokenizer. + Mirrors the encoding helper from the deleted + `tests/reasoning/test_mistral_reasoning_parser.py`. + """ + think_start = "[THINK]" + think_end = "[/THINK]" + len_start = len(think_start) + len_end = len(think_end) + + index_start = output.find(think_start) + index_end = output.find(think_end) + + out: list[int] = [] + if index_start != -1: + out += tokenizer.tokenizer.encode(output[:index_start], False, False) + out += [tokenizer.instruct.BEGIN_THINK] + + if index_end != -1: + middle = output[index_start + len_start : index_end] + suffix = output[index_end + len_end :] + out += tokenizer.tokenizer.encode(middle, False, False) + out += [tokenizer.instruct.END_THINK] + out += tokenizer.tokenizer.encode(suffix, False, False) + else: + out += tokenizer.tokenizer.encode( + output[index_start + len_start :], False, False + ) + elif index_end != -1: + out += tokenizer.tokenizer.encode(output[:index_end], False, False) + out += [tokenizer.instruct.END_THINK] + out += tokenizer.tokenizer.encode(output[index_end + len_end :], False, False) + else: + out += tokenizer.tokenizer.encode(output, False, False) + return out + + +_TEST_CASES = [ + # v13 special-token encoding: [THINK]…[/THINK] with trailing content. + pytest.param( + False, + "[THINK]r[/THINK]c", + "r", + "c", + id="v13_basic", + ), + pytest.param( + True, + "[THINK]r[/THINK]c", + "r", + "c", + id="v13_basic_streaming", + ), + # No trailing content after reasoning ends. + pytest.param( + False, + "[THINK]r[/THINK]", + "r", + None, + id="v13_no_trailing_content", + ), + pytest.param( + True, + "[THINK]r[/THINK]", + "r", + None, + id="v13_no_trailing_content_streaming", + ), + # Multi-line reasoning and multi-line content. + pytest.param( + False, + "[THINK]a\nb[/THINK]c\nd", + "a\nb", + "c\nd", + id="v13_multiline", + ), + pytest.param( + True, + "[THINK]a\nb[/THINK]c\nd", + "a\nb", + "c\nd", + id="v13_multiline_streaming", + ), + # No reasoning at all: output is pure content. + pytest.param( + False, + "hello world", + None, + "hello world", + id="v13_no_reasoning", + ), + pytest.param( + True, + "hello world", + None, + "hello world", + id="v13_no_reasoning_streaming", + ), + # [THINK] opened but never closed; reasoning accumulates, content is None. + pytest.param( + False, + "[THINK]r", + "r", + None, + id="v13_think_no_end", + ), + pytest.param( + True, + "[THINK]r", + "r", + None, + id="v13_think_no_end_streaming", + ), + # Stray [/THINK] with no matching [THINK]: no reasoning, content follows. + pytest.param( + False, + "[/THINK]This is the rest", + None, + "This is the rest", + id="v13_stray_think_end", + ), + pytest.param( + True, + "[/THINK]This is the rest", + None, + "This is the rest", + id="v13_stray_think_end_streaming", + ), + # Content before [THINK]: leading segment joins with trailing segment. + pytest.param( + False, + "Before\n[THINK]r[/THINK]\nAfter", + "r", + "Before\n\nAfter", + id="v13_leading_content", + ), + pytest.param( + True, + "Before\n[THINK]r[/THINK]\nAfter", + "r", + "Before\n\nAfter", + id="v13_leading_content_streaming", + ), + # Empty output: no reasoning, no content. + pytest.param( + False, + "", + None, + None, + id="v13_empty", + ), + pytest.param( + True, + "", + None, + None, + id="v13_empty_streaming", + ), +] + + +@pytest.mark.parametrize( + "streaming, output, expected_reasoning, expected_content", + _TEST_CASES, +) +def test_mistral_reasoning_v13( + streaming: bool, + output: str, + expected_reasoning: str | None, + expected_content: str | None, + mistral_tokenizer: MistralTokenizer, +) -> None: + output_tokens = _encode_v13(mistral_tokenizer, output) + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)( + mistral_tokenizer + ) + reasoning, content = run_reasoning_extraction_mistral( + parser, output_tokens, streaming=streaming + ) + assert reasoning == expected_reasoning + assert content == expected_content + + +def test_system_prompt_think_ignored( + mistral_tokenizer: MistralTokenizer, +) -> None: + """Streaming content extraction must not + include `[THINK]`/`[/THINK]` examples from the system prompt. + """ + output = "[THINK]real[/THINK]answer" + output_tokens = _encode_v13(mistral_tokenizer, output) + for streaming in (False, True): + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser( + _PARSER_NAME + )(mistral_tokenizer) + reasoning, content = run_reasoning_extraction_mistral( + parser, output_tokens, streaming=streaming + ) + assert reasoning == "real" + assert content == "answer" + + +def test_reasoning_with_tool_calls( + mistral_tokenizer: MistralTokenizer, +) -> None: + """Reasoning extraction with a tool call in the generated output.""" + tool_calls_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]") + if tool_calls_token_id is None: + pytest.skip("Tokenizer does not have [TOOL_CALLS] in vocab") + + # Token sequence representing: [THINK]r[/THINK][TOOL_CALLS]f{"a":1} + output_tokens = ( + [mistral_tokenizer.instruct.BEGIN_THINK] + + mistral_tokenizer.tokenizer.encode("r", False, False) + + [mistral_tokenizer.instruct.END_THINK] + + [tool_calls_token_id] + + mistral_tokenizer.tokenizer.encode('f{"a":1}', False, False) + ) + for streaming in (False, True): + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser( + _PARSER_NAME + )(mistral_tokenizer) + reasoning, _content = run_reasoning_extraction_mistral( + parser, output_tokens, streaming=streaming + ) + assert reasoning == "r" + assert "[TOOL_CALLS]" not in (reasoning or "") + + +@pytest.mark.parametrize("streaming", [False, True], ids=["nonstream", "stream"]) +def test_reasoning_v11_plain_text_think( + mistral_v11_tokenizer: MistralTokenizer, streaming: bool +) -> None: + """v11 tokenizers use plain-text ``/`` (no special tokens).""" + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)( + mistral_v11_tokenizer + ) + + tokens = mistral_v11_tokenizer.tokenizer.encode("rc", False, False) + reasoning, content = run_reasoning_extraction_mistral( + parser, tokens, streaming=streaming + ) + assert reasoning == "r" + assert content == "c" + + +def test_is_reasoning_end( + mistral_tokenizer: MistralTokenizer, +) -> None: + """is_reasoning_end returns True iff reasoning has finished.""" + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)( + mistral_tokenizer + ) + + # Complete reasoning: END_THINK present → True. + complete_ids = _encode_v13(mistral_tokenizer, "[THINK]r[/THINK]c") + assert parser.is_reasoning_end(complete_ids) + + # Open-only: no END_THINK, BEGIN_THINK present → False. + open_ids = _encode_v13(mistral_tokenizer, "[THINK]r") + assert not parser.is_reasoning_end(open_ids) + + # No reasoning markers at all → False. + no_reasoning_ids = _encode_v13(mistral_tokenizer, "hello") + assert not parser.is_reasoning_end(no_reasoning_ids) + + # Explicit END_THINK closes reasoning → True. + explicit_end_ids = ( + [mistral_tokenizer.instruct.BEGIN_THINK] + + mistral_tokenizer.tokenizer.encode("r", False, False) + + [mistral_tokenizer.instruct.END_THINK] + ) + assert parser.is_reasoning_end(explicit_end_ids) + + # [TOOL_CALLS] acts as implicit reasoning-end marker. + tool_calls_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]") + if tool_calls_token_id is None: + pytest.skip("Tokenizer does not have [TOOL_CALLS] in vocab") + + implicit_end_ids = ( + [mistral_tokenizer.instruct.BEGIN_THINK] + + mistral_tokenizer.tokenizer.encode("r", False, False) + + [tool_calls_token_id] + ) + assert parser.is_reasoning_end(implicit_end_ids) + + +def _stream_parse_delta(parser, tokenizer, gen_ids, request, prompt_ids): + """Stream `gen_ids` through `parse_delta` and reconstruct reasoning/content.""" + reasoning = "" + content = "" + previous_tokens: list[str] | None = None + prefix_offset = 0 + read_offset = 0 + for i, tok in enumerate(gen_ids): + new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally( + tokenizer=tokenizer, + all_input_ids=gen_ids[: i + 1], + prev_tokens=previous_tokens, + prefix_offset=prefix_offset, + read_offset=read_offset, + skip_special_tokens=True, + spaces_between_special_tokens=True, + ) + previous_tokens = ( + previous_tokens + new_tokens if previous_tokens else new_tokens + ) + delta_message = parser.parse_delta( + delta_text=delta_text, + delta_token_ids=[tok], + request=request, + prompt_token_ids=prompt_ids, + finished=(i == len(gen_ids) - 1), + ) + if delta_message is not None: + reasoning += delta_message.reasoning or "" + content += delta_message.content or "" + return reasoning, content + + +def test_parse_delta_keeps_reasoning_open_after_closed_think_prompt( + mistral_tokenizer: MistralTokenizer, +) -> None: + """A closed `[THINK]` block in the prompt must not finalize reasoning. + + Regression: reasoning system prompts end the prompt with a closed + `[THINK]...[/THINK]` block. With a parser-injected grammar the generation's + reasoning must still be captured as ``reasoning_content`` rather than + leaking into ``content`` (the prompt-based reasoning-end heuristic must be + skipped). This exercises ``DelegatingParser.parse_delta``'s prompt check, + which the isolated reasoning-parser tests do not cover. + """ + parser_cls = ParserManager.get_parser( + tool_parser_name="mistral", + reasoning_parser_name="mistral", + enable_auto_tools=True, + ) + parser = parser_cls(mistral_tokenizer, None) + request = parser.adjust_request( + ChatCompletionRequest( + messages=[], + model="test", + tools=_SAMPLE_TOOLS, + tool_choice="auto", + ) + ) + assert getattr(request, "_grammar_from_parser", False) + + prompt_ids = _encode_v13(mistral_tokenizer, "[THINK]system reasoning[/THINK]") + gen_ids = _encode_v13( + mistral_tokenizer, + "[THINK]because two plus two is four[/THINK]The answer is 4.", + ) + + reasoning, content = _stream_parse_delta( + parser, mistral_tokenizer, gen_ids, request, prompt_ids + ) + + assert "because two plus two is four" in reasoning + assert "because two plus two is four" not in content + assert "The answer is 4." in content + + +def _encode_gen(tokenizer: MistralTokenizer, text: str) -> list[int]: + """Encode `text` mapping `[THINK]`, `[/THINK]`, `[TOOL_CALLS]`, `[ARGS]` + to their special-token IDs; all other text uses the base tokenizer. + """ + vocab = tokenizer.get_vocab() + markers: dict[str, int] = { + "[THINK]": tokenizer.instruct.BEGIN_THINK, + "[/THINK]": tokenizer.instruct.END_THINK, + "[TOOL_CALLS]": vocab["[TOOL_CALLS]"], + "[ARGS]": vocab["[ARGS]"], + } + out: list[int] = [] + remaining = text + while remaining: + earliest: int | None = None + earliest_marker: str | None = None + for marker in markers: + idx = remaining.find(marker) + if idx != -1 and (earliest is None or idx < earliest): + earliest = idx + earliest_marker = marker + if earliest_marker is None: + out += tokenizer.tokenizer.encode(remaining, False, False) + break + assert earliest is not None # set together with earliest_marker + if earliest > 0: + out += tokenizer.tokenizer.encode(remaining[:earliest], False, False) + out.append(markers[earliest_marker]) + remaining = remaining[earliest + len(earliest_marker) :] + return out + + +def _stream_turn( + parser, + tokenizer: MistralTokenizer, + gen_ids: list[int], + request, + prompt_ids: list[int], +) -> tuple[str, str, list[tuple[str, str]]]: + """Stream `gen_ids` through `parse_delta`, returning `(reasoning, content, + tool_calls)`. + + `tool_calls` is a list of `(name, arguments)` tuples ordered by + `DeltaToolCall.index`. Arguments are concatenated across deltas. + """ + reasoning = "" + content = "" + tool_call_acc: dict[int, dict[str, str]] = {} + previous_tokens: list[str] | None = None + prefix_offset = 0 + read_offset = 0 + for i, tok in enumerate(gen_ids): + new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally( + tokenizer=tokenizer, + all_input_ids=gen_ids[: i + 1], + prev_tokens=previous_tokens, + prefix_offset=prefix_offset, + read_offset=read_offset, + skip_special_tokens=True, + spaces_between_special_tokens=True, + ) + previous_tokens = ( + previous_tokens + new_tokens if previous_tokens else new_tokens + ) + delta_message = parser.parse_delta( + delta_text=delta_text, + delta_token_ids=[tok], + request=request, + prompt_token_ids=prompt_ids, + finished=(i == len(gen_ids) - 1), + ) + if delta_message is not None: + reasoning += delta_message.reasoning or "" + content += delta_message.content or "" + for tc in delta_message.tool_calls: + if tc.index not in tool_call_acc: + tool_call_acc[tc.index] = {"name": "", "args": ""} + if tc.function: + if tc.function.name: + tool_call_acc[tc.index]["name"] = tc.function.name + if tc.function.arguments: + tool_call_acc[tc.index]["args"] += tc.function.arguments + tool_calls = [ + (tool_call_acc[k]["name"], tool_call_acc[k]["args"]) + for k in sorted(tool_call_acc) + ] + return reasoning, content, tool_calls + + +def test_parse_delta_multi_turn_reason_and_tool( + mistral_tokenizer: MistralTokenizer, +) -> None: + """Two-turn loop: each assistant turn carries reasoning + a tool call. + + For each turn, the mistral-common conversation is encoded into `prompt_ids` + before streaming the next generation. Turn 2's prompt therefore contains + a `[THINK]`/`[/THINK]` block and `[TOOL_CALLS]` from turn 1 — the + accumulated tool-call/result history that single-turn tests never exercise. + + Asserts that across both turns the parser correctly classifies think tokens + as `reasoning_content` (never leaking into `content`), and that the tool + call name/arguments are reconstructed correctly from the streaming deltas. + """ + parser_cls = ParserManager.get_parser( + tool_parser_name="mistral", + reasoning_parser_name="mistral", + enable_auto_tools=True, + ) + request = parser_cls(mistral_tokenizer, None).adjust_request( + ChatCompletionRequest( + messages=[], + model="test", + tools=_SAMPLE_TOOLS, + tool_choice="auto", + ) + ) + assert getattr(request, "_grammar_from_parser", False) + + cities = ["Dallas", "Paris"] + conversation: list = [UserMessage(content="What is the weather in Dallas?")] + + for i, city in enumerate(cities): + # Fresh parser per turn: streaming state must not bleed across turns. + parser = parser_cls(mistral_tokenizer, None) + + prompt_ids = mistral_tokenizer.instruct.encode_instruct( + InstructRequest(messages=conversation) + ).tokens + + gen_str = ( + f"[THINK]I should check {city} weather.[/THINK]" + f'[TOOL_CALLS]get_weather[ARGS]{{"city": "{city}"}}' + ) + gen_ids = _encode_gen(mistral_tokenizer, gen_str) + + reasoning, content, tool_calls = _stream_turn( + parser, mistral_tokenizer, gen_ids, request, prompt_ids + ) + + assert f"check {city} weather" in reasoning, f"turn {i}: reasoning not captured" + assert f"check {city} weather" not in content, ( + f"turn {i}: reasoning leaked into content" + ) + assert "[THINK]" not in content, f"turn {i}: [THINK] leaked" + assert "[TOOL_CALLS]" not in content, f"turn {i}: [TOOL_CALLS] leaked" + assert len(tool_calls) == 1, f"turn {i}: expected 1 tool call" + name, args = tool_calls[0] + assert name == "get_weather", f"turn {i}: wrong tool name {name!r}" + assert city in args, f"turn {i}: city not in args {args!r}" + + # Extend conversation with this turn's history for the next turn. + tc_id = f"tc{i:07d}" # 9-char alphanumeric id: "tc0000000", "tc0000001" + conversation.extend( + [ + AssistantMessage( + content=[ + ThinkChunk( + thinking=f"I should check {city} weather.", + closed=True, + ) + ], + tool_calls=[ + ToolCall( + id=tc_id, + function=FunctionCall( + name="get_weather", + arguments=json.dumps({"city": city}), + ), + ) + ], + ), + ToolMessage( + content=f"{city}: 70F sunny", + tool_call_id=tc_id, + ), + ] + ) + if i < len(cities) - 1: + conversation.append(UserMessage(content=f"And in {cities[i + 1]}?")) diff --git a/tests/tool_parsers/test_mistral_tool_parser.py b/tests/parser/mistral/test_tool_calls.py similarity index 53% rename from tests/tool_parsers/test_mistral_tool_parser.py rename to tests/parser/mistral/test_tool_calls.py index 4f75e2f576..99f99008d1 100644 --- a/tests/tool_parsers/test_mistral_tool_parser.py +++ b/tests/parser/mistral/test_tool_calls.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch import partial_json_parser import pytest +import regex as re from mistral_common.protocol.instruct.messages import AssistantMessage from mistral_common.protocol.instruct.request import InstructRequest from mistral_common.protocol.instruct.tool_calls import ( @@ -28,21 +29,26 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ) from vllm.entrypoints.openai.engine.protocol import ( + DeltaFunctionCall, DeltaMessage, DeltaToolCall, ExtractedToolCallInformation, StructuralTagResponseFormat, ) +from vllm.parser.engine.events import EventType +from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine +from vllm.parser.mistral import _DEFAULT_JSON_SCHEMA, MistralParser, mistral_config +from vllm.parser.parser_manager import ParserManager from vllm.sampling_params import StructuredOutputsParams from vllm.tokenizers import TokenizerLike, get_tokenizer from vllm.tokenizers.detokenizer_utils import detokenize_incrementally from vllm.tokenizers.mistral import MistralTokenizer -from vllm.tool_parsers.mistral_tool_parser import ( - _DEFAULT_JSON_SCHEMA, - MistralToolParser, -) +from vllm.tool_parsers.mistral_tool_parser import MistralToolParser _DUMMY_REQUEST = ChatCompletionRequest(messages=[], model="test") +# Minimal request for the engine driver: tool_choice=None falls through to +# auto extraction without triggering "tools must be set" validation. +_ENGINE_REQUEST = ChatCompletionRequest(messages=[], model="test", tool_choice=None) @pytest.fixture(scope="module") @@ -71,6 +77,8 @@ def mistral_tool_parser(mistral_tokenizer): def non_mistral_parser() -> MistralToolParser: mock_tokenizer = MagicMock() mock_tokenizer.get_vocab.return_value = {"[TOOL_CALLS]": 1} + # Ensure the legacy (non-grammar) path is taken by MistralParser. + mock_tokenizer.supports_grammar = False return MistralToolParser(mock_tokenizer) @@ -96,9 +104,18 @@ def assert_tool_calls( assert actual_tool_call.function.name == expected_tool_call.function.name, ( f"got wrong function name:${actual_tool_call.function.name}" ) - assert ( - actual_tool_call.function.arguments == expected_tool_call.function.arguments - ), f"got wrong function argument:${actual_tool_call.function.arguments}" + actual_args = actual_tool_call.function.arguments + # Streaming deltas (DeltaToolCall) may have partial JSON because the + # engine holds back the final closing brace until finish() is called. + # Apply partial_json_parser so the comparison works for both the + # legacy (complete JSON) and engine (potentially partial) paths. + if isinstance(actual_tool_call, DeltaToolCall) and actual_args is not None: + actual_args = partial_json_parser.ensure_json( + actual_args, Allow.OBJ | Allow.STR + ) + assert actual_args == expected_tool_call.function.arguments, ( + f"got wrong function argument:${actual_tool_call.function.arguments}" + ) def fix_tool_call_tokenization( @@ -106,45 +123,82 @@ def fix_tool_call_tokenization( mistral_tool_parser: MistralToolParser, mistral_tokenizer: TokenizerLike, ): - """ - Replaces the textual token sequence for [TOOL_CALLS] - with its single special token ID. - """ - textual_tool_call_token_ids = mistral_tokenizer.encode( - text=mistral_tool_parser.bot_token, - add_special_tokens=False, - ) - # textual_tool_call_token_ids must not contain special tokens like bos, eos etc - special_tool_call_token_ids = [mistral_tool_parser.bot_token_id] + """Replace textual token sequences for the control markers ([TOOL_CALLS] + and [ARGS]) with their single special token IDs. - # If the input is too short to contain the sequence, no replacement is possible - if not tokens or len(tokens) < len(textual_tool_call_token_ids): - return tokens + Real generation emits these as control tokens; encoding the marker *string* + can instead yield literal text tokens. The engine detects markers by id, so + the test stream must carry the special ids. + """ + vocab = mistral_tokenizer.get_vocab() + markers: list[tuple[str, int]] = [ + ( + mistral_tool_parser._parser_engine.bot_token, + mistral_tool_parser._parser_engine.bot_token_id, + ) + ] + args_id = vocab.get("[ARGS]") + if args_id is not None: + markers.append(("[ARGS]", args_id)) - result_tokens = [] + # (textual token sequence, special id) pairs to replace. + replacements: list[tuple[list[int], int]] = [] + for text, special_id in markers: + if special_id is None: + continue + textual = mistral_tokenizer.encode(text=text, add_special_tokens=False) + if textual: + replacements.append((textual, special_id)) + + result_tokens: list[int] = [] i = 0 - target_len = len(textual_tool_call_token_ids) - while i < len(tokens): - # Check if the slice from the current position matches the target sequence - if tokens[i : i + target_len] == textual_tool_call_token_ids: - # If it matches, add the replacement and jump the index forward - result_tokens.extend(special_tool_call_token_ids) - i += target_len + for textual, special_id in replacements: + if tokens[i : i + len(textual)] == textual: + result_tokens.append(special_id) + i += len(textual) + break else: - # Otherwise, just add the current token and move to the next one result_tokens.append(tokens[i]) i += 1 return result_tokens +def encode_mistral_output(tokenizer: MistralTokenizer, text: str) -> list[int]: + """Encode generated output with control markers as their special-token ids. + + Real generation emits ``[TOOL_CALLS]``/``[ARGS]`` as single control tokens. + Encoding the marker *strings* can instead glue them to neighbours (e.g. + ``}[``) and yield literal text tokens, which the id-based engine never sees. + Split on the markers and insert their special ids so the token stream + matches real generation. + """ + vocab = tokenizer.get_vocab() + marker_ids = { + "[TOOL_CALLS]": vocab.get("[TOOL_CALLS]"), + "[ARGS]": vocab.get("[ARGS]"), + } + ids: list[int] = [] + for part in re.split(r"(\[TOOL_CALLS\]|\[ARGS\])", text): + if part in marker_ids: + # Fall back to text encoding here and the id-based engine path + # silently stops being exercised while the tests still pass. + assert marker_ids[part] is not None, f"{part} missing from vocab" + ids.append(marker_ids[part]) + elif part: + ids.extend(tokenizer.encode(part, add_special_tokens=False)) + return ids + + def stream_delta_message_generator( mistral_tool_parser: MistralToolParser, mistral_tokenizer: TokenizerLike, model_output: str | None, tools: list[tuple[str, str]] | None, chunk_size: int = 1, + driver: str = "legacy", + reasoning_parser: str | None = None, ) -> Generator[DeltaMessage, None, None]: if ( isinstance(mistral_tokenizer, MistralTokenizer) @@ -179,6 +233,33 @@ def stream_delta_message_generator( all_token_ids, mistral_tool_parser, mistral_tokenizer ) + if isinstance(mistral_tokenizer, MistralTokenizer): + # Real serving streams only generated tokens; encode_instruct adds + # framing (BOS/EOS) the parser never receives. Strip them so the + # id-based engine detection sees a realistic stream. + if all_token_ids and all_token_ids[0] == mistral_tokenizer.bos_token_id: + all_token_ids = all_token_ids[1:] + if all_token_ids and all_token_ids[-1] == mistral_tokenizer.eos_token_id: + all_token_ids = all_token_ids[:-1] + + if driver == "engine": + _parser_cls = ParserManager.get_parser( + tool_parser_name="mistral", + reasoning_parser_name=reasoning_parser, + enable_auto_tools=True, + ) + _engine_parser = _parser_cls(mistral_tokenizer, None) + if reasoning_parser is not None: + _req_with_tools = ChatCompletionRequest( + messages=[], + model="test", + tools=SAMPLE_TOOLS_DICTS, + tool_choice="auto", + ) + _engine_req = _engine_parser.adjust_request(_req_with_tools) + else: + _engine_req = _ENGINE_REQUEST + previous_text = "" previous_tokens = None prefix_offset = 0 @@ -215,15 +296,24 @@ def stream_delta_message_generator( current_token_ids = all_token_ids[: i + 1] current_text = previous_text + pending_text - delta_message = mistral_tool_parser.extract_tool_calls_streaming( - previous_text, - current_text, - pending_text, - previous_token_ids, - current_token_ids, - pending_token_ids, - request=_DUMMY_REQUEST, - ) + if driver == "legacy": + delta_message = mistral_tool_parser.extract_tool_calls_streaming( + previous_text, + current_text, + pending_text, + previous_token_ids, + current_token_ids, + pending_token_ids, + request=_DUMMY_REQUEST, + ) + else: # engine: per-delta windows, mirroring parse_delta behaviour + delta_message = _engine_parser.parse_delta( + delta_text=pending_text, + delta_token_ids=list(pending_token_ids), + request=_engine_req, + prompt_token_ids=[1], + finished=(i == len(all_token_ids) - 1), + ) if delta_message: yield delta_message @@ -231,6 +321,14 @@ def stream_delta_message_generator( pending_text = "" pending_token_ids = [] + # For the legacy driver the engine holds back the final closing brace + # in _args_buffer until finish_streaming() is called, mirroring how + # the real serving layer flushes at end of stream. + if driver == "legacy": + flush_delta = mistral_tool_parser.finish_streaming() + if flush_delta: + yield flush_delta + @pytest.mark.parametrize( "parser_fixture", @@ -424,8 +522,14 @@ def test_extract_tool_calls_pre_v11_regex_fallback_fails( "single_tool_add", "single_tool_weather", "multiple_tool_calls", + "single_tool_add_args", + "single_tool_weather_args", + "multiple_tool_calls_args", + "content_before_tool_args", "complex", "wrong_json", + "trailing_text_after_json", + "trailing_text_after_args_json", ], argnames=["model_output", "expected_tool_calls", "expected_content"], argvalues=[ @@ -471,6 +575,60 @@ def test_extract_tool_calls_pre_v11_regex_fallback_fails( ], None, ), + ( + # v11+ emits an explicit [ARGS] separator between name and args. + """[TOOL_CALLS]add_this_and_that[ARGS]{"a": 3.5, "b": 4}""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add_this_and_that", + arguments=json.dumps({"a": 3.5, "b": 4}), + ) + ) + ], + None, + ), + ( + """[TOOL_CALLS]get_current_weather[ARGS]{"city": "San Francisco", "state": "CA", "unit": "celsius"}""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps( + {"city": "San Francisco", "state": "CA", "unit": "celsius"} + ), + ) + ) + ], + None, + ), + ( + """[TOOL_CALLS]add[ARGS]{"a": 3.5, "b": 4}[TOOL_CALLS]multiply[ARGS]{"a": 3, "b": 6}""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": 3.5, "b": 4}) + ) + ), + ToolCall( + function=FunctionCall( + name="multiply", arguments=json.dumps({"a": 3, "b": 6}) + ) + ), + ], + None, + ), + ( + """hi[TOOL_CALLS]add[ARGS]{"a": 1, "b": 2}""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": 1, "b": 2}) + ) + ) + ], + "hi", + ), ( # Complex """hi{hi[TOOL_CALLS]bash{"command": "print(\\"hello world!\\")\\nre.compile(r\'{}\')""", # noqa: E501 @@ -501,6 +659,33 @@ def test_extract_tool_calls_pre_v11_regex_fallback_fails( ], "hi{hi", ), + ( + # gh#48975: trailing text after the JSON object must not leak into + # arguments (v11+ name{args} format has no terminator). + """[TOOL_CALLS]get_current_weather{"city": "San Francisco"}Estimating the weather...""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps({"city": "San Francisco"}), + ) + ) + ], + None, + ), + ( + # gh#48975 with the explicit [ARGS] separator. + """[TOOL_CALLS]get_current_weather[ARGS]{"city": "San Francisco"}Estimating the weather...""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps({"city": "San Francisco"}), + ) + ) + ], + None, + ), ], ) def test_extract_tool_calls( @@ -517,17 +702,128 @@ def test_extract_tool_calls( def test_extract_tool_calls_v11_without_args_skipped(mistral_tool_parser): + # Before Stage 2: the legacy path skipped tool calls with no arg separator, + # returning tools_called=True with an empty list. After Stage 2 the engine + # path is used for v11; it recognises the name and emits a tool call with + # empty arguments ("{}"). The semantic result is equivalent — a valid zero- + # argument call — and is strictly more correct than silently dropping it. model_output = "[TOOL_CALLS]toolname_no_args" result = mistral_tool_parser.extract_tool_calls( model_output, request=_DUMMY_REQUEST ) - assert result == ExtractedToolCallInformation( - tools_called=True, tool_calls=[], content=None + assert result.tools_called + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].function.name == "toolname_no_args" + assert result.tool_calls[0].function.arguments == "{}" + assert result.content is None + + +def test_extract_tool_calls_malformed_name_before_marker_emits_empty_name( + mistral_tool_parser, +): + model_output = 'bash[TOOL_CALLS]{"command": "cat file.py"}' + result = mistral_tool_parser.extract_tool_calls( + model_output, request=_DUMMY_REQUEST ) + assert result.tools_called + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].function.name == "" + assert result.tool_calls[0].function.arguments == json.dumps( + {"command": "cat file.py"} + ) + assert result.content == "bash" + + +def test_extract_tool_calls_well_formed_name_unaffected(mistral_tool_parser): + model_output = '[TOOL_CALLS]bash[ARGS]{"command": "ls -la"}' + result = mistral_tool_parser.extract_tool_calls( + model_output, request=_DUMMY_REQUEST + ) + assert result.tools_called + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].function.name == "bash" + assert result.tool_calls[0].function.arguments == json.dumps({"command": "ls -la"}) + assert result.content is None + + +def test_extract_tool_calls_empty_name_unparsable_args_no_crash( + mistral_tool_parser, +): + # Empty name AND args that never form valid JSON: still emitted (raw + # text as arguments) instead of raising. + model_output = "bash[TOOL_CALLS]{not valid json at all" + result = mistral_tool_parser.extract_tool_calls( + model_output, request=_DUMMY_REQUEST + ) + assert result.tools_called + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].function.name == "" + assert result.tool_calls[0].function.arguments == "{not valid json at all" + assert result.content == "bash" + + +def test_extract_tool_calls_streaming_malformed_name_before_marker( + mistral_tool_parser, +): + # Streaming counterpart: once the args start the slot can never go back to + # naming, so the (empty) name is already final and streams immediately + # alongside the args. + model_output = 'bash[TOOL_CALLS]{"command": "cat file.py"}' + mid_delta = mistral_tool_parser.extract_tool_calls_streaming( + previous_text="", + current_text=model_output, + delta_text=model_output, + previous_token_ids=[], + current_token_ids=[], + delta_token_ids=[], + request=_DUMMY_REQUEST, + ) + assert mid_delta is not None + assert mid_delta.content == "bash" + assert len(mid_delta.tool_calls) == 1 + assert mid_delta.tool_calls[0].function.name == "" + assert mid_delta.tool_calls[0].function.arguments == '{"command": "cat file.py"' + + final_delta = mistral_tool_parser.finish_streaming() + assert final_delta is not None + assert len(final_delta.tool_calls) == 1 + assert final_delta.tool_calls[0].function.name is None + assert final_delta.tool_calls[0].function.arguments == "}" + + +def test_extract_tool_calls_parallel_malformed_then_well_formed( + mistral_tool_parser, +): + # A malformed empty-name call followed by a well-formed one must not + # regress into brace-splitting: nested braces in the second call's + # arguments must stay intact, and both calls must be correctly separated. + model_output = ( + 'bash[TOOL_CALLS]{"command": "ls"}' + '[TOOL_CALLS]submit[ARGS]{"answer": {"nested": 1}}' + ) + result = mistral_tool_parser.extract_tool_calls( + model_output, request=_DUMMY_REQUEST + ) + assert result.tools_called + assert len(result.tool_calls) == 2 + assert result.tool_calls[0].function.name == "" + assert result.tool_calls[0].function.arguments == json.dumps({"command": "ls"}) + assert result.tool_calls[1].function.name == "submit" + assert result.tool_calls[1].function.arguments == json.dumps( + {"answer": {"nested": 1}} + ) + assert result.content == "bash" def _test_extract_tool_calls_streaming( - tool_parser, tokenizer, model_output, tools, expected_tool_calls, expected_content + tool_parser, + tokenizer, + model_output, + tools, + expected_tool_calls, + expected_content, + driver: str = "legacy", + reasoning_parser: str | None = None, ): other_content: str = "" function_names: list[str] = [] @@ -536,7 +832,12 @@ def _test_extract_tool_calls_streaming( tool_call_ids: list[str | None] = [] for delta_message in stream_delta_message_generator( - tool_parser, tokenizer, model_output, tools + tool_parser, + tokenizer, + model_output, + tools, + driver=driver, + reasoning_parser=reasoning_parser, ): # role should never be streamed from tool parser assert not delta_message.role @@ -547,12 +848,36 @@ def _test_extract_tool_calls_streaming( streamed_tool_calls = delta_message.tool_calls if streamed_tool_calls and len(streamed_tool_calls) > 0: + # On the final finished=True delta the engine's _flush_engine_parsers + # may append the flushed '}' as a separate DeltaToolCall for the + # same index without coalescing. Merge entries sharing an index so + # the "one diff per delta" invariant can be checked after merging. + if len(streamed_tool_calls) > 1: + assert len({tc.index for tc in streamed_tool_calls}) == 1, ( + "only within-chunk finish/flush of one tool is allowed; " + "distinct indices in one delta indicate a regression: " + f"{[tc.index for tc in streamed_tool_calls]}" + ) + by_index: dict[int, DeltaToolCall] = {} + for tc in streamed_tool_calls: + existing = by_index.get(tc.index) + if existing is None: + by_index[tc.index] = tc + else: + if tc.id and not existing.id: + existing.id = tc.id + if tc.type and not existing.type: + existing.type = tc.type + if tc.function and existing.function and tc.function.arguments: + existing.function.arguments = ( + existing.function.arguments or "" + ) + tc.function.arguments + streamed_tool_calls = list(by_index.values()) + # make sure only one diff is present - correct even for parallel assert len(streamed_tool_calls) == 1 tool_call = streamed_tool_calls[0] - assert len(tool_parser.prev_tool_call_arr) > 0 - # if a new tool is being called, set up empty arguments if tool_call.index != tool_call_idx: tool_call_idx = tool_call.index @@ -595,34 +920,8 @@ def _test_extract_tool_calls_streaming( ] assert_tool_calls(actual_tool_calls, expected_tool_calls) - if expected_tool_calls: - assert len(tool_parser.streamed_args_for_tool) == len(expected_tool_calls) - assert len(tool_parser.prev_tool_call_arr) == len(expected_tool_calls) - for i in range(len(expected_tool_calls)): - assert ( - tool_parser.prev_tool_call_arr[i]["arguments"] - == tool_parser.streamed_args_for_tool[i] - ) - assert tool_parser.streamed_args_for_tool[i] == function_args_strs[i] - assert ( - tool_parser.prev_tool_call_arr[i]["name"] - == expected_tool_calls[i].function.name - ) - - # Simulate the serving layer's unstreamed-args check - index = len(tool_parser.prev_tool_call_arr) - 1 - args = tool_parser.prev_tool_call_arr[index].get("arguments", {}) - expected_call = ( - args if isinstance(args, str) else json.dumps(args, ensure_ascii=False) - ) - actual_call = tool_parser.streamed_args_for_tool[index] - remaining_call = expected_call.replace(actual_call, "", 1) - assert remaining_call == "" - else: - assert len(tool_parser.streamed_args_for_tool) == 0 - assert len(tool_parser.prev_tool_call_arr) == 0 - +@pytest.mark.parametrize("driver", ["legacy", "engine"]) @pytest.mark.parametrize( ids=[ "no_tools", @@ -748,6 +1047,7 @@ def test_extract_tool_calls_streaming_pre_v11_tokenizer( model_output, expected_tool_calls, expected_content, + driver, ): _test_extract_tool_calls_streaming( mistral_pre_v11_tool_parser, @@ -756,9 +1056,15 @@ def test_extract_tool_calls_streaming_pre_v11_tokenizer( None, expected_tool_calls, expected_content, + driver=driver, ) +@pytest.mark.parametrize( + "driver,reasoning_parser", + [("legacy", None), ("engine", None), ("engine", "mistral")], + ids=["legacy", "engine_no_reasoning", "engine_with_reasoning"], +) @pytest.mark.parametrize( ids=[ "single_tool_add", @@ -826,6 +1132,8 @@ def test_extract_tool_calls_streaming( tools, expected_tool_calls, expected_content, + driver, + reasoning_parser, ): _test_extract_tool_calls_streaming( mistral_tool_parser, @@ -834,17 +1142,19 @@ def test_extract_tool_calls_streaming( tools, expected_tool_calls, expected_content, + driver=driver, + reasoning_parser=reasoning_parser, ) +# Drives extract_tool_calls_streaming directly (bespoke detokenization loop), +# not via stream_delta_message_generator — driver parametrization not needed. def test_extract_tool_calls_streaming_v11_no_tools( mistral_tool_parser, mistral_tokenizer ): model_output = "This is a test" - if isinstance(mistral_tokenizer, MistralTokenizer): - all_token_ids = mistral_tokenizer.encode(model_output) - else: - all_token_ids = mistral_tokenizer.encode(model_output, add_special_tokens=False) + # add_special_tokens=False: real serving streams only generated tokens. + all_token_ids = mistral_tokenizer.encode(model_output, add_special_tokens=False) skip_special = isinstance(mistral_tokenizer, MistralTokenizer) collected_content = "" previous_text = "" @@ -887,8 +1197,47 @@ def test_extract_tool_calls_streaming_v11_no_tools( previous_text = current_text assert collected_content == model_output - assert len(mistral_tool_parser.streamed_args_for_tool) == 0 - assert len(mistral_tool_parser.prev_tool_call_arr) == 0 + + +def test_mistral_parser_drops_eos_from_output(mistral_tokenizer): + """EOS token must never surface as content/reasoning; literal EOS string + must be preserved when the EOS token id is absent. + + The engine drops special tokens by id, so: + Case A: the real EOS token id causes the text to be dropped. + Case B: the same EOS string with a non-EOS token id is preserved. + """ + if not isinstance(mistral_tokenizer, MistralTokenizer): + pytest.skip("Requires MistralTokenizer") + + eos_id: int = mistral_tokenizer.eos_token_id + eos_text: str = mistral_tokenizer.decode([eos_id]) + assert eos_text + + # Case A: real EOS token — text must be dropped. + parser_a = MistralParser(mistral_tokenizer) + parser_a.initialize_streaming() + events_a = parser_a._feed(eos_text, [eos_id]) + events_a.extend(parser_a._engine.finish()) + delta_a = parser_a._events_to_delta(events_a, finished=True) + content_a = (delta_a.content if delta_a else None) or "" + reasoning_a = (delta_a.reasoning if delta_a else None) or "" + assert eos_text not in content_a + assert eos_text not in reasoning_a + + # Case B: EOS text with a non-EOS token id — text must be preserved. + # This proves content equal to the EOS string is not silently dropped + # when it did not come from the real EOS token. + non_eos_ids = mistral_tokenizer.encode(text="hello", add_special_tokens=False) + non_eos_id = next(tid for tid in non_eos_ids if tid != eos_id) + parser_b = MistralParser(mistral_tokenizer) + parser_b.initialize_streaming() + events_b = parser_b._feed(eos_text, [non_eos_id]) + events_b.extend(parser_b._engine.finish()) + delta_b = parser_b._events_to_delta(events_b, finished=True) + content_b = (delta_b.content if delta_b else None) or "" + reasoning_b = (delta_b.reasoning if delta_b else None) or "" + assert eos_text in content_b or eos_text in reasoning_b @pytest.mark.parametrize( @@ -961,6 +1310,55 @@ def test_extract_tool_calls_streaming_v11_no_tools( "bla", id="v11-content_before_tool", ), + pytest.param( + "mistral_tool_parser", + "mistral_tokenizer", + """[TOOL_CALLS]add_this_and_that[ARGS]{"a": 3.5, "b": 4}""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add_this_and_that", + arguments=json.dumps({"a": 3.5, "b": 4}), + ) + ) + ], + "", + id="v11-single_tool_add_args", + ), + pytest.param( + "mistral_tool_parser", + "mistral_tokenizer", + """[TOOL_CALLS]add[ARGS]{"a": 3.5, "b": 4}[TOOL_CALLS]multiply[ARGS]{"a": 3, "b": 6}""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": 3.5, "b": 4}) + ) + ), + ToolCall( + function=FunctionCall( + name="multiply", arguments=json.dumps({"a": 3, "b": 6}) + ) + ), + ], + "", + id="v11-multiple_tool_calls_args", + ), + pytest.param( + "mistral_tool_parser", + "mistral_tokenizer", + """bla[TOOL_CALLS]add_this_and_that[ARGS]{"a": 3.5, "b": 4}""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="add_this_and_that", + arguments=json.dumps({"a": 3.5, "b": 4}), + ) + ) + ], + "bla", + id="v11-content_before_tool_args", + ), pytest.param( "mistral_tool_parser", "mistral_tokenizer", @@ -1116,11 +1514,15 @@ def test_extract_tool_calls_streaming_one_chunk( tool_parser = request.getfixturevalue(parser_fixture) tokenizer = request.getfixturevalue(tokenizer_fixture) - if isinstance(tokenizer, MistralTokenizer): - all_token_ids = tokenizer.encode(model_output) + # Real serving streams only generated tokens (no BOS/EOS framing) with the + # control markers as special ids. + if isinstance(tokenizer, MistralTokenizer) and tokenizer.version >= 11: + all_token_ids = encode_mistral_output(tokenizer, model_output) else: all_token_ids = tokenizer.encode(model_output, add_special_tokens=False) - all_token_ids = fix_tool_call_tokenization(all_token_ids, tool_parser, tokenizer) + all_token_ids = fix_tool_call_tokenization( + all_token_ids, tool_parser, tokenizer + ) delta_message = tool_parser.extract_tool_calls_streaming( previous_text="", @@ -1131,6 +1533,26 @@ def test_extract_tool_calls_streaming_one_chunk( delta_token_ids=all_token_ids, request=_DUMMY_REQUEST, ) + # The engine buffers the final closing brace ('}') in _args_buffer until + # finish_streaming() is called, mirroring real serving. Flush it and + # merge the result so the asserted arguments are complete. + flush_delta = tool_parser.finish_streaming() + if flush_delta is not None and flush_delta.tool_calls: + if delta_message is not None and delta_message.tool_calls: + for flush_tc in flush_delta.tool_calls: + for existing_tc in delta_message.tool_calls: + if existing_tc.index == flush_tc.index and flush_tc.function: + existing_tc.function = ( + existing_tc.function or DeltaFunctionCall() + ) + existing_tc.function.arguments = ( + existing_tc.function.arguments or "" + ) + (flush_tc.function.arguments or "") + elif delta_message is not None: + delta_message.tool_calls = flush_delta.tool_calls + else: + delta_message = flush_delta + assert isinstance(delta_message, DeltaMessage) assert len(delta_message.tool_calls) == len(expected_tool_calls) @@ -1145,13 +1567,6 @@ def test_extract_tool_calls_streaming_one_chunk( @pytest.mark.parametrize( "parser_fixture, model_output, fake_count, two_phase", [ - pytest.param( - "mistral_tool_parser", - '[TOOL_CALLS]add{"a": 1, "b": 2}', - 20, - True, - id="v11", - ), pytest.param( "mistral_pre_v11_tool_parser", '[TOOL_CALLS] [{"name": "add", "arguments":{"a": 1, "b": 2}}]', @@ -1164,7 +1579,11 @@ def test_extract_tool_calls_streaming_one_chunk( def test_fast_detokenization_text_detection( parser_fixture, model_output, fake_count, two_phase, request ): - """Regression: bot_token in text but not token_ids (PR #37209).""" + """Regression: bot_token in text but not token_ids (PR #37209). + + Only the pre-v11 legacy path detects the marker from text; v11+ routes + through the engine and detects the marker by token id. + """ parser = request.getfixturevalue(parser_fixture) # Token IDs that do NOT contain bot_token_id. fake_token_ids = list(range(99, 99 + fake_count)) @@ -1210,34 +1629,24 @@ def test_fast_detokenization_text_detection( assert delta_message.tool_calls[0].function.name == "add" -@pytest.mark.parametrize( - "parser_fixture, patched_method, current_text", - [ - ( - "mistral_tool_parser", - "_extract_tool_calls_streaming", - "[TOOL_CALLS]add{}", - ), - ( - "mistral_pre_v11_tool_parser", - "_extract_tool_calls_streaming_pre_v11_tokenizer", - '[TOOL_CALLS] [{"name":"a","arguments":{}}]', - ), - ], - ids=["v11", "pre_v11"], -) def test_extract_tool_calls_streaming_exception_returns_none( - parser_fixture, patched_method, current_text, request + mistral_pre_v11_tool_parser, ): - parser = request.getfixturevalue(parser_fixture) - with patch.object(parser, patched_method, side_effect=RuntimeError("boom")): + # v11 routes through the ParserEngine and does not swallow exceptions; + # only the pre-v11 legacy state-machine path has explicit exception handling. + parser = mistral_pre_v11_tool_parser + patched_method = "_extract_tool_calls_streaming_pre_v11_tokenizer" + current_text = '[TOOL_CALLS] [{"name":"a","arguments":{}}]' + with patch.object( + parser._parser_engine, patched_method, side_effect=RuntimeError("boom") + ): result = parser.extract_tool_calls_streaming( previous_text="", current_text=current_text, delta_text=current_text, previous_token_ids=[], - current_token_ids=[parser.bot_token_id], - delta_token_ids=[parser.bot_token_id], + current_token_ids=[parser._parser_engine.bot_token_id], + delta_token_ids=[parser._parser_engine.bot_token_id], request=_DUMMY_REQUEST, ) assert result is None @@ -1368,14 +1777,17 @@ def test_adjust_request_unsupported_grammar_for_tokenizer(mistral_tokenizer) -> @pytest.mark.parametrize( "tool_choice,expected_skip", - [("auto", False), ("none", True)], - ids=["auto_skip_false", "none_skip_true"], + [("auto", False), ("none", False)], + ids=["auto_skip_false", "none_skip_false"], ) def test_adjust_request_non_mistral_tokenizer( non_mistral_parser: MistralToolParser, tool_choice: str, expected_skip: bool, ) -> None: + # MistralParser (ParserEngine) always sets skip_special_tokens=False so + # that special tokens like [TOOL_CALLS] are visible to the parser even + # when tool_choice="none". request = _make_request(tool_choice=tool_choice) result = non_mistral_parser.adjust_request(request) @@ -1572,22 +1984,23 @@ def test_adjust_request_tool_choice_with_json_schema_factory_routing( assert len(result.structured_outputs.grammar) > 0 -def test_grammar_from_tool_parser_default_false() -> None: +def test_grammar_from_parser_default_false() -> None: request = _make_request() - assert request._grammar_from_tool_parser is False + assert request._grammar_from_parser is False -def test_grammar_from_tool_parser_set_by_adjust_request( +def test_grammar_from_parser_set_by_adjust_request( mistral_tool_parser: MistralToolParser, ) -> None: request = _make_request() result = mistral_tool_parser.adjust_request(request) - assert result._grammar_from_tool_parser is True + assert result._grammar_from_parser is True +@pytest.mark.parametrize("driver", ["legacy", "engine"]) @pytest.mark.parametrize("chunk_size", [2, 3, 4, 5]) def test_streaming_pre_v11_parallel_calls_batched_deltas( - mistral_pre_v11_tool_parser, mistral_pre_v11_tokenizer, chunk_size + mistral_pre_v11_tool_parser, mistral_pre_v11_tokenizer, chunk_size, driver ): """A batched delta spanning the boundary between two parallel calls must keep them on distinct indices (the bug collapsed both onto index 0).""" @@ -1605,6 +2018,7 @@ def test_streaming_pre_v11_parallel_calls_batched_deltas( model_output, tools=None, chunk_size=chunk_size, + driver=driver, ): for tool_call in delta_message.tool_calls or []: if tool_call.index != idx: @@ -1619,3 +2033,623 @@ def test_streaming_pre_v11_parallel_calls_batched_deltas( assert len(args) == 2 # trailing args of the final call are flushed by the serving layer assert json.loads(args[0]) == {"a": 3.5, "b": 4} + + +@pytest.mark.parametrize( + "reasoning_encoding", + ["text", "special_token"], + ids=["text_encoding", "special_token_encoding"], +) +def test_content_tool_calls_transition_emits_reasoning_end( + mistral_tokenizer, + reasoning_encoding, +): + """(CONTENT, TOOL_CALLS) must emit REASONING_END when reasoning is enabled.""" + + cfg = mistral_config(reasoning_encoding=reasoning_encoding) + engine = StreamingParserEngine( + config=cfg, tokenizer=mistral_tokenizer, vocab=mistral_tokenizer.get_vocab() + ) + engine.skip_tool_parsing = True + + bot_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]") + assert bot_token_id is not None + + events = engine.feed(delta_text="[TOOL_CALLS]", delta_token_ids=[bot_token_id]) + event_types = [e.type for e in events] + + assert EventType.REASONING_END in event_types, ( + f"REASONING_END missing for reasoning_encoding={reasoning_encoding!r}; " + f"got events: {event_types}" + ) + # Tool call start must NOT appear in skip_tool_parsing mode + assert EventType.TOOL_CALL_START not in event_types + + +@pytest.mark.parametrize("chunk_size", [1, 3]) +@pytest.mark.parametrize( + "tools,expected_tool_calls,expected_content", + [ + ( + [("add", '{"a": 3, "b": 4}')], + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": 3, "b": 4}) + ) + ) + ], + "", + ), + ( + [ + ("add", '{"a": 3.5, "b": 4}'), + ( + "get_current_weather", + '{"city": "San Francisco", "state": "CA", "unit": "celsius"}', + ), + ], + [ + ToolCall( + function=FunctionCall( + name="add", arguments=json.dumps({"a": 3.5, "b": 4}) + ) + ), + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps( + { + "city": "San Francisco", + "state": "CA", + "unit": "celsius", + } + ), + ) + ), + ], + "", + ), + ], + ids=["single_tool_add", "parallel_tools"], +) +def test_reasoning_active_no_think_block_no_leak( + mistral_tool_parser, + mistral_tokenizer, + tools, + expected_tool_calls, + expected_content, + chunk_size, +): + """Tool call without a preceding think block must not leak as content + when the reasoning parser is active.""" + accumulated_content = "" + function_names: list[str] = [] + function_args_strs: list[str] = [] + tool_call_idx = -1 + tool_call_ids: list[str | None] = [] + + for delta_message in stream_delta_message_generator( + mistral_tool_parser, + mistral_tokenizer, + model_output=None, + tools=tools, + chunk_size=chunk_size, + driver="engine", + reasoning_parser="mistral", + ): + if delta_message.content: + accumulated_content += delta_message.content + + for tool_call in delta_message.tool_calls or []: + if tool_call.index != tool_call_idx: + tool_call_idx = tool_call.index + function_args_strs.append("") + tool_call_ids.append(None) + if tool_call.id and not tool_call_ids[tool_call.index]: + tool_call_ids[tool_call.index] = tool_call.id + if tool_call.function: + if tool_call.function.name: + function_names.append(tool_call.function.name) + if tool_call.function.arguments: + function_args_strs[tool_call.index] += tool_call.function.arguments + + assert "[TOOL_CALLS]" not in accumulated_content + assert "[ARGS]" not in accumulated_content + assert accumulated_content == expected_content + + actual_tool_calls = [ + ToolCall( + id=tc_id, + function=FunctionCall( + name=name, + arguments=partial_json_parser.ensure_json(args, Allow.OBJ | Allow.STR), + ), + ) + for tc_id, name, args in zip(tool_call_ids, function_names, function_args_strs) + ] + assert_tool_calls(actual_tool_calls, expected_tool_calls) + + +# --------------------------------------------------------------------------- +# Pre-v11 guided schema injection and bare-array extraction tests +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "tool_choice,expected_items_count", + [ + ("required", 2), # all tools + ( + {"type": "function", "function": {"name": "get_weather"}}, + 1, + ), # named: single tool + ], + ids=["required", "named"], +) +def test_adjust_request_pre_v11_guided_schema_injected( + mistral_pre_v11_tool_parser: MistralToolParser, + tool_choice: object, + expected_items_count: int, +) -> None: + request = _make_request(tool_choice=tool_choice) + result = mistral_pre_v11_tool_parser.adjust_request(request) + + assert result.structured_outputs is not None + schema = result.structured_outputs.json + if isinstance(schema, str): + schema = json.loads(schema) + assert schema["type"] == "array" + assert schema["minItems"] == 1 + items = schema["items"] + assert "anyOf" in items + assert len(items["anyOf"]) == expected_items_count + for entry in items["anyOf"]: + props = entry["properties"] + assert "name" in props + assert "arguments" in props + + +@pytest.mark.parametrize( + "response_format", + [{"type": "json_object"}, {"type": "json_schema", "json_schema": {"name": "s"}}], + ids=["json_object", "json_schema"], +) +def test_adjust_request_pre_v11_required_clears_response_format( + mistral_pre_v11_tool_parser: MistralToolParser, + response_format: dict, +) -> None: + """required + response_format must inject the tool schema and clear + response_format so the tool schema is the sole structured-output + constraint, matching the base ToolParser (otherwise the request either + hits the "multiple constraints" engine error or, with no constraint, + rambles to finish_reason='length').""" + request = _make_request(tool_choice="required", response_format=response_format) + result = mistral_pre_v11_tool_parser.adjust_request(request) + + assert result.response_format is None + assert result.structured_outputs is not None + schema = result.structured_outputs.json + if isinstance(schema, str): + schema = json.loads(schema) + assert schema["type"] == "array" + + +def test_adjust_request_non_mistral_tokenizer_required_injects_schema( + non_mistral_parser: MistralToolParser, +) -> None: + """The guided-schema injection must also fire for non-Mistral (e.g. HF-mode) + tokenizers driving the Mistral tool parser, mirroring the base ToolParser so + required + response_format does not fall back to an unconstrained ramble.""" + request = _make_request( + tool_choice="required", response_format={"type": "json_object"} + ) + result = non_mistral_parser.adjust_request(request) + + assert result.response_format is None + assert result.structured_outputs is not None + assert result.structured_outputs.json is not None + + +@pytest.mark.parametrize( + "tool_choice", + ["auto", "none"], + ids=["auto", "none"], +) +def test_adjust_request_pre_v11_no_injection_for_auto_none( + mistral_pre_v11_tool_parser: MistralToolParser, + tool_choice: str, +) -> None: + request = _make_request(tool_choice=tool_choice) + result = mistral_pre_v11_tool_parser.adjust_request(request) + + assert result.structured_outputs is None + + +def test_legacy_extract_tool_calls_guided_bare_array_required( + mistral_pre_v11_tool_parser: MistralToolParser, +) -> None: + model_output = '[{"name": "get_current_weather", "arguments": {"city": "Dallas"}}]' + request = _make_request(tool_choice="required") + + result = mistral_pre_v11_tool_parser.extract_tool_calls( + model_output, request=request + ) + + assert result.tools_called + assert len(result.tool_calls) == 1 + tc = result.tool_calls[0] + assert tc.function.name == "get_current_weather" + assert json.loads(tc.function.arguments) == {"city": "Dallas"} + assert isinstance(tc.id, str) + assert len(tc.id) == 9 + assert result.content is None + + +def test_legacy_extract_tool_calls_none_with_tools( + mistral_pre_v11_tool_parser: MistralToolParser, +) -> None: + model_output = ( + '[TOOL_CALLS] [{"name": "get_current_weather",' + ' "arguments": {"city": "Dallas"}}]' + ) + request = _make_request(tool_choice="none") + + result = mistral_pre_v11_tool_parser.extract_tool_calls( + model_output, request=request + ) + + assert not result.tools_called + assert result.tool_calls == [] + assert result.content == model_output + + +def test_guided_streaming_required_pre_v11( + mistral_pre_v11_tool_parser: MistralToolParser, + mistral_pre_v11_tokenizer, +) -> None: + model_output = '[{"name": "get_current_weather", "arguments": {"city": "Dallas"}}]' + request = _make_request(tool_choice="required") + all_token_ids = mistral_pre_v11_tokenizer.encode( + model_output, add_special_tokens=False + ) + + function_name: str | None = None + function_args = "" + tool_call_id: str | None = None + previous_text = "" + previous_tokens = None + prefix_offset = 0 + read_offset = 0 + + for i, token_id in enumerate(all_token_ids): + (new_tokens, delta_text, prefix_offset, read_offset) = detokenize_incrementally( + tokenizer=mistral_pre_v11_tokenizer, + all_input_ids=all_token_ids[: i + 1], + prev_tokens=previous_tokens, + prefix_offset=prefix_offset, + read_offset=read_offset, + skip_special_tokens=False, + spaces_between_special_tokens=True, + ) + previous_tokens = ( + (previous_tokens + new_tokens) if previous_tokens else new_tokens + ) + current_text = previous_text + delta_text + + delta_message = mistral_pre_v11_tool_parser.extract_tool_calls_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=delta_text, + previous_token_ids=all_token_ids[:i], + current_token_ids=all_token_ids[: i + 1], + delta_token_ids=[token_id], + request=request, + ) + previous_text = current_text + + if delta_message and delta_message.tool_calls: + for tc in delta_message.tool_calls: + if tc.id and not tool_call_id: + tool_call_id = tc.id + if tc.function: + if tc.function.name: + function_name = tc.function.name + if tc.function.arguments: + function_args += tc.function.arguments + + assert function_name == "get_current_weather" + assert tool_call_id is not None + assert len(tool_call_id) == 9 + assert json.loads(function_args) == {"city": "Dallas"} + + +# --------------------------------------------------------------------------- +# Malformed-name-before-marker matrix: {malformed, well-formed, mixed +# parallel} x {streaming, non-streaming}. +# +# These tests drive a real MistralParser built from the module's +# `mistral_tokenizer` fixture (grammar-capable v11+ tokenizer, so the +# declarative ParserEngine path is exercised, not the pre-v11 legacy state +# machine) with actual bash/search_replace/submit tools, mirroring the tools +# present in the real captured traces this bug was found in. +# --------------------------------------------------------------------------- + +_BASH_SEARCH_SUBMIT_TOOLS_DICTS = [ + { + "type": "function", + "function": { + "name": "bash", + "description": "Run a bash command", + "parameters": { + "type": "object", + "properties": {"command": {"type": "string"}}, + "required": ["command"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "search_replace", + "description": "Search and replace text in a file", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "search": {"type": "string"}, + "replace": {"type": "string"}, + }, + "required": ["path", "search", "replace"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "submit", + "description": "Submit the final answer", + "parameters": { + "type": "object", + "properties": {"answer": {"type": "object"}}, + "required": ["answer"], + }, + }, + }, +] + + +def _malformed_matrix_request() -> ChatCompletionRequest: + return ChatCompletionRequest( + messages=[], + model="test", + tools=_BASH_SEARCH_SUBMIT_TOOLS_DICTS, + tool_choice="auto", + ) + + +def _stream_via_parse_delta( + parser: MistralParser, + tokenizer: MistralTokenizer, + model_output: str, + request: ChatCompletionRequest, +) -> tuple[str, list[str | None], list[str]]: + """Feed `model_output` through `parser.parse_delta` token-by-token. + + This is the exact API production serving drives on every generated + token (see `ParserEngine.parse_delta`), unlike + `extract_tool_calls_streaming` which some legacy call sites drive with + pre-computed previous/current text windows. + """ + parser.initialize_streaming() + all_token_ids = encode_mistral_output(tokenizer, model_output) + + content = "" + tool_call_idx = -1 + names: list[str | None] = [] + args: list[str] = [] + + previous_tokens = None + prefix_offset = 0 + read_offset = 0 + for i, token_id in enumerate(all_token_ids): + new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally( + tokenizer=tokenizer, + all_input_ids=all_token_ids[: i + 1], + prev_tokens=previous_tokens, + prefix_offset=prefix_offset, + read_offset=read_offset, + skip_special_tokens=False, + spaces_between_special_tokens=True, + ) + previous_tokens = ( + previous_tokens + new_tokens if previous_tokens else new_tokens + ) + finished = i == len(all_token_ids) - 1 + + delta_message = parser.parse_delta( + delta_text=delta_text, + delta_token_ids=[token_id], + request=request, + prompt_token_ids=[1], + finished=finished, + ) + if delta_message is None: + continue + if delta_message.content: + content += delta_message.content + for tool_call in delta_message.tool_calls or []: + if tool_call.index != tool_call_idx: + tool_call_idx = tool_call.index + names.append(None) + args.append("") + if tool_call.function: + if tool_call.function.name is not None: + names[tool_call.index] = tool_call.function.name + if tool_call.function.arguments: + args[tool_call.index] += tool_call.function.arguments + + return content, names, args + + +_MALFORMED_MATRIX_CASES = [ + pytest.param( + # 2x2 corner: name present, `[ARGS]` token present. + '[TOOL_CALLS]bash[ARGS]{"command": "ls -la"}', + ["bash"], + [json.dumps({"command": "ls -la"})], + None, + id="wellformed_single", + ), + pytest.param( + # 2x2 corner: name present, no `[ARGS]` token. + '[TOOL_CALLS]bash{"command": "ls -la"}', + ["bash"], + [json.dumps({"command": "ls -la"})], + None, + id="wellformed_single_no_args_token", + ), + pytest.param( + # 2x2 corner: name skipped, no `[ARGS]` token. "bash" lands in + # content before the marker, then generation jumps straight to + # args (no NAME slot, no [ARGS]). + 'bash[TOOL_CALLS]{"command": "cat file.py"}', + [""], + [json.dumps({"command": "cat file.py"})], + "bash", + id="malformed_single", + ), + pytest.param( + # 2x2 corner: name skipped, `[ARGS]` token present. + 'bash[TOOL_CALLS][ARGS]{"command": "cat file.py"}', + [""], + [json.dumps({"command": "cat file.py"})], + "bash", + id="malformed_single_args_token", + ), + pytest.param( + # parallel well-formed x2; the second call's nested braces in its + # arguments must not confuse call splitting. + '[TOOL_CALLS]bash[ARGS]{"command": "ls"}' + '[TOOL_CALLS]submit[ARGS]{"answer": {"nested": 1}}', + ["bash", "submit"], + [json.dumps({"command": "ls"}), json.dumps({"answer": {"nested": 1}})], + None, + id="parallel_wellformed", + ), + pytest.param( + # parallel malformed x2: both calls skip the NAME slot entirely + # (marker immediately followed by `{`), each ending with an empty + # name and correctly separated (nested-brace) arguments. + 'bash[TOOL_CALLS]{"command": "ls"}[TOOL_CALLS]{"answer": {"nested": 1}}', + ["", ""], + [json.dumps({"command": "ls"}), json.dumps({"answer": {"nested": 1}})], + "bash", + id="parallel_malformed", + ), + pytest.param( + # parallel malformed x2, `[ARGS]` token variant: both calls skip + # the NAME slot but enter TOOL_ARGS via `[ARGS]` instead of `{`. + 'bash[TOOL_CALLS][ARGS]{"command": "ls"}' + '[TOOL_CALLS][ARGS]{"answer": {"nested": 1}}', + ["", ""], + [json.dumps({"command": "ls"}), json.dumps({"answer": {"nested": 1}})], + "bash", + id="parallel_malformed_args_token", + ), + pytest.param( + # mixed: malformed first call, well-formed second call. The + # important case: proves calls are split on the `[TOOL_CALLS]` + # marker, not on braces — the first call's simple args and the + # second call's nested-brace args must not bleed into each other. + 'bash[TOOL_CALLS]{"command": "ls"}' + '[TOOL_CALLS]submit[ARGS]{"answer": {"nested": 1}}', + ["", "submit"], + [json.dumps({"command": "ls"}), json.dumps({"answer": {"nested": 1}})], + "bash", + id="mixed_malformed_then_wellformed", + ), + pytest.param( + # mixed, `[ARGS]` token variant: malformed first call enters + # TOOL_ARGS via `[ARGS]` instead of `{`, second call well-formed. + 'bash[TOOL_CALLS][ARGS]{"command": "ls"}' + '[TOOL_CALLS]submit[ARGS]{"answer": {"nested": 1}}', + ["", "submit"], + [json.dumps({"command": "ls"}), json.dumps({"answer": {"nested": 1}})], + "bash", + id="mixed_malformed_args_token_then_wellformed", + ), + pytest.param( + # mixed: well-formed first call, malformed second call (NAME slot + # skipped entirely for the second call). + '[TOOL_CALLS]bash[ARGS]{"command": "ls"}[TOOL_CALLS]{"answer": {"nested": 1}}', + ["bash", ""], + [json.dumps({"command": "ls"}), json.dumps({"answer": {"nested": 1}})], + None, + id="mixed_wellformed_then_malformed", + ), +] + + +@pytest.mark.parametrize( + "model_output,expected_names,expected_args,expected_content", + _MALFORMED_MATRIX_CASES, +) +def test_malformed_tool_call_matrix_non_streaming( + mistral_tokenizer, + model_output, + expected_names, + expected_args, + expected_content, +): + parser = MistralParser(mistral_tokenizer) + request = _malformed_matrix_request() + + result = parser.extract_tool_calls_from_content(model_output, request) + + assert result.tools_called + assert [tc.function.name for tc in result.tool_calls] == expected_names + assert [tc.function.arguments for tc in result.tool_calls] == expected_args + assert result.content == expected_content + + +@pytest.mark.parametrize( + "model_output,expected_names,expected_args,expected_content", + _MALFORMED_MATRIX_CASES, +) +def test_malformed_tool_call_matrix_streaming( + mistral_tokenizer, + model_output, + expected_names, + expected_args, + expected_content, +): + parser = MistralParser(mistral_tokenizer) + request = _malformed_matrix_request() + + content, names, args = _stream_via_parse_delta( + parser, mistral_tokenizer, model_output, request + ) + + assert content == (expected_content or "") + assert names == expected_names + assert args == expected_args + + +# Real payload captured from a production SWE-bench-agent. +_REAL_LEAK_CONTENT = 'bash[TOOL_CALLS]{"command": "cd /testbed && git diff xarray/core/weighted.py"}ometracediff --git a/xarray/core/weighted.py b/xarray/core/weighted.py\\nindex 5f2d138e..9c38b0b7 100644\\n--- a/xarray/core/weighted.py\\n+++ b/xarray/core/weighted.py\\n@@ -1,4 +1,6 @@\\n from typing import TYPE_CHECKING, Hashable, Iterable, Optional, Union, overload\\n+\\n+import numpy as np\\n \\n from .computation import dot\\n from .options import _get_keep_attrs\\n@@ -105,6 +107,10 @@ class Weighted:\\n )\\n \\n self.obj = obj\\n- self.weights = weights\\n+ # Ensure weights are numeric (float64) to avoid boolean arithmetic issues\\n+ # in dot products and weighted operations\\n+ if not np.issubdtype(weights.dtype, np.number):\\n+ weights = weights.astype(np.float64)\\n+ self.weights = weights\\n "}现在 hareview the changes we made:bash[ARGS]{"command": "cat /testbed/xarray/core/weighted.py | head -120 | tail -20"}' # noqa: E501 + + +def test_extract_tool_calls_real_captured_malformed_output(mistral_tokenizer): + parser = MistralParser(mistral_tokenizer) + request = _malformed_matrix_request() + + result = parser.extract_tool_calls_from_content(_REAL_LEAK_CONTENT, request) + + assert result.tools_called + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].function.name == "" + assert result.tool_calls[0].function.arguments == json.dumps( + {"command": "cd /testbed && git diff xarray/core/weighted.py"} + ) + assert result.content == "bash" diff --git a/tests/reasoning/test_mistral_reasoning_parser.py b/tests/reasoning/test_mistral_reasoning_parser.py deleted file mode 100644 index d6da723f80..0000000000 --- a/tests/reasoning/test_mistral_reasoning_parser.py +++ /dev/null @@ -1,348 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest - -from tests.reasoning.utils import run_reasoning_extraction_mistral -from vllm.reasoning import ReasoningParser, ReasoningParserManager -from vllm.tokenizers.mistral import MistralTokenizer - -parser_name = "mistral" - - -@pytest.fixture(scope="module") -def mistral_tokenizer(): - mistral_tokenizer = MistralTokenizer.from_pretrained( - "mistralai/Magistral-Small-2509" - ) - return mistral_tokenizer - - -INVALID_SIMPLE_REASONING = { - "output": "This is a reasoning section[/THINK]This is the rest", - "reasoning": None, - "content": "This is a reasoning sectionThis is the rest", - "is_reasoning_end": False, -} -INVALID_COMPLETE_REASONING = { - "output": "This is a reasoning section[/THINK]", - "reasoning": None, - "content": "This is a reasoning section", - "is_reasoning_end": False, -} -NO_CONTENT = { - "output": "[THINK]This is reasoning", - "reasoning": "This is reasoning", - "content": None, - "is_reasoning_end": False, -} -NO_REASONING = { - "output": "This is content", - "reasoning": None, - "content": "This is content", - "is_reasoning_end": False, -} -NO_REASONING_STREAMING = { - "output": "This is a reasoning section", - "reasoning": None, - "content": "This is a reasoning section", - "is_reasoning_end": False, -} -INVALID_MULTIPLE_LINES = { - "output": "This\nThat[/THINK]This is the rest\nThat", - "reasoning": None, - "content": "This\nThatThis is the rest\nThat", - "is_reasoning_end": False, -} -INVALID_SHORTEST_REASONING_NO_STREAMING = { - "output": "[/THINK]This is the rest", - "reasoning": None, - "content": "This is the rest", - "is_reasoning_end": False, -} -INVALID_SHORTEST_REASONING = { - "output": "[/THINK]This is the rest", - "reasoning": None, - "content": "This is the rest", - "is_reasoning_end": False, -} -REASONING_WITH_THINK = { - "output": "[THINK]This is a reasoning section[/THINK]This is the rest", - "reasoning": "This is a reasoning section", - "content": "This is the rest", - "is_reasoning_end": True, -} -COMPLETE_REASONING_WITH_THINK = { - "output": "[THINK]This is a reasoning section[/THINK]", - "reasoning": "This is a reasoning section", - "content": None, - "is_reasoning_end": True, -} -MULTIPLE_LINES_WITH_THINK = { - "output": "[THINK]This\nThat[/THINK]This is the rest\nThat", - "reasoning": "This\nThat", - "content": "This is the rest\nThat", - "is_reasoning_end": True, -} -INVALID_SHORTEST_REASONING_NO_STREAMING_WITH_THINK = { - "output": "[/THINK]This is the rest", - "reasoning": None, - "content": "This is the rest", - "is_reasoning_end": False, -} -INVALID_SHORTEST_REASONING_WITH_THINK = { - "output": "[/THINK]This is the rest", - "reasoning": None, - "content": "This is the rest", - "is_reasoning_end": False, -} -THINK_NO_END = { - "output": "[THINK]This is a reasoning section", - "reasoning": "This is a reasoning section", - "content": None, - "is_reasoning_end": False, -} -EMPTY = { - "output": "", - "reasoning": None, - "content": "", - "is_reasoning_end": False, -} -EMPTY_STREAMING = { - "output": "", - "reasoning": None, - "content": None, - "is_reasoning_end": False, -} -NEW_LINE = { - "output": "Before\n[THINK]This is a reasoning section[/THINK]\nThis is the rest", - "reasoning": "This is a reasoning section", - "content": "Before\n\nThis is the rest", - "is_reasoning_end": True, -} -NEW_LINE_STREAMING = { - "output": "Before\n[THINK]This is a reasoning section[/THINK]\nThis is the rest", - "reasoning": "This is a reasoning section", - "content": "Before\n\nThis is the rest", - "is_reasoning_end": True, -} - -TEST_CASES = [ - pytest.param( - False, - INVALID_SIMPLE_REASONING, - id="invalid_simple_reasoning", - ), - pytest.param( - True, - INVALID_SIMPLE_REASONING, - id="invalid_simple_reasoning_streaming", - ), - pytest.param( - False, - INVALID_COMPLETE_REASONING, - id="invalid_complete_reasoning", - ), - pytest.param( - True, - INVALID_COMPLETE_REASONING, - id="invalid_complete_reasoning_streaming", - ), - pytest.param( - False, - NO_CONTENT, - id="no_content", - ), - pytest.param( - False, - NO_REASONING, - id="no_reasoning", - ), - pytest.param( - True, - NO_REASONING_STREAMING, - id="no_reasoning_token_streaming", - ), - pytest.param( - False, - INVALID_MULTIPLE_LINES, - id="invalid_multiple_lines", - ), - pytest.param( - True, - INVALID_MULTIPLE_LINES, - id="invalid_multiple_lines_streaming", - ), - pytest.param( - True, - INVALID_SHORTEST_REASONING, - id="invalid_shortest", - ), - pytest.param( - False, - INVALID_SHORTEST_REASONING_NO_STREAMING, - id="invalid_shortest_streaming", - ), - pytest.param( - False, - REASONING_WITH_THINK, - id="reasoning_with_think", - ), - pytest.param( - True, - REASONING_WITH_THINK, - id="reasoning_with_think_streaming", - ), - pytest.param( - False, - COMPLETE_REASONING_WITH_THINK, - id="complete_reasoning_with_think", - ), - pytest.param( - True, - COMPLETE_REASONING_WITH_THINK, - id="complete_reasoning_with_think_streaming", - ), - pytest.param( - False, - MULTIPLE_LINES_WITH_THINK, - id="multiple_lines_with_think", - ), - pytest.param( - True, - MULTIPLE_LINES_WITH_THINK, - id="multiple_lines_with_think_streaming", - ), - pytest.param( - False, - INVALID_SHORTEST_REASONING_NO_STREAMING_WITH_THINK, - id="invalid_shortest_with_think", - ), - pytest.param( - True, - INVALID_SHORTEST_REASONING_WITH_THINK, - id="invalid_shortest_with_think_streaming", - ), - pytest.param( - False, - THINK_NO_END, - id="think_no_end", - ), - pytest.param( - True, - THINK_NO_END, - id="think_no_end_streaming", - ), - pytest.param( - False, - EMPTY, - id="empty", - ), - pytest.param( - True, - EMPTY_STREAMING, - id="empty_streaming", - ), - pytest.param( - False, - NEW_LINE, - id="new_line", - ), - pytest.param( - True, - NEW_LINE_STREAMING, - id="new_line_streaming", - ), -] - - -@pytest.mark.parametrize("streaming, param_dict", TEST_CASES) -def test_mistral_reasoning( - streaming: bool, - param_dict: dict, - mistral_tokenizer: MistralTokenizer, -): - output = param_dict["output"] - - index_think = output.find("[THINK]") - len_think = len("[THINK]") - index_end_think = output.find("[/THINK]") - len_end_think = len("[/THINK]") - - # encode everything to tokens ids - output_tokens = [] - if index_think != -1: - output_before_think = output[:index_think] - output_tokens += mistral_tokenizer.tokenizer.encode( - output_before_think, False, False - ) - output_tokens += [mistral_tokenizer.instruct.BEGIN_THINK] - - if index_end_think != -1: - output_middle = output[index_think + len_think : index_end_think] - output_after_think = output[index_end_think + len_end_think :] - output_tokens += mistral_tokenizer.tokenizer.encode( - output_middle, False, False - ) - output_tokens += [mistral_tokenizer.instruct.END_THINK] - output_tokens += mistral_tokenizer.tokenizer.encode( - output_after_think, False, False - ) - else: - output_middle = output[index_think + len_think :] - output_tokens += mistral_tokenizer.tokenizer.encode( - output_middle, False, False - ) - elif index_end_think != -1: - output_before_think = output[:index_end_think] - output_after_think = output[index_end_think + len_end_think :] - output_tokens += mistral_tokenizer.tokenizer.encode( - output_before_think, False, False - ) - output_tokens += [mistral_tokenizer.instruct.END_THINK] - output_tokens += mistral_tokenizer.tokenizer.encode( - output_after_think, False, False - ) - else: - output_tokens += mistral_tokenizer.tokenizer.encode(output, False, False) - - parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)( - mistral_tokenizer - ) - - reasoning, content = run_reasoning_extraction_mistral( - parser, output_tokens, streaming=streaming - ) - - assert reasoning == param_dict["reasoning"] - assert content == param_dict["content"] - - # Test is_reasoning_end - is_reasoning_end = parser.is_reasoning_end(output_tokens) - assert is_reasoning_end == param_dict["is_reasoning_end"] - - # Test extract_content - if param_dict["content"] is not None: - # Handle the case where there are tokens outputted before Thinking. - # This should not occur if the model is well trained and prompted. - if "[THINK]" in param_dict["output"] and not param_dict["output"].startswith( - "[THINK]" - ): - before_content = param_dict["output"].split("[THINK]")[0] - before_token_ids = mistral_tokenizer.tokenizer.encode( - before_content, bos=False, eos=False - ) - left_to_encode = param_dict["content"][len(before_content) :] - # Normal situation. - else: - before_token_ids = [] - left_to_encode = param_dict["content"] - - content_tokens = parser.extract_content_ids(output_tokens) - expected_token_ids = before_token_ids + mistral_tokenizer.tokenizer.encode( - left_to_encode, bos=False, eos=False - ) - assert content_tokens == expected_token_ids - else: - content = parser.extract_content_ids(output_tokens) - assert content == [] diff --git a/tests/tokenizers_/test_mistral.py b/tests/tokenizers_/test_mistral.py index 47abbd8128..fe06f6b328 100644 --- a/tests/tokenizers_/test_mistral.py +++ b/tests/tokenizers_/test_mistral.py @@ -8,11 +8,13 @@ import llguidance import pytest from mistral_common.exceptions import InvalidMessageStructureException from mistral_common.guidance.grammar_factory import GrammarFactory -from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy +from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy, SpecialTokens +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.tokenizers.mistral import ( MistralTokenizer, _validate_apply_chat_template_args, + validate_request_params, ) @@ -1247,7 +1249,9 @@ class TestMistralTokenizer: expected_strings = ( '[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}] I am an AI\n\nHello world ![TOOL_CALLS][{"name": "get_weather", "arguments": {"city": "Paris"}, "id": "123456789"}] {"content": {"temperature": 20, "unit": "celsius"}, "call_id": "123456789"}', # noqa: E501 - 'I am an AI[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}]Hello world ![TOOL_CALLS]get_weather{"city": "Paris"}{"temperature": 20, "unit": "celsius"}', # noqa: E501 + # v11+ tool-call decode emits the explicit [ARGS] separator between + # the function name and its JSON arguments (get_weather[ARGS]{...}). + 'I am an AI[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}]Hello world ![TOOL_CALLS]get_weather[ARGS]{"city": "Paris"}{"temperature": 20, "unit": "celsius"}', # noqa: E501 ) assert ( @@ -1498,6 +1502,7 @@ class TestMistralTokenizer: "get", "_", "weather", + "[ARGS]", '{"', "city", '":', @@ -2234,3 +2239,140 @@ class TestMistralTokenizer: decoded = mistral_tokenizer.tokenizer.decode(output, SpecialTokenPolicy.KEEP) assert "[THINK]2+2 equals 4[/THINK]" in decoded + + +def test_convert_ids_to_tokens_pre_args_tekken(): + """convert_ids_to_tokens must not raise on pre-[ARGS] Tekken tokenizers. + + Tekken tokenizers before v11 (e.g. Ministral-8B-Instruct-2410) have no + [ARGS] special token. This guards the is_special([ARGS]) gate in + MistralTokenizer.convert_ids_to_tokens, which must skip [ARGS] rather than + call get_special_token(args) unconditionally (which raises on Tekken). + """ + tokenizer = MistralTokenizer.from_pretrained("mistralai/Ministral-8B-Instruct-2410") + assert tokenizer.is_tekken + assert not tokenizer.tokenizer.is_special(SpecialTokens.args) + + ids = tokenizer.encode("Hello world !", add_special_tokens=False) + tokens = tokenizer.convert_ids_to_tokens(ids, skip_special_tokens=True) + assert len(tokens) > 0 + + +# --------------------------------------------------------------------------- +# validate_request_params – reasoning_effort validation (no tokenizer needed) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("reasoning_effort", [None, "none", "high"]) +def test_validate_request_params_valid_effort(reasoning_effort: str | None) -> None: + request = ChatCompletionRequest( + model="test-model", + messages=[], + reasoning_effort=reasoning_effort, + ) + assert validate_request_params(request) is None + + +def test_validate_request_params_rejects_unsupported_effort() -> None: + request = ChatCompletionRequest( + model="test-model", + messages=[], + reasoning_effort="medium", + ) + with pytest.raises(ValueError, match="reasoning_effort"): + validate_request_params(request) + + +# --------------------------------------------------------------------------- +# apply_chat_template reasoning_effort passthrough (v15 tokenizer) +# --------------------------------------------------------------------------- + +_PASSTHROUGH_MESSAGES = [{"role": "user", "content": "Hello"}] + + +@pytest.fixture(scope="module") +def v15_mistral_tokenizer() -> MistralTokenizer: + """Load the v15 Mistral tokenizer; skip if unavailable.""" + try: + return MistralTokenizer.from_pretrained("mistralai/Mistral-Small-4-119B-2603") + except Exception: + pytest.skip("v15 tokenizer unavailable") + + +@pytest.fixture(scope="module") +def v13_mistral_tokenizer() -> MistralTokenizer: + """Load the v13 Mistral tokenizer; skip if unavailable.""" + try: + return MistralTokenizer.from_pretrained("mistralai/Magistral-Small-2509") + except Exception: + pytest.skip("v13 tokenizer unavailable") + + +def test_v15_apply_chat_template_passes_reasoning_effort_high( + monkeypatch: pytest.MonkeyPatch, + v15_mistral_tokenizer: MistralTokenizer, +) -> None: + captured_kwargs: list[dict] = [] + + def fake_apply_chat_template(**kwargs): + captured_kwargs.append(kwargs) + return [0] + + monkeypatch.setattr( + v15_mistral_tokenizer.transformers_tokenizer, + "apply_chat_template", + fake_apply_chat_template, + ) + + v15_mistral_tokenizer.apply_chat_template( + messages=_PASSTHROUGH_MESSAGES, + reasoning_effort="high", + ) + + assert captured_kwargs[-1]["reasoning_effort"] == "high" + + +def test_v15_apply_chat_template_passes_reasoning_effort_none_by_default( + monkeypatch: pytest.MonkeyPatch, + v15_mistral_tokenizer: MistralTokenizer, +) -> None: + captured_kwargs: list[dict] = [] + + def fake_apply_chat_template(**kwargs): + captured_kwargs.append(kwargs) + return [0] + + monkeypatch.setattr( + v15_mistral_tokenizer.transformers_tokenizer, + "apply_chat_template", + fake_apply_chat_template, + ) + + v15_mistral_tokenizer.apply_chat_template(messages=_PASSTHROUGH_MESSAGES) + + assert "reasoning_effort" in captured_kwargs[-1] + assert captured_kwargs[-1]["reasoning_effort"] is None + + +def test_pre_v15_apply_chat_template_omits_reasoning_effort( + monkeypatch: pytest.MonkeyPatch, + v13_mistral_tokenizer: MistralTokenizer, +) -> None: + captured_kwargs: list[dict] = [] + + def fake_apply_chat_template(**kwargs): + captured_kwargs.append(kwargs) + return [0] + + monkeypatch.setattr( + v13_mistral_tokenizer.transformers_tokenizer, + "apply_chat_template", + fake_apply_chat_template, + ) + + v13_mistral_tokenizer.apply_chat_template( + messages=_PASSTHROUGH_MESSAGES, + reasoning_effort="high", + ) + + assert "reasoning_effort" not in captured_kwargs[-1] diff --git a/tests/tool_use/mistral/test_mistral_tool_calls.py b/tests/tool_use/mistral/test_mistral_tool_calls.py index 6dcfd43a94..c7e29833fa 100644 --- a/tests/tool_use/mistral/test_mistral_tool_calls.py +++ b/tests/tool_use/mistral/test_mistral_tool_calls.py @@ -219,7 +219,7 @@ _NOT_SET = object() pytest.param( [WEATHER_TOOL], "required", - 30, + 9, id="required", ), ], diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index ab04a6d27f..c59c6c9e93 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -533,8 +533,8 @@ class ChatCompletionRequest(OpenAIBaseModel): msg["tool_calls"] = list(tool_calls) return self - _grammar_from_tool_parser: bool = PrivateAttr(default=False) - """CAUTION: Should only be set by ``ToolParser.adjust_request``.""" + _grammar_from_parser: bool = PrivateAttr(default=False) + """CAUTION: Should only be set by the parser-engine adapter's adjust_request.""" def build_chat_params( self, diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 4c05db0b8a..f149a305f6 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -65,7 +65,6 @@ from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import BeamSearchParams, SamplingParams from vllm.tokenizers import TokenizerLike from vllm.utils.collection_utils import as_list -from vllm.utils.mistral import is_mistral_tool_parser logger = init_logger(__name__) @@ -157,15 +156,6 @@ class OpenAIServingChat(GenerateBaseServing): model_name=self.model_config.model, is_harmony=self.model_config.hf_config.model_type == "gpt_oss", ) - if ( - self.parser_cls is not None - and is_mistral_tool_parser(self.parser_cls.tool_parser_cls) - and self.parser_cls.reasoning_parser_cls is not None - ): - from vllm.tool_parsers.mistral_tool_parser import MistralToolParser - - MistralToolParser.model_can_reason = True - self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none self.enable_prompt_tokens_details = enable_prompt_tokens_details @@ -350,7 +340,7 @@ class OpenAIServingChat(GenerateBaseServing): else: if not request.include_reasoning: reasoning_ended = True - elif request._grammar_from_tool_parser: + elif request._grammar_from_parser: # The Mistral grammar already includes an optional # `think?` rule that handles both reasoning and # non-reasoning outputs. diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index bf38780967..8dad837613 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -284,7 +284,7 @@ class DerenderChatRequest(BaseModel): Required by the parsing so that tool/reasoning parsers can receive the full request context they expect (request.tools, request.tool_choice, - request._grammar_from_tool_parser, etc.). + request._grammar_from_parser, etc.). """ # --8<-- [end:derender-chat-request] diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py index 895f7516a5..048e714cb4 100644 --- a/vllm/parser/engine/parser_engine.py +++ b/vllm/parser/engine/parser_engine.py @@ -396,6 +396,9 @@ class ParserEngine(Parser): return True return find_tool_name(self._tools, name) + def _accept_tool_name(self, name: str) -> bool: + return bool(name) and self._is_valid_tool_name(name) + # ── Private helpers ───────────────────────────────────────────── def _check_skip_tool_parsing( @@ -807,7 +810,7 @@ class ParserEngine(Parser): deltas: list[DeltaToolCall], name: str | None, ) -> None: - if not name or not self._is_valid_tool_name(name): + if name is None or not self._accept_tool_name(name): return slot = self._tool_slots[idx] slot.name = name @@ -866,8 +869,8 @@ class ParserEngine(Parser): slot = self._tool_slots[idx] if not slot.name_sent: - name = slot.name or self._try_extract_name(idx) - if name and self._is_valid_tool_name(name): + name = slot.name or self._try_extract_name(idx) or "" + if self._accept_tool_name(name): slot.name = name slot.name_sent = True slot.string_keys = self._streamable_string_keys( @@ -1042,7 +1045,7 @@ class ParserEngine(Parser): else: args_json = "{}" - if name and self._is_valid_tool_name(name): + if self._accept_tool_name(name): self._ensure_tool_id(slot, name) args_json = self._fix_arg_types(args_json, name) tool_calls.append( diff --git a/vllm/parser/engine/registered_adapters.py b/vllm/parser/engine/registered_adapters.py index d259ade545..57865e96b5 100644 --- a/vllm/parser/engine/registered_adapters.py +++ b/vllm/parser/engine/registered_adapters.py @@ -15,6 +15,7 @@ from vllm.parser.glm47_moe import Glm47MoeParser from vllm.parser.inkling import InklingParser from vllm.parser.kimi_k2 import KimiK2Parser from vllm.parser.minimax_m2 import MinimaxM2Parser +from vllm.parser.mistral import MistralParser from vllm.parser.nemotron_v3 import NemotronV3Parser from vllm.parser.qwen3 import Qwen3Parser from vllm.parser.seed_oss import SeedOssParser @@ -68,3 +69,8 @@ from vllm.parser.seed_oss import SeedOssParser InklingParserReasoningAdapter, InklingParserToolAdapter, ) = make_adapters(InklingParser) + +( + MistralParserReasoningAdapter, + MistralParserToolAdapter, +) = make_adapters(MistralParser) diff --git a/vllm/parser/mistral.py b/vllm/parser/mistral.py index 52f16136ee..4f7c199202 100644 --- a/vllm/parser/mistral.py +++ b/vllm/parser/mistral.py @@ -3,80 +3,972 @@ from __future__ import annotations +import functools +import json from collections.abc import Sequence -from typing import TYPE_CHECKING +from enum import Enum, auto +from random import choices +from string import ascii_letters, digits +from typing import TYPE_CHECKING, Any, Literal -from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall -from vllm.parser.abstract_parser import DelegatingParser +import ijson +import regex as re +from mistral_common.protocol.instruct.tool_calls import ( + NamedToolChoice as MistralNamedToolChoice, +) +from mistral_common.protocol.instruct.tool_calls import ( + Tool as MistralTool, +) +from mistral_common.protocol.instruct.tool_calls import ( + ToolChoice as MistralToolChoice, +) +from mistral_common.protocol.instruct.tool_calls import ( + ToolChoiceEnum as MistralToolChoiceEnum, +) +from mistral_common.tokens.tokenizers.base import SpecialTokens +from pydantic import Field + +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionNamedToolChoiceParam, +) +from vllm.entrypoints.openai.engine.protocol import ( + DeltaFunctionCall, + DeltaMessage, + DeltaToolCall, + ExtractedToolCallInformation, + FunctionCall, + ToolCall, +) +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.logger import init_logger +from vllm.parser.engine.events import EventType +from vllm.parser.engine.parser_engine import ParserEngine +from vllm.parser.engine.parser_engine_config import ( + ParserEngineConfig, + ParserState, + Transition, +) +from vllm.sampling_params import StructuredOutputsParams +from vllm.utils.mistral import is_mistral_tokenizer if TYPE_CHECKING: from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ) - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest + from vllm.parser.engine.events import SemanticEvent + from vllm.parser.engine.parser_engine import ToolCallSlot + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.abstract_tool_parser import Tool + +logger = init_logger(__name__) + +_ALPHANUMERIC = ascii_letters + digits +_DEFAULT_JSON_SCHEMA: dict[str, Any] = { + "anyOf": [{"type": "object"}, {"type": "array"}] +} + +# Special token strings as emitted/decoded by MistralTokenizer. +_TOOL_CALLS = SpecialTokens.tool_calls.value +_ARGS = SpecialTokens.args.value +_THINK_START_SPECIAL = SpecialTokens.begin_think.value +_THINK_END_SPECIAL = SpecialTokens.end_think.value + +# Plain-text reasoning markers used by v11 tokenizers. +_THINK_START_TEXT = "" +_THINK_END_TEXT = "" + +_OPEN_BRACE = "{" -class MistralParser(DelegatingParser): - def __init__(self, tokenizer, tools=None, *args, **kwargs): - super().__init__(tokenizer, tools, *args, **kwargs) - from vllm.tool_parsers.mistral_tool_parser import MistralToolParser +class StreamingState(Enum): + """Streaming parsing state for pre-v11 tool call extraction.""" - if not isinstance(self._tool_parser, MistralToolParser): - raise ValueError( - "MistralParser requires --tool-call-parser mistral, " - f"got {self._tool_parser.__class__.__name__}." + WAITING_FOR_TOOL_START = auto() + WAITING_FOR_TOOL_KEY = auto() + PARSING_NAME = auto() + PARSING_NAME_COMPLETED = auto() + WAITING_FOR_ARGUMENTS_START = auto() + PARSING_ARGUMENTS = auto() + PARSING_ARGUMENTS_COMPLETED = auto() + TOOL_COMPLETE = auto() + ALL_TOOLS_COMPLETE = auto() + + +class MistralToolCall(ToolCall): + """ToolCall with a Mistral-compatible random alphanumeric id.""" + + id: str = Field(default_factory=lambda: MistralToolCall.generate_random_id()) + + @staticmethod + def generate_random_id() -> str: + # Mistral Tool Call Ids must be alphanumeric with a length of 9. + # https://github.com/mistralai/mistral-common/blob/21ee9f6cee3441e9bb1e6ed2d10173f90bd9b94b/src/mistral_common/protocol/instruct/validator.py#L299 + return "".join(choices(_ALPHANUMERIC, k=9)) + + @staticmethod + def is_valid_id(id: str) -> bool: + return id.isalnum() and len(id) == 9 + + +def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool: + if is_mistral_tokenizer(model_tokenizer): + return model_tokenizer.version < 11 + vocab: dict[str, int] = getattr(model_tokenizer, "get_vocab", lambda: {})() + return _ARGS not in vocab + + +@functools.cache +def mistral_config( + *, + reasoning_encoding: Literal["special_token", "text", "none"], + name: str = "mistral", +) -> ParserEngineConfig: + """Return a :class:`ParserEngineConfig` for the Mistral output format. + + Args: + reasoning_encoding: Reasoning token format. + ``"special_token"`` – v13+ ``[THINK]``/``[/THINK]`` special + tokens placed in both ``terminals`` and ``token_id_terminals``. + ``"text"`` – v11 ````/```` plain text placed in + ``terminals`` only; no ``token_id_terminals`` for think tokens. + ``"none"`` – no reasoning support. + name: Name embedded in the returned config (used for debugging). + + Returns: + A frozen :class:`ParserEngineConfig` with ``initial_state=CONTENT``. + """ + if reasoning_encoding == "special_token": + think_start = _THINK_START_SPECIAL + think_end = _THINK_END_SPECIAL + reasoning_terminals: dict[str, str] = { + "THINK_START": think_start, + "THINK_END": think_end, + } + reasoning_token_id_terminals: dict[str, str] = { + "THINK_START": think_start, + "THINK_END": think_end, + } + elif reasoning_encoding == "text": + think_start = _THINK_START_TEXT + think_end = _THINK_END_TEXT + reasoning_terminals = { + "THINK_START": think_start, + "THINK_END": think_end, + } + # Text think markers have no token-id terminals. + reasoning_token_id_terminals = {} + else: + reasoning_terminals = {} + reasoning_token_id_terminals = {} + + if reasoning_encoding != "none": + reasoning_transitions: dict[tuple[ParserState, str], Transition] = { + (ParserState.CONTENT, "THINK_START"): Transition( + ParserState.REASONING, + (), + ), + # Absorb a duplicate/re-emitted THINK_START inside reasoning. + (ParserState.REASONING, "THINK_START"): Transition( + ParserState.REASONING, + (), + ), + (ParserState.REASONING, "THINK_END"): Transition( + ParserState.CONTENT, + (EventType.REASONING_END,), + ), + # Absorb stray THINK_END that arrives after reasoning ended. + (ParserState.CONTENT, "THINK_END"): Transition( + ParserState.CONTENT, + (), + ), + # [TOOL_CALLS] directly from reasoning implicitly ends it. + (ParserState.REASONING, "TOOL_CALLS"): Transition( + ParserState.TOOL_NAME, + (EventType.REASONING_END, EventType.TOOL_CALL_START), + ), + } + else: + reasoning_transitions = {} + + return ParserEngineConfig( + name=name, + initial_state=ParserState.CONTENT, + terminals={ + **reasoning_terminals, + "TOOL_CALLS": _TOOL_CALLS, + "ARGS": _ARGS, + "OPEN_BRACE": _OPEN_BRACE, + }, + token_id_terminals={ + **reasoning_token_id_terminals, + "TOOL_CALLS": _TOOL_CALLS, + "ARGS": _ARGS, + }, + transitions={ + **reasoning_transitions, + # A tool call from content implicitly ends reasoning when enabled. + (ParserState.CONTENT, "TOOL_CALLS"): Transition( + ParserState.TOOL_NAME, + (EventType.REASONING_END, EventType.TOOL_CALL_START) + if reasoning_encoding != "none" + else (EventType.TOOL_CALL_START,), + ), + # NAME→ARGS via explicit [ARGS] separator (v11+): consumed, no events. + (ParserState.TOOL_NAME, "ARGS"): Transition( + ParserState.TOOL_ARGS, + (), + ), + # NAME→ARGS via "{": carried as ARG_VALUE_CHUNK so the opening + # brace lands in the JSON argument buffer (fallback for name{args}). + (ParserState.TOOL_NAME, "OPEN_BRACE"): Transition( + ParserState.TOOL_ARGS, + (EventType.ARG_VALUE_CHUNK,), + ), + # Parallel tool calls: next [TOOL_CALLS] ends current call. + (ParserState.TOOL_ARGS, "TOOL_CALLS"): Transition( + ParserState.TOOL_NAME, + (EventType.TOOL_CALL_END, EventType.TOOL_CALL_START), + ), + }, + stream_arg_deltas=True, + tool_args_json=True, + strip_trailing_reasoning_whitespace=True, + drop_whitespace_only_content_before_tools=True, + ) + + +class MistralParser(ParserEngine): + """Mistral parser: engine-based reasoning + ``[TOOL_CALLS]`` tool calls. + + Reasoning encoding is auto-detected from the tokenizer: + + - ``"special_token"`` – ``[THINK]`` present in vocab (v13+). + - ``"text"`` – tokenizer supports grammar but has no ``[THINK]`` (v11). + - ``"none"`` – no grammar support; reasoning disabled. + + Tool calls use the ``[TOOL_CALLS]func_name{...}`` format. The opening + ``{`` doubles as the NAME→ARGS separator and is included in the argument + buffer via an ``ARG_VALUE_CHUNK`` event on the transition (mirrors + kimi_k2, JSON args, `tool_args_json=True`). + + When the tokenizer does not support grammar (``_reasoning_encoding == + "none"``), the legacy ``[TOOL_CALLS]``-based extraction path is used + instead of the declarative engine, handling both pre-v11 JSON-array and + v11+ ``funcname{args}`` formats. + """ + + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + **kwargs, + ) -> None: + vocab = tokenizer.get_vocab() + self._reasoning_encoding: Literal["special_token", "text", "none"] + if _THINK_START_SPECIAL in vocab: + self._reasoning_encoding = "special_token" + elif getattr(tokenizer, "supports_grammar", False): + self._reasoning_encoding = "text" + else: + self._reasoning_encoding = "none" + + kwargs.setdefault( + "parser_engine_config", + mistral_config(reasoning_encoding=self._reasoning_encoding), + ) + super().__init__(tokenizer, tools, **kwargs) + + self._tool_calls_token_id: int | None = self.vocab.get(_TOOL_CALLS) + + # Tool calls use the legacy parser for all tokenizer versions; + # reasoning is handled by the engine. + self.bot_token: str = _TOOL_CALLS + self.bot_token_id: int | None = self._tool_calls_token_id + if self.bot_token_id is None: + raise RuntimeError( + "Mistral parser could not locate the tool call token in the tokenizer!" ) - def _maybe_force_auto_tool_parsing( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> None: - # When the Mistral grammar factory injected structured outputs, - # the model emits v11+ format ([TOOL_CALLS]name{args}) that the - # named/required parsers can't handle. Disable them so all - # tool_choice modes fall back to auto tool parsing via - # extract_tool_calls. - if getattr(request, "_grammar_from_tool_parser", False): - assert self._tool_parser is not None - self._tool_parser.supports_required_and_named = False + # Legacy tool-call streaming state. + self.prev_tool_call_arr: list[dict[str, Any]] = [] + self.current_tool_id: int = -1 + self.streaming_state: StreamingState = StreamingState.WAITING_FOR_TOOL_START + self.tool_call_started: bool = False + self.current_tool_name: str | None = None + self.current_tool_mistral_id: str | None = None + self.starting_new_tool: bool = False + self.streamed_args_for_tool: list[str] = [] + self._is_pre_v11: bool = _is_pre_v11_tokeniser(tokenizer) + self.parse_coro = None + if self._is_pre_v11: + self.parse_coro = ijson.parse_coro( + self.update_stream_state_pre_v11_tokenizer() + ) + self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL) - def parse( + def adjust_request( + self, request: ChatCompletionRequest | ResponsesRequest + ) -> ChatCompletionRequest | ResponsesRequest: + if not isinstance(request, ResponsesRequest) and request._grammar_from_parser: + return request + so_non_supported_attributes = [ + "regex", + "choice", + "grammar", + # whitespace_pattern is not a constraint type but an option; + # Mistral grammar factory does not support it. + "whitespace_pattern", + "structural_tag", + ] + any_so_non_supported_active = request.structured_outputs is not None and any( + getattr(request.structured_outputs, attribute) is not None + for attribute in so_non_supported_attributes + ) + response_format_non_supported_active = ( + isinstance(request, ResponsesRequest) + or request.response_format is not None + and request.response_format.type == "structural_tag" + ) + + if ( + not is_mistral_tokenizer(self.model_tokenizer) + or isinstance(request, ResponsesRequest) + or not self.model_tokenizer.supports_grammar + or any_so_non_supported_active + or response_format_non_supported_active + ): + request = super().adjust_request(request) + if request.tools and request.tool_choice != "none": + # Keep special tokens so the [TOOL_CALLS] marker + # survives for tool detection. + request.skip_special_tokens = False + # Inject a guided JSON schema for pre-v11 required/named tool choice + # so the model emits a well-formed bare JSON array instead of + # rambling. tool_choice forces a tool call, so a json_object / + # json_schema response_format is cleared here (the tool schema is + # the sole structured-output constraint), mirroring the base + # ToolParser; a structural_tag response_format is left untouched. + req_tool_choice = request.tool_choice + is_required_or_named = req_tool_choice == "required" or isinstance( + req_tool_choice, ChatCompletionNamedToolChoiceParam + ) + response_format = getattr(request, "response_format", None) + response_format_overridable = response_format is None or ( + response_format.type in ("text", "json_object", "json_schema") + ) + # This runs only in the non-grammar (legacy) branch, so it covers + # both pre-v11 Mistral tokenizers and non-Mistral (e.g. HF-mode) + # tokenizers driving the Mistral tool parser. The remaining guards + # keep it off when the user supplied their own structured output. + if ( + not isinstance(request, ResponsesRequest) + and request.tools + and request.structured_outputs is None + and is_required_or_named + and response_format_overridable + ): + schema = self._build_guided_schema_pre_v11(request) + if schema is not None: + request.structured_outputs = StructuredOutputsParams(json=schema) + request.response_format = None + return request + + json_schema: dict[str, Any] | None = None + if request.structured_outputs is not None: + if request.structured_outputs.json_object is not None: + json_schema = _DEFAULT_JSON_SCHEMA + elif request.structured_outputs.json is not None: + if isinstance(request.structured_outputs.json, str): + json_schema = json.loads(request.structured_outputs.json) + else: + json_schema = request.structured_outputs.json + else: + raise ValueError( + "Unsupported request.structured_outputs for MistralParser. " + "Only `json` and `json_object` are supported." + ) + elif ( + request.response_format is not None + and request.response_format.type != "text" + ): + if request.response_format.type == "json_object": + json_schema = _DEFAULT_JSON_SCHEMA + elif request.response_format.type == "json_schema": + if request.response_format.json_schema is not None: + json_schema = request.response_format.json_schema.json_schema + else: + json_schema = _DEFAULT_JSON_SCHEMA + else: + raise ValueError( + "MistralParser only accepts `text`, `json_object` or " + f"`json_schema`, got {request.response_format=}" + ) + request.response_format = None + + grammar_factory = self.model_tokenizer.grammar_factory + + # Rendering grammar is cached in mistral-common given tools, template and mode. + template = grammar_factory.select_jinja_template() + + mistral_tools = ( + [MistralTool.from_openai(tool.model_dump()) for tool in request.tools] + if request.tools is not None + else None + ) + + tool_choice: MistralToolChoice + match request.tool_choice: + case "none" | "auto" | "required": + tool_choice = MistralToolChoiceEnum(request.tool_choice) + case None: + tool_choice = MistralToolChoiceEnum.auto + # _ == Named tool choice + case _: + tool_choice = MistralNamedToolChoice.model_validate( + { + "type": "function", + "function": {"name": request.tool_choice.function.name}, + } + ) + + match tool_choice, json_schema is not None: + case MistralToolChoiceEnum.none, True: + lark_grammar = grammar_factory.get_lark_for_json_schema( + template=template, json_schema=json_schema + ) + case _, _: + lark_grammar = grammar_factory.get_lark_from_jinja( + template=template, + mode=tool_choice, + tools=mistral_tools, + json_schema=json_schema, + parallel_tool_calls=request.parallel_tool_calls, + json_only=False, + ) + + request.structured_outputs = StructuredOutputsParams(grammar=lark_grammar) + request._grammar_from_parser = True + return request + + def _build_guided_schema_pre_v11( + self, + request: ChatCompletionRequest, + ) -> dict[str, Any] | None: + """Build a guided JSON schema for pre-v11 required/named tool choice. + + The schema enforces the Mistral-native array format + ``[{"name": ..., "arguments": {...}}]`` so the model emits a parseable + bare JSON array instead of free-form text. + + Args: + request: The chat completion request carrying `tools` and + `tool_choice`. + + Returns: + A JSON Schema dict, or ``None`` if the named tool is not found. + """ + tool_choice = request.tool_choice + tools = request.tools or [] + extra: dict[str, Any] = {} + + if tool_choice == "required": + applicable_tools = tools + else: + # Named tool choice — restrict to the single requested tool. + assert isinstance(tool_choice, ChatCompletionNamedToolChoiceParam) + chosen_name = tool_choice.function.name + applicable_tools = [t for t in tools if t.function.name == chosen_name] + if not applicable_tools: + logger.warning( + "Named tool %r not found in tools list; " + "skipping guided schema injection.", + chosen_name, + ) + return None + extra["maxItems"] = 1 + + any_of = [ + { + "type": "object", + "properties": { + "name": {"type": "string", "enum": [tool.function.name]}, + "arguments": tool.function.parameters or {"type": "object"}, + }, + "required": ["name", "arguments"], + } + for tool in applicable_tools + ] + + return { + "type": "array", + "minItems": 1, + **extra, + "items": { + "type": "object", + "anyOf": any_of, + }, + } + + def _ensure_tool_id(self, slot: ToolCallSlot, name: str) -> None: + """Assign a Mistral-compatible 9-char alphanumeric id to `slot`.""" + if not slot.id: + slot.id = MistralToolCall.generate_random_id() + + def extract_tool_calls_from_content( + self, + content: str, + request: ChatCompletionRequest, + ) -> ExtractedToolCallInformation: + if self._is_pre_v11: + return self._legacy_extract_tool_calls(content, request) + return super().extract_tool_calls_from_content(content, request) + + def _legacy_extract_tool_calls( + self, + model_output: str, + request: ChatCompletionRequest, + ) -> ExtractedToolCallInformation: + """Pre-v11 non-streaming extraction. + + Handles ``[TOOL_CALLS][{...}]`` and guided bare-array formats. + """ + # tool_choice="none" with tools: never produce tool calls. + if request.tool_choice == "none" and request.tools: + return ExtractedToolCallInformation( + tools_called=False, tool_calls=[], content=model_output + ) + + content: str | None = None + if self.bot_token in model_output: + content_and_raw_tool_calls = model_output.split(self.bot_token) + content = content_and_raw_tool_calls[0] + raw_tool_calls = content_and_raw_tool_calls[1:] + # pre-v11: content[BOT] [{tool_call1},{tool_call2}] + if len(raw_tool_calls) != 1: + raise ValueError( + "Only one BOT token should have been outputted, " + f"but got {model_output}." + ) + stringified_tool_calls = raw_tool_calls[0].strip() + elif request.tool_choice == "required" or isinstance( + request.tool_choice, ChatCompletionNamedToolChoiceParam + ): + # Guided bare-array output (no [TOOL_CALLS] marker). + stringified_tool_calls = model_output.strip() + else: + return ExtractedToolCallInformation( + tools_called=False, tool_calls=[], content=model_output + ) + + try: + # Use raw_decode to parse the first valid JSON value, + # ignoring trailing tokens the model may emit after + # the tool call array. + tool_calls, _ = json.JSONDecoder().raw_decode(stringified_tool_calls) + except json.JSONDecodeError: + try: + raw_tool_call = self.tool_call_regex.findall(stringified_tool_calls)[0] + tool_calls = json.loads(raw_tool_call) + tool_calls = [ + { + "name": tool_call["name"], + "arguments": json.dumps( + tool_call.get("arguments", {}), + ensure_ascii=False, + ), + } + for tool_call in tool_calls + ] + except (IndexError, json.JSONDecodeError): + logger.exception("Error in extracting tool call from response.") + return ExtractedToolCallInformation( + tools_called=False, + tool_calls=[], + content=stringified_tool_calls, + ) + else: + tool_calls = [ + { + "name": tool_call["name"], + "arguments": json.dumps( + tool_call.get("arguments", {}), + ensure_ascii=False, + ), + } + for tool_call in tool_calls + ] + + mistral_tool_calls: list[MistralToolCall] = [ + MistralToolCall( + type="function", + function=FunctionCall( + name=tool_call["name"], + arguments=tool_call.get("arguments", "{}"), + ), + ) + for tool_call in tool_calls + ] + + return ExtractedToolCallInformation( + tools_called=True, + tool_calls=mistral_tool_calls, + content=content if content and content.strip() else None, + ) + + def extract_tool_calls_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + request: ChatCompletionRequest | ResponsesRequest, + ) -> DeltaMessage | None: + if not self._is_pre_v11: + return super().extract_tool_calls_streaming( + previous_text, + current_text, + delta_text, + previous_token_ids, + current_token_ids, + delta_token_ids, + request, + ) + # Pre-v11: latch on [TOOL_CALLS] or on the first content of a guided + # required/named request (bare JSON array, no special token). + if self.bot_token_id in delta_token_ids or self.bot_token in delta_text: + self.tool_call_started = True + elif not self.tool_call_started and delta_text: + is_guided = request.tool_choice == "required" or isinstance( + request.tool_choice, ChatCompletionNamedToolChoiceParam + ) + if is_guided: + self.tool_call_started = True + if not self.tool_call_started: + return DeltaMessage(content=delta_text) + try: + return self._extract_tool_calls_streaming_pre_v11_tokenizer( + delta_text=delta_text, + delta_token_ids=delta_token_ids, + ) + except Exception: + logger.exception("Error trying to handle streaming tool call.") + return None + + @ijson.coroutine + def update_stream_state_pre_v11_tokenizer(self): + while True: + (prefix, event, value) = yield + + if prefix == "item" and event == "start_map": + self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY + self.starting_new_tool = True + if prefix == "item" and event == "map_key" and value == "name": + self.streaming_state = StreamingState.PARSING_NAME + if prefix == "item.name" and event == "string": + self.current_tool_name = value + self.streaming_state = StreamingState.PARSING_NAME_COMPLETED + if prefix == "item" and event == "map_key" and value == "arguments": + self.streaming_state = StreamingState.WAITING_FOR_ARGUMENTS_START + if prefix == "item.arguments" and event == "start_map": + self.streaming_state = StreamingState.PARSING_ARGUMENTS + if prefix == "item.arguments" and event == "end_map": + self.streaming_state = StreamingState.PARSING_ARGUMENTS_COMPLETED + if prefix == "item" and event == "end_map": + self.streaming_state = StreamingState.TOOL_COMPLETE + if prefix == "" and event == "end_array": + self.streaming_state = StreamingState.ALL_TOOLS_COMPLETE + + def _extract_tool_calls_streaming_pre_v11_tokenizer( + self, + delta_text: str, + delta_token_ids: Sequence[int], + ) -> DeltaMessage | None: + """Extract tool calls for pre-v11 Mistral models. + + Handles ``[TOOL_CALLS][{"name": "add", "arguments":{"a": 3.5}}]``. + """ + assert self.parse_coro is not None + content = None + delta_tool_calls: list[DeltaToolCall] = [] + current_tool_call: DeltaToolCall = DeltaToolCall( + index=self.current_tool_id, type="function" + ) + current_tool_call_modified = False + if self.bot_token_id in delta_token_ids or self.bot_token in delta_text: + # this is the first tool call + if not delta_text.startswith(self.bot_token): + content = delta_text.split(self.bot_token)[0] + delta_text = "".join(delta_text.split(self.bot_token)[1:]) + + # ijson gives no text index per event, so split the delta manually + # to know where each event is emitted from. + while len(delta_text) > 0: + streaming_state_before_parse = self.streaming_state + + if self.streaming_state == StreamingState.WAITING_FOR_TOOL_START: + delta_to_be_parsed, delta_text = self._split_delta( + delta_text=delta_text, + stop_after_opening_curly_braces=1, + ) + elif self.streaming_state == StreamingState.WAITING_FOR_TOOL_KEY: + delta_to_be_parsed, delta_text = self._split_delta( + delta_text=delta_text, + stop_after_colon=1, + stop_after_opening_curly_braces=1, + ) + elif self.streaming_state == StreamingState.PARSING_NAME: + delta_to_be_parsed, delta_text = self._split_delta( + delta_text=delta_text, + stop_after_comma=1, + stop_after_closing_brackets=1, + ) + elif self.streaming_state == StreamingState.WAITING_FOR_ARGUMENTS_START: + delta_to_be_parsed, delta_text = self._split_delta( + delta_text=delta_text, + stop_after_opening_curly_braces=1, + ) + elif self.streaming_state == StreamingState.PARSING_ARGUMENTS: + delta_to_be_parsed, delta_text = self._split_delta( + delta_text=delta_text, + stop_after_closing_curly_braces=1, + ) + elif self.streaming_state in [ + StreamingState.PARSING_ARGUMENTS_COMPLETED, + StreamingState.PARSING_NAME_COMPLETED, + ]: + delta_to_be_parsed, delta_text = self._split_delta( + delta_text=delta_text, + stop_after_closing_curly_braces=1, + stop_after_closing_brackets=1, + ) + elif self.streaming_state == StreamingState.TOOL_COMPLETE: + delta_to_be_parsed, delta_text = self._split_delta( + delta_text=delta_text, + stop_after_opening_curly_braces=1, + stop_after_closing_brackets=1, + ) + elif self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE: + content = delta_text + delta_text = "" + else: + delta_to_be_parsed = delta_text + delta_text = "" + + if self.streaming_state != StreamingState.ALL_TOOLS_COMPLETE: + self.parse_coro.send(delta_to_be_parsed.encode("utf-8")) + + # start_map is the authoritative new-tool signal and survives + # batched deltas, unlike comparing pre/post streaming states. + if self.starting_new_tool: + self.starting_new_tool = False + if current_tool_call_modified: + if self.current_tool_mistral_id is not None: + current_tool_call.id = self.current_tool_mistral_id + self.current_tool_mistral_id = None + self._track_streamed_args_pre_v11(current_tool_call) + delta_tool_calls.append(current_tool_call) + current_tool_call_modified = False + self.current_tool_id += 1 + self.streamed_args_for_tool.append("") + self.prev_tool_call_arr.append({}) + self.current_tool_mistral_id = MistralToolCall.generate_random_id() + current_tool_call = DeltaToolCall( + index=self.current_tool_id, + type="function", + ) + if current_tool_call.function is None: + current_tool_call.function = DeltaFunctionCall() + + if self.current_tool_name is not None: + current_tool_call_modified = True + current_tool_call.function.name = self.current_tool_name + self.prev_tool_call_arr[self.current_tool_id]["name"] = ( + self.current_tool_name + ) + self.current_tool_name = None + if self.streaming_state == StreamingState.PARSING_NAME_COMPLETED: + self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY + if self.streaming_state in [ + StreamingState.PARSING_ARGUMENTS, + StreamingState.PARSING_ARGUMENTS_COMPLETED, + ]: + if self.streaming_state == StreamingState.PARSING_ARGUMENTS_COMPLETED: + self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY + current_tool_call_modified = True + if current_tool_call.function.arguments is None: + current_tool_call.function.arguments = delta_to_be_parsed + else: + current_tool_call.function.arguments += delta_to_be_parsed + if streaming_state_before_parse != StreamingState.PARSING_ARGUMENTS: + # It's the first chunk of arg. let's lstrip it + current_tool_call.function.arguments = ( + current_tool_call.function.arguments.lstrip() + ) + + if current_tool_call_modified: + if self.current_tool_mistral_id is not None: + current_tool_call.id = self.current_tool_mistral_id + self.current_tool_mistral_id = None + self._track_streamed_args_pre_v11(current_tool_call) + delta_tool_calls.append(current_tool_call) + + if content or len(delta_tool_calls) > 0: + delta_message = DeltaMessage() + if content: + delta_message.content = content + if len(delta_tool_calls) > 0: + delta_message.tool_calls = delta_tool_calls + return delta_message + else: + if self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE: + return DeltaMessage() + else: + return None + + def _track_streamed_args_pre_v11(self, tool_call: DeltaToolCall) -> None: + r"""Accumulate `tool_call` arguments into the streaming state.""" + if tool_call.function is not None and tool_call.function.arguments is not None: + self.streamed_args_for_tool[self.current_tool_id] += ( + tool_call.function.arguments + ) + self.prev_tool_call_arr[self.current_tool_id]["arguments"] = ( + self.streamed_args_for_tool[self.current_tool_id] + ) + + def _split_delta( + self, + delta_text: str, + stop_after_quotes: int = -1, + stop_after_opening_curly_braces: int = -1, + stop_after_closing_curly_braces: int = -1, + stop_after_closing_brackets: int = -1, + stop_after_colon: int = -1, + stop_after_comma: int = -1, + ) -> tuple[str, str]: + delta_to_be_parsed = "" + for i, c in enumerate(delta_text): + if c in ['"', "'"]: + delta_to_be_parsed += c + stop_after_quotes -= 1 + if stop_after_quotes == 0: + return (delta_to_be_parsed, delta_text[i + 1 :]) + elif c == "{": + delta_to_be_parsed += c + stop_after_opening_curly_braces -= 1 + if stop_after_opening_curly_braces == 0: + return (delta_to_be_parsed, delta_text[i + 1 :]) + elif c == "}": + delta_to_be_parsed += c + stop_after_closing_curly_braces -= 1 + if stop_after_closing_curly_braces == 0: + return (delta_to_be_parsed, delta_text[i + 1 :]) + elif c == "]": + delta_to_be_parsed += c + stop_after_closing_brackets -= 1 + if stop_after_closing_brackets == 0: + return (delta_to_be_parsed, delta_text[i + 1 :]) + elif c == ":": + delta_to_be_parsed += c + stop_after_colon -= 1 + if stop_after_colon == 0: + return (delta_to_be_parsed, delta_text[i + 1 :]) + elif c == ",": + delta_to_be_parsed += c + stop_after_comma -= 1 + if stop_after_comma == 0: + return (delta_to_be_parsed, delta_text[i + 1 :]) + else: + delta_to_be_parsed += c + + return (delta_to_be_parsed, "") + + def is_reasoning_end(self, input_ids: list[int]) -> bool: + if self._reasoning_encoding == "none": + return True + if super().is_reasoning_end(input_ids): + return True + # [TOOL_CALLS] acts as an implicit reasoning-end marker + if self.bot_token_id is not None: + reasoning_start_id = self._reasoning_start_token_id + for i in range(len(input_ids) - 1, -1, -1): + if ( + reasoning_start_id is not None + and input_ids[i] == reasoning_start_id + ): + return False + if input_ids[i] == self.bot_token_id: + return True + return False + + def extract_reasoning( self, model_output: str, request: ChatCompletionRequest | ResponsesRequest, - enable_auto_tools: bool = False, - model_output_token_ids: Sequence[int] = (), - ) -> tuple[str | None, str | None, list[FunctionCall] | None]: - self._maybe_force_auto_tool_parsing(request) - reasoning, content, tool_calls = super().parse( - model_output, - request, - enable_auto_tools, - model_output_token_ids, - ) - if tool_calls: - from vllm.tool_parsers.mistral_tool_parser import MistralToolCall + ) -> tuple[str | None, str | None]: + if self._reasoning_encoding == "none": + return None, model_output + return super().extract_reasoning(model_output, request) - # Named/required tool_choice builds FunctionCalls without - # ID, backfill with Mistral-format IDs. - for tc in tool_calls: - if not tc.id: - tc.id = MistralToolCall.generate_random_id() - return reasoning, content, tool_calls + def _accept_tool_name(self, name: str) -> bool: + # Once `[ARGS]` or the opening `{` has moved the slot past its name, + # that name is final, empty included. Emitting the call with "" + # surfaces the malformed generation instead of dropping it. + return self._is_valid_tool_name(name) - def parse_delta( + def _try_extract_name(self, idx: int) -> str | None: + # No JSON-embedded "name" key in this format; the slot name is final. + return self._tool_slots[idx].name + + def _extract_name_and_args(self, raw_body: str) -> tuple[str, str]: + # Never a {"name": ..., "arguments": ...} envelope -- a literal "name" + # argument key must not be mistaken for the tool name. + return "", self._extract_args_json(raw_body, "") + + def _handle_arg_chunk( self, - delta_text: str, - delta_token_ids: list[int], - request: ChatCompletionRequest | ResponsesRequest, - prompt_token_ids: list[int] | None = None, - *, - finished: bool, - ) -> DeltaMessage | None: - self._maybe_force_auto_tool_parsing(request) - return super().parse_delta( - delta_text, - delta_token_ids, - request, - prompt_token_ids, - finished=finished, + event: SemanticEvent, + deltas: list[DeltaToolCall], + ) -> None: + """Emit the opening ``{`` as an arg delta when the name is first sent. + + When the TOOL_NAME→TOOL_ARGS transition fires, ``{`` arrives as an + ARG_VALUE_CHUNK before ``name_sent`` is True. The parent emits the + name delta but not the ``{`` arg delta. This override re-emits the + current chunk so streaming clients receive a valid JSON prefix. + """ + idx = event.tool_index + name_sent_before = ( + 0 <= idx < len(self._tool_slots) and self._tool_slots[idx].name_sent ) + super()._handle_arg_chunk(event, deltas) + if ( + event.value + and not name_sent_before + and 0 <= idx < len(self._tool_slots) + and self._tool_slots[idx].name_sent + ): + deltas.append( + DeltaToolCall( + index=idx, + function=DeltaFunctionCall(arguments=event.value), + ) + ) + + def _extract_args_json(self, raw_args: str, func_name: str) -> str: + """Return the first complete JSON value in ``raw_args``. + + v11+ tool calls are emitted as ``name{args}`` with no terminator, so a + model may append ordinary text after the closing brace. Parse the first + JSON value with ``raw_decode`` and drop any trailing output, mirroring + the pre-v11 path (fixes gh#48975). + """ + stripped = raw_args.strip() + if not stripped: + return "{}" + try: + _, end = json.JSONDecoder().raw_decode(stripped) + except json.JSONDecodeError: + return stripped + return stripped[:end] diff --git a/vllm/parser/parser_manager.py b/vllm/parser/parser_manager.py index d9cfb759c7..865f749a79 100644 --- a/vllm/parser/parser_manager.py +++ b/vllm/parser/parser_manager.py @@ -109,8 +109,6 @@ class ParserManager: if reasoning_parser_cls is None and tool_parser_cls is None: return None - from vllm.utils.mistral import is_mistral_tool_parser - if is_harmony: from vllm.parser.harmony import HarmonyParser @@ -118,13 +116,6 @@ class ParserManager: HarmonyParser.tool_parser_cls = tool_parser_cls return HarmonyParser - if is_mistral_tool_parser(tool_parser_cls): - from vllm.parser.mistral import MistralParser - - MistralParser.reasoning_parser_cls = reasoning_parser_cls - MistralParser.tool_parser_cls = tool_parser_cls - return MistralParser - if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3": from vllm.parser.kimi_k3 import KimiK3Parser diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index d645ec5020..b5111a9ec6 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -106,7 +106,7 @@ _REASONING_PARSERS_TO_REGISTER = { ), "mistral": ( "mistral_reasoning_parser", - "MistralReasoningParser", + "MistralParserReasoningAdapter", ), "nemotron_v3": ( "nemotron_v3_engine_reasoning_parser", diff --git a/vllm/reasoning/mistral_reasoning_parser.py b/vllm/reasoning/mistral_reasoning_parser.py index c224c3c165..6adbb08cb5 100644 --- a/vllm/reasoning/mistral_reasoning_parser.py +++ b/vllm/reasoning/mistral_reasoning_parser.py @@ -1,162 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Iterable, Sequence -from functools import cached_property -from typing import TYPE_CHECKING +from vllm.parser.engine.registered_adapters import MistralParserReasoningAdapter -from vllm.reasoning import ReasoningParser -from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser -from vllm.tokenizers.mistral import MistralTokenizer - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - - -class MistralReasoningParser(BaseThinkingReasoningParser): - """ - Reasoning parser for Mistral models. - - The Mistral models uses `[THINK]`...`[/THINK]` tokens to denote reasoning - text. This parser extracts the reasoning content from the model output. - - A valid reasoning trace should always start with a `[THINK]` token and end with - a `[/THINK]` token. - - If `[THINK]` token is not generated, then this parser only returns content. - """ - - def __init__(self, tokenizer: MistralTokenizer, *args, **kwargs): - if not isinstance(tokenizer, MistralTokenizer): - raise ValueError("The tokenizer must be an instance of MistralTokenizer.") - - ReasoningParser.__init__(self, tokenizer, *args, **kwargs) - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ReasoningParser " - "constructor during construction." - ) - - self.start_token_id = tokenizer.tokenizer.get_special_token(self.start_token) - self.end_token_id = tokenizer.tokenizer.get_special_token(self.end_token) - - if self.start_token_id is None or self.end_token_id is None: - raise RuntimeError( - "Mistral reasoning parser could not locate think start/end " - "tokens in the tokenizer!" - ) - - @cached_property - def start_token(self) -> str: - """The token that starts reasoning content.""" - from mistral_common.tokens.tokenizers.base import SpecialTokens - - return SpecialTokens.begin_think - - @cached_property - def end_token(self) -> str: - """The token that ends reasoning content.""" - from mistral_common.tokens.tokenizers.base import SpecialTokens - - return SpecialTokens.end_think - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - has_eot_token = False - - for id in reversed(input_ids): - if id == self.start_token_id: - # Reasoning ends only if a BOT token is found before a EOT token. - return has_eot_token - elif id == self.end_token_id: - has_eot_token = True - return False - - def is_reasoning_end_streaming( - self, input_ids: Sequence[int], delta_ids: Iterable[int] - ) -> bool: - if self.end_token_id in delta_ids: - return True - # Grammar's think? is optional — if [THINK] was never generated, - # reasoning was skipped entirely. - return self.start_token_id not in input_ids - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - """ - Extract the content - """ - has_bot_token = False - has_eot_token = False - bot_token_index = -1 - eot_token_index = -1 - # One for loop instead of multiple lookups - for i, token_id in enumerate(input_ids): - # We filter that we have multiple BOT tokens which should not - # happen for a well prompted trained model - if token_id == self.start_token_id and not has_bot_token: - has_bot_token = True - bot_token_index = i - elif token_id == self.end_token_id: - has_eot_token = True - eot_token_index = i - break - - # 1. Only BOT has been outputted - if has_bot_token and not has_eot_token: - # Should be = [] if model is well prompted and trained. - return input_ids[:bot_token_index] - # 2. Neither BOT or EOT have been outputted - elif not has_bot_token and not has_eot_token: - return input_ids - # 3. Both BOT and EOT have been outputted. - elif has_bot_token and has_eot_token: - return input_ids[:bot_token_index] + input_ids[eot_token_index + 1 :] - # 4. Only EOT has been outputted => this should not have occurred for a model - # well prompted and trained. - else: - return input_ids[:eot_token_index] + input_ids[eot_token_index + 1 :] - - def extract_reasoning( - self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" - ) -> tuple[str | None, str | None]: - """ - Extract reasoning content from the model output. - """ - if not model_output: - return (None, "") - - # Check if the start token is present in the model output, remove it - # if it is present. - prev_bot_token, bot_token, post_bot_token = model_output.partition( - self.start_token - ) - - has_bot_token = bool(bot_token) - # Valid EOT tokens should follow BOT token - has_valid_eot_token = has_bot_token and self.end_token in post_bot_token - - # 1. If there is BOT token followed by EOT token - if has_bot_token and has_valid_eot_token: - prev_eot_token, _, post_eot_token = post_bot_token.partition(self.end_token) - # If model is well prompted and trained prev_bot_token should be "" - content = prev_bot_token + post_eot_token - return prev_eot_token, content if content else None - # 2. Only BOT token - elif has_bot_token: - # If model is well prompted and trained prev_bot_token should be "" - return post_bot_token, prev_bot_token if prev_bot_token else None - # 3. EOT token has been outputted without BOT or neither has been outputted - else: - has_non_valid_eot_token = self.end_token in prev_bot_token - # 3.a EOT token has been outputted without BOT - # If model is well prompted and trained `has_non_valid_eot_token` should - # be `False` and the parser outputs all tokens as 'content' - if has_non_valid_eot_token: - prev_eot_token, _, post_eot_token = prev_bot_token.partition( - self.end_token - ) - return None, prev_eot_token + post_eot_token - # 3.b neither BOT or EOT have been outputted - else: - return None, prev_bot_token +__all__ = ["MistralParserReasoningAdapter"] diff --git a/vllm/tokenizers/mistral.py b/vllm/tokenizers/mistral.py index 3d8bf37eac..fe2cb3fd57 100644 --- a/vllm/tokenizers/mistral.py +++ b/vllm/tokenizers/mistral.py @@ -21,7 +21,6 @@ from mistral_common.tokens.tokenizers.base import ( ) from mistral_common.tokens.tokenizers.instruct import ( InstructTokenizerBase, - InstructTokenizerV13, ) from mistral_common.tokens.tokenizers.mistral import ( MistralTokenizer as MistralCommonTokenizer, @@ -461,6 +460,7 @@ class MistralTokenizer(TokenizerLike): def convert_tokens_to_string(self, tokens: list[str]) -> str: to_decode_special_tokens = { SpecialTokens.tool_calls, + SpecialTokens.args, SpecialTokens.begin_think, SpecialTokens.end_think, } @@ -517,11 +517,20 @@ class MistralTokenizer(TokenizerLike): non_skip_special_tokens_ids = { self.tokenizer.get_special_token(SpecialTokens.tool_calls), } - if isinstance(self.instruct, InstructTokenizerV13): - if self.instruct.BEGIN_THINK: - non_skip_special_tokens_ids.add(self.instruct.BEGIN_THINK) - if self.instruct.END_THINK: - non_skip_special_tokens_ids.add(self.instruct.END_THINK) + # [ARGS] only exists in v11+ tool-call tokenizers; older tokenizers + # raise (Tekken) or return unk (SPM) for it. + if self.tokenizer.is_special(SpecialTokens.args): + non_skip_special_tokens_ids.add( + self.tokenizer.get_special_token(SpecialTokens.args) + ) + # [THINK]/[/THINK] only exist in v13+ reasoning tokenizers; use the + # same is_special gate as [ARGS] above so newer versions are covered + # without an isinstance check on the instruct tokenizer. + for think_token in (SpecialTokens.begin_think, SpecialTokens.end_think): + if self.tokenizer.is_special(think_token): + non_skip_special_tokens_ids.add( + self.tokenizer.get_special_token(think_token) + ) ids_kept = [ i diff --git a/vllm/tool_parsers/mistral_tool_parser.py b/vllm/tool_parsers/mistral_tool_parser.py index 026098a873..36a23764c6 100644 --- a/vllm/tool_parsers/mistral_tool_parser.py +++ b/vllm/tool_parsers/mistral_tool_parser.py @@ -3,767 +3,37 @@ from __future__ import annotations -import json -from collections.abc import Sequence -from enum import Enum, auto -from random import choices -from string import ascii_letters, digits -from typing import Any +from typing import TYPE_CHECKING -import ijson -import regex as re -from mistral_common.protocol.instruct.tool_calls import ( - NamedToolChoice as MistralNamedToolChoice, -) -from mistral_common.protocol.instruct.tool_calls import ( - Tool as MistralTool, -) -from mistral_common.protocol.instruct.tool_calls import ( - ToolChoice as MistralToolChoice, -) -from mistral_common.protocol.instruct.tool_calls import ( - ToolChoiceEnum as MistralToolChoiceEnum, -) -from pydantic import Field +from vllm.parser.engine.registered_adapters import MistralParserToolAdapter -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.entrypoints.openai.engine.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.sampling_params import StructuredOutputsParams -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.utils.mistral import is_mistral_tokenizer - -logger = init_logger(__name__) - -ALPHANUMERIC = ascii_letters + digits - -_DEFAULT_JSON_SCHEMA = {"anyOf": [{"type": "object"}, {"type": "array"}]} +if TYPE_CHECKING: + from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -class StreamingState(Enum): - """Enum for tracking the current streaming parsing state.""" +class MistralToolParser(MistralParserToolAdapter): # type: ignore[valid-type, misc] + # Marked so is_mistral_tool_parser() in vllm.utils.mistral recognises + # this adapter and lets adjust_request() fire for tool_choice="none" + # on grammar-capable Mistral tokenizers. + IS_MISTRAL_TOOL_PARSER = True - WAITING_FOR_TOOL_START = auto() - WAITING_FOR_TOOL_KEY = ( - auto() - ) # waiting for the "name" or "arguments" key to be complete - PARSING_NAME = auto() - PARSING_NAME_COMPLETED = auto() - WAITING_FOR_ARGUMENTS_START = auto() - PARSING_ARGUMENTS = auto() - PARSING_ARGUMENTS_COMPLETED = auto() - TOOL_COMPLETE = auto() - ALL_TOOLS_COMPLETE = auto() - - -class MistralToolCall(ToolCall): - id: str = Field(default_factory=lambda: MistralToolCall.generate_random_id()) - - @staticmethod - def generate_random_id(): - # Mistral Tool Call Ids must be alphanumeric with a length of 9. - # https://github.com/mistralai/mistral-common/blob/21ee9f6cee3441e9bb1e6ed2d10173f90bd9b94b/src/mistral_common/protocol/instruct/validator.py#L299 - return "".join(choices(ALPHANUMERIC, k=9)) - - @staticmethod - def is_valid_id(id: str) -> bool: - return id.isalnum() and len(id) == 9 - - -def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool: - if is_mistral_tokenizer(model_tokenizer): - return model_tokenizer.version < 11 - # For HF tokenizers, check if [ARGS] token exists in vocab - # which indicates a v11+ equivalent tokenizer - vocab: dict[str, int] = getattr(model_tokenizer, "get_vocab", lambda: {})() - return "[ARGS]" not in vocab - - -class MistralToolParser(ToolParser): - r"""Tool call parser for Mistral models, intended for use with either: - - - `mistral_common `_ - (recommended) - - the `examples/tool_chat_template_mistral.jinja` template. - - Used when `--enable-auto-tool-choice --tool-call-parser mistral` are all - set. - """ - - IS_MISTRAL_TOOL_PARSER = True # used by vllm.utils.mistral - - # Used to generate correct grammar in `adjust_request` - model_can_reason: bool = False - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - if not is_mistral_tokenizer(self.model_tokenizer): - logger.info("Non-Mistral tokenizer detected when using a Mistral model...") - - # initialize properties used for state when parsing tool calls in - # streaming mode - self.prev_tool_call_arr: list[dict[str, Any]] = [] - self.current_tool_id: int = -1 - self.streaming_state: StreamingState = StreamingState.WAITING_FOR_TOOL_START - - # For streaming pre v11 tokenizer tool calls - self.current_tool_name: str | None = None - self.current_tool_mistral_id: str | None = None - self.starting_new_tool = False - self._is_pre_v11 = _is_pre_v11_tokeniser(self.model_tokenizer) - if self._is_pre_v11: - self.parse_coro = ijson.parse_coro( - self.update_stream_state_pre_v11_tokenizer() - ) - - self.bot_token = "[TOOL_CALLS]" - self.bot_token_id = self.vocab.get(self.bot_token) - self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL) - - if self.bot_token_id is None: - raise RuntimeError( - "Mistral Tool Parser could not locate the tool call token in " - "the tokenizer!" - ) + # Mistral emits tool calls in the [TOOL_CALLS]name[ARGS]{...} format, not the + # plain JSON array / bare-args that the serving layer's generic required and + # named handlers expect. Route required/named through this parser's own + # extraction (treated like "auto") so the [TOOL_CALLS] format is parsed + # correctly instead of crashing or dumping the raw envelope into arguments. + supports_required_and_named = False def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest + self, + request: ChatCompletionRequest | ResponsesRequest, ) -> ChatCompletionRequest | ResponsesRequest: - so_non_supported_attributes = [ - "regex", - "choice", - "grammar", - # whitespace_pattern is not a constraint type but an option; - # Mistral grammar factory does not support it. - "whitespace_pattern", - "structural_tag", - ] - any_so_non_supported_active = request.structured_outputs is not None and any( - getattr(request.structured_outputs, attribute) is not None - for attribute in so_non_supported_attributes - ) - response_format_non_supported_active = ( - isinstance(request, ResponsesRequest) - or request.response_format is not None - and request.response_format.type == "structural_tag" - ) - - if ( - not is_mistral_tokenizer(self.model_tokenizer) - or isinstance(request, ResponsesRequest) - or not self.model_tokenizer.supports_grammar - or any_so_non_supported_active - or response_format_non_supported_active - ): - request = super().adjust_request(request) - if request.tools and request.tool_choice != "none": - # Do not skip special tokens when using chat template - # with Mistral parser as TOOL_CALL token is needed - # for tool detection. - # Note: we don't want skip_special_tokens=False - # with MistralTokenizer as it is incompatible - request.skip_special_tokens = False - return request - - json_schema: dict[str, Any] | None = None - if request.structured_outputs is not None: - if request.structured_outputs.json_object is not None: - json_schema = _DEFAULT_JSON_SCHEMA - elif request.structured_outputs.json is not None: - if isinstance(request.structured_outputs.json, str): - json_schema = json.loads(request.structured_outputs.json) - else: - json_schema = request.structured_outputs.json - else: - raise ValueError( - "Unsupported request.structured_outputs for MistralToolParser. " - "Only `json` and `json_object` are supported." - ) - elif ( - request.response_format is not None - and request.response_format.type != "text" - ): - if request.response_format.type == "json_object": - json_schema = _DEFAULT_JSON_SCHEMA - elif request.response_format.type == "json_schema": - if request.response_format.json_schema is not None: - json_schema = request.response_format.json_schema.json_schema - else: - json_schema = _DEFAULT_JSON_SCHEMA - else: - raise ValueError( - "MistralToolParser only accepts `text`, `json_object` or " - f"`json_schema`, got {request.response_format=}" - ) - # Structured Outputs will be defined. - request.response_format = None - - grammar_factory = self.model_tokenizer.grammar_factory - - # TODO: Once unified parser, improve this. - # The issue is figuring out when a model is a reasoning one or not. - template = grammar_factory.select_jinja_template( - reasoning=self.model_can_reason - ) - - mistral_tools = ( - [MistralTool.from_openai(tool.model_dump()) for tool in request.tools] - if request.tools is not None - else None - ) - - tool_choice: MistralToolChoice - match request.tool_choice: - case "none" | "auto" | "required": - tool_choice = MistralToolChoiceEnum(request.tool_choice) - case None: - tool_choice = MistralToolChoiceEnum.auto - # _ == Named tool choice - case _: - tool_choice = MistralNamedToolChoice.model_validate( - { - "type": "function", - "function": {"name": request.tool_choice.function.name}, - } - ) - - # Rendering grammar is cached in mistral-common given tools, template and mode. - match tool_choice, json_schema is not None: - case MistralToolChoiceEnum.none, True: - lark_grammar = grammar_factory.get_lark_for_json_schema( - template=template, json_schema=json_schema - ) - case _, _: - lark_grammar = grammar_factory.get_lark_from_jinja( - template=template, - mode=tool_choice, - tools=mistral_tools, - json_schema=json_schema, - parallel_tool_calls=request.parallel_tool_calls, - json_only=False, - ) - - request.structured_outputs = StructuredOutputsParams(grammar=lark_grammar) - request._grammar_from_tool_parser = True - return request - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - """ - Extract the tool calls from a complete model response. - - Content and tool calls formatting depends on the Mistral's tokenizer version - used to train the model: - - - < v11: `content[BOT] [{tool_call1},{tool_call2}]` - - >= v11: `content[BOT]tool_name1{args_call1}[BOT]tool_name2{args_call2}` - - with [BOT] the tool call token. - - Note: - For tokenizer versions >= v11, tool calls with arguments wrongly formatted - are still returned as tool calls. This is to allow the model to know it - tried to make a tool call. It reduces chance of another failure and - prevents that the context is filled with tool calls wrongly placed in - assistant message contents. - """ - - # If the tool call token is not present, return a text response - if self.bot_token not in model_output: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - content_and_raw_tool_calls = model_output.split(self.bot_token) - content = content_and_raw_tool_calls[0] - raw_tool_calls = content_and_raw_tool_calls[1:] - - # >= v11: content[BOT]tool_name1{args_call1}[BOT]tool_name2{args_call2} - if not self._is_pre_v11: - tool_calls = [] - for raw_tool_call in raw_tool_calls: - if "{" not in raw_tool_call: - continue - - end_name = raw_tool_call.find("{") - tool_name, args = ( - raw_tool_call[:end_name], - raw_tool_call[end_name:], - ) - - # HF tokenizers may include [ARGS] in the text - tool_name = tool_name.replace("[ARGS]", "") - tool_calls.append({"name": tool_name, "arguments": args}) - - # < v11: content[BOT] [{tool_call1},{tool_call2}] - else: - if len(raw_tool_calls) != 1: - raise ValueError( - "Only one BOT token should have been outputted, " - f"but got {model_output}." - ) - stringified_tool_calls = raw_tool_calls[0].strip() - try: - # Use raw_decode to parse the first valid JSON value, - # ignoring trailing tokens the model may emit after - # the tool call array. - tool_calls, _ = json.JSONDecoder().raw_decode(stringified_tool_calls) - except json.JSONDecodeError: - try: - raw_tool_call = self.tool_call_regex.findall( - stringified_tool_calls - )[0] - tool_calls = json.loads(raw_tool_call) - tool_calls = [ - { - "name": tool_call["name"], - "arguments": json.dumps( - tool_call.get("arguments", {}), - ensure_ascii=False, - ), - } - for tool_call in tool_calls - ] - except (IndexError, json.JSONDecodeError): - logger.exception("Error in extracting tool call from response.") - return ExtractedToolCallInformation( - tools_called=False, - tool_calls=[], - content=stringified_tool_calls, - ) - else: - tool_calls = [ - { - "name": tool_call["name"], - "arguments": json.dumps( - tool_call.get("arguments", {}), - ensure_ascii=False, - ), - } - for tool_call in tool_calls - ] - - mistral_tool_calls: list[MistralToolCall] = [ - MistralToolCall( - type="function", - function=FunctionCall( - name=tool_call["name"], - arguments=tool_call.get("arguments", "{}"), - ), - ) - for tool_call in tool_calls - ] - - return ExtractedToolCallInformation( - tools_called=True, - tool_calls=mistral_tool_calls, - content=content if content.strip() else None, - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - has_bot_token = ( - self.bot_token_id in current_token_ids or self.bot_token in current_text - ) - if not has_bot_token: - # if the tool call token is not in the tokens generated so far, - # append output to contents since it's not a tool - return DeltaMessage(content=delta_text) - - # if the tool call token IS in the tokens generated so far, that - # means we're parsing as tool calls now - try: - if self._is_pre_v11: - return self._extract_tool_calls_streaming_pre_v11_tokenizer( - delta_text=delta_text, - delta_token_ids=delta_token_ids, - ) - else: - return self._extract_tool_calls_streaming( - delta_text=delta_text, delta_token_ids=delta_token_ids - ) - except Exception: - logger.exception("Error trying to handle streaming tool call.") - return None - - def _extract_tool_calls_streaming( - self, - delta_text: str, - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - """ - Extracts tool calls for Mistral models - doing tool calls of the following format: - `[TOOL_CALLS]add{"a": 3.5, "b": 4}` - """ - additional_content: str = "" - if self.streaming_state == StreamingState.WAITING_FOR_TOOL_START: - # this is the first tool call - if self.bot_token not in delta_text: - return DeltaMessage(content=delta_text) - if not delta_text.startswith(self.bot_token): - additional_content += delta_text.split(self.bot_token)[0] - delta_text = self.bot_token + "".join( - delta_text.split(self.bot_token)[1:] - ) - - delta_tool_calls = self._generate_delta_tool_call(delta_text) - if not additional_content and len(delta_tool_calls) == 0: - if self.streaming_state in [ - StreamingState.PARSING_ARGUMENTS, - StreamingState.PARSING_ARGUMENTS_COMPLETED, - StreamingState.TOOL_COMPLETE, - StreamingState.ALL_TOOLS_COMPLETE, - ]: - # Return an empty DeltaMessage once the tool calls are all done - # so that finish_reason gets set. - return DeltaMessage() - else: - # return None when the tool is not likely to be finished - # This can occur when the name is being parsed for example - # and we wait for the name to be complete - # before sending the function name - return None - - delta = DeltaMessage() - if additional_content: - delta.content = additional_content - if len(delta_tool_calls) > 0: - delta.tool_calls = delta_tool_calls - - return delta - - def _generate_delta_tool_call(self, delta_text: str) -> list[DeltaToolCall]: - if delta_text == "" or delta_text is None: - return [] - delta_function_name = None - tool_id = None - if self.streaming_state not in [ - StreamingState.PARSING_NAME, - StreamingState.PARSING_ARGUMENTS, - ] and delta_text.startswith(self.bot_token): - self.current_tool_id += 1 - self.streamed_args_for_tool.append("") - self.prev_tool_call_arr.append({}) - self.streaming_state = StreamingState.PARSING_NAME - delta_text = delta_text.replace(self.bot_token, "", 1) - if self.streaming_state == StreamingState.PARSING_NAME: - if self.current_tool_name is None: - self.current_tool_name = "" - # The name stops where the arguments start - # And the arguments start with the `{` char - if "{" in delta_text: - tool_id = MistralToolCall.generate_random_id() - delta_function_name = delta_text.split("{")[0] - self.current_tool_name += delta_function_name - # HF tokenizers may include [ARGS] in the text - self.current_tool_name = self.current_tool_name.replace("[ARGS]", "") - self.prev_tool_call_arr[self.current_tool_id]["name"] = ( - self.current_tool_name - ) - delta_text = delta_text[len(delta_function_name) :] - self.streaming_state = StreamingState.PARSING_ARGUMENTS - else: - # we want to send the tool name once it's complete - self.current_tool_name += delta_text - return [] - if self.streaming_state == StreamingState.PARSING_ARGUMENTS: - next_function_text = None - if self.bot_token in delta_text: - # current tool call is over - delta_arguments = "" - delta_arguments += delta_text.split(self.bot_token)[0] - next_function_text = delta_text[len(delta_arguments) :] - self.streaming_state = StreamingState.TOOL_COMPLETE - else: - delta_arguments = delta_text - self.streamed_args_for_tool[self.current_tool_id] += delta_arguments - self.prev_tool_call_arr[self.current_tool_id]["arguments"] = ( - self.streamed_args_for_tool[self.current_tool_id] - ) - ret = [] - if self.current_tool_name or delta_arguments: - ret += [ - DeltaToolCall( - index=self.current_tool_id, - type="function", - id=tool_id, - function=DeltaFunctionCall( - name=self.current_tool_name, arguments=delta_arguments - ).model_dump(exclude_none=True), - ) - ] - self.current_tool_name = None - if next_function_text: - ret += self._generate_delta_tool_call(next_function_text) - return ret - # Should not happen - return [] - - @ijson.coroutine - def update_stream_state_pre_v11_tokenizer(self): - while True: - (prefix, event, value) = yield - - if prefix == "item" and event == "start_map": - self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY - self.starting_new_tool = True - if prefix == "item" and event == "map_key" and value == "name": - self.streaming_state = StreamingState.PARSING_NAME - if prefix == "item.name" and event == "string": - self.current_tool_name = value - self.streaming_state = StreamingState.PARSING_NAME_COMPLETED - if prefix == "item" and event == "map_key" and value == "arguments": - self.streaming_state = StreamingState.WAITING_FOR_ARGUMENTS_START - if prefix == "item.arguments" and event == "start_map": - self.streaming_state = StreamingState.PARSING_ARGUMENTS - if prefix == "item.arguments" and event == "end_map": - self.streaming_state = StreamingState.PARSING_ARGUMENTS_COMPLETED - if prefix == "item" and event == "end_map": - self.streaming_state = StreamingState.TOOL_COMPLETE - if prefix == "" and event == "end_array": - self.streaming_state = StreamingState.ALL_TOOLS_COMPLETE - - def _extract_tool_calls_streaming_pre_v11_tokenizer( - self, - delta_text: str, - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - """ - Extracts tool calls for Mistral models - doing tool calls of the following format: - `[TOOL_CALLS][{"name": "add", "arguments":{"a": 3.5, "b": 4}}` - """ - assert self.parse_coro is not None - content = None - delta_tool_calls: list[DeltaToolCall] = [] - current_tool_call: DeltaToolCall = DeltaToolCall( - index=self.current_tool_id, type="function" - ) - current_tool_call_modified = False - if self.bot_token_id in delta_token_ids or self.bot_token in delta_text: - # this is the first tool call - if not delta_text.startswith(self.bot_token): - content = delta_text.split(self.bot_token)[0] - delta_text = "".join(delta_text.split(self.bot_token)[1:]) - - # Cut smartly the delta text to catch the ijson events - # as ijson does not give us the index in the text at each event. - # We need to cut so that we know - # where in the text the events are emitted from. - while len(delta_text) > 0: - streaming_state_before_parse = self.streaming_state - - if self.streaming_state == StreamingState.WAITING_FOR_TOOL_START: - delta_to_be_parsed, delta_text = self._split_delta( - delta_text=delta_text, - stop_after_opening_curly_braces=1, - ) - elif self.streaming_state == StreamingState.WAITING_FOR_TOOL_KEY: - # Wait until another key is sent - # or the current tool is completed - delta_to_be_parsed, delta_text = self._split_delta( - delta_text=delta_text, - stop_after_colon=1, - stop_after_opening_curly_braces=1, - # if the tool ends, we want to separate - # at the start of the next tool - ) - elif self.streaming_state == StreamingState.PARSING_NAME: - delta_to_be_parsed, delta_text = self._split_delta( - delta_text=delta_text, - stop_after_comma=1, - stop_after_closing_brackets=1, - ) - elif self.streaming_state == StreamingState.WAITING_FOR_ARGUMENTS_START: - delta_to_be_parsed, delta_text = self._split_delta( - delta_text=delta_text, - stop_after_opening_curly_braces=1, - ) - elif self.streaming_state == StreamingState.PARSING_ARGUMENTS: - delta_to_be_parsed, delta_text = self._split_delta( - delta_text=delta_text, - stop_after_closing_curly_braces=1, - # we could be more clever - # by listening to item.arguments.* start_map events - # and know how many curly braces we can allow - ) - elif self.streaming_state in [ - StreamingState.PARSING_ARGUMENTS_COMPLETED, - StreamingState.PARSING_NAME_COMPLETED, - ]: - delta_to_be_parsed, delta_text = self._split_delta( - delta_text=delta_text, - stop_after_closing_curly_braces=1, - stop_after_closing_brackets=1, - ) - elif self.streaming_state == StreamingState.TOOL_COMPLETE: - delta_to_be_parsed, delta_text = self._split_delta( - delta_text=delta_text, - stop_after_opening_curly_braces=1, - stop_after_closing_brackets=1, - ) - elif self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE: - content = delta_text - delta_text = "" - else: - delta_to_be_parsed = delta_text - delta_text = "" - - if self.streaming_state != StreamingState.ALL_TOOLS_COMPLETE: - self.parse_coro.send(delta_to_be_parsed.encode("utf-8")) - - # Given the parsed text and the possible streaming state change, - # let's add to the tool delta - # start_map is the authoritative new-tool signal and survives - # batched deltas, unlike comparing pre/post streaming states - if self.starting_new_tool: - self.starting_new_tool = False - if current_tool_call_modified: - if self.current_tool_mistral_id is not None: - current_tool_call.id = self.current_tool_mistral_id - self.current_tool_mistral_id = None - self._track_streamed_args_pre_v11(current_tool_call) - delta_tool_calls.append(current_tool_call) - current_tool_call_modified = False - self.current_tool_id += 1 - self.streamed_args_for_tool.append("") - self.prev_tool_call_arr.append({}) - self.current_tool_mistral_id = MistralToolCall.generate_random_id() - current_tool_call = DeltaToolCall( - index=self.current_tool_id, - type="function", - ) - if current_tool_call.function is None: - current_tool_call.function = DeltaFunctionCall() - - if self.current_tool_name is not None: - # we have the complete tool name - current_tool_call_modified = True - current_tool_call.function.name = self.current_tool_name - self.prev_tool_call_arr[self.current_tool_id]["name"] = ( - self.current_tool_name - ) - self.current_tool_name = None - if self.streaming_state == StreamingState.PARSING_NAME_COMPLETED: - self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY - if self.streaming_state in [ - StreamingState.PARSING_ARGUMENTS, - StreamingState.PARSING_ARGUMENTS_COMPLETED, - ]: - if self.streaming_state == StreamingState.PARSING_ARGUMENTS_COMPLETED: - self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY - # the delta_to_be_parsed is part of arguments. - current_tool_call_modified = True - if current_tool_call.function.arguments is None: - current_tool_call.function.arguments = delta_to_be_parsed - else: - current_tool_call.function.arguments += delta_to_be_parsed - if streaming_state_before_parse != StreamingState.PARSING_ARGUMENTS: - # It's the first chunk of arg. let's lstrip it - current_tool_call.function.arguments = ( - current_tool_call.function.arguments.lstrip() - ) - - if current_tool_call_modified: - if self.current_tool_mistral_id is not None: - current_tool_call.id = self.current_tool_mistral_id - self.current_tool_mistral_id = None - self._track_streamed_args_pre_v11(current_tool_call) - delta_tool_calls.append(current_tool_call) - - if content or len(delta_tool_calls) > 0: - delta_message = DeltaMessage() - if content: - delta_message.content = content - if len(delta_tool_calls) > 0: - delta_message.tool_calls = delta_tool_calls - return delta_message - else: - if self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE: - return DeltaMessage() - else: - return None - - def _track_streamed_args_pre_v11(self, tool_call: DeltaToolCall) -> None: - r"""Accumulate `tool_call` arguments into the streaming state.""" - if tool_call.function is not None and tool_call.function.arguments is not None: - self.streamed_args_for_tool[self.current_tool_id] += ( - tool_call.function.arguments - ) - self.prev_tool_call_arr[self.current_tool_id]["arguments"] = ( - self.streamed_args_for_tool[self.current_tool_id] - ) - - def _split_delta( - self, - delta_text: str, - stop_after_quotes: int = -1, - stop_after_opening_curly_braces: int = -1, - stop_after_closing_curly_braces: int = -1, - stop_after_closing_brackets: int = -1, - stop_after_colon: int = -1, - stop_after_comma=-1, - ) -> tuple[str, str]: - delta_to_be_parsed = "" - for i, c in enumerate(delta_text): - if c in ['"', "'"]: - delta_to_be_parsed += c - stop_after_quotes -= 1 - if stop_after_quotes == 0: - return (delta_to_be_parsed, delta_text[i + 1 :]) - elif c == "{": - delta_to_be_parsed += c - stop_after_opening_curly_braces -= 1 - if stop_after_opening_curly_braces == 0: - return (delta_to_be_parsed, delta_text[i + 1 :]) - elif c == "}": - delta_to_be_parsed += c - stop_after_closing_curly_braces -= 1 - if stop_after_closing_curly_braces == 0: - return (delta_to_be_parsed, delta_text[i + 1 :]) - elif c == "]": - delta_to_be_parsed += c - stop_after_closing_brackets -= 1 - if stop_after_closing_brackets == 0: - return (delta_to_be_parsed, delta_text[i + 1 :]) - elif c == ":": - delta_to_be_parsed += c - stop_after_colon -= 1 - if stop_after_colon == 0: - return (delta_to_be_parsed, delta_text[i + 1 :]) - elif c == ",": - delta_to_be_parsed += c - stop_after_comma -= 1 - if stop_after_comma == 0: - return (delta_to_be_parsed, delta_text[i + 1 :]) - else: - delta_to_be_parsed += c - - return (delta_to_be_parsed, "") + # Skip the base ToolParser tool_choice -> structured_outputs + # conversion. Mistral enforces tool_choice via the grammar ``mode`` + # (auto/none/required/named); forwarding a tool-derived json_schema + # would be unioned into the grammar by mistral-common, allowing raw + # JSON in place of a real [TOOL_CALLS] call. Genuine user-provided + # structured output on the request is still honored by the engine's + # adjust_request. + return self._parser_engine.adjust_request(request) diff --git a/vllm/tool_parsers/streaming.py b/vllm/tool_parsers/streaming.py index c3b8a11ba3..d318d5b963 100644 --- a/vllm/tool_parsers/streaming.py +++ b/vllm/tool_parsers/streaming.py @@ -99,7 +99,7 @@ def extract_named_tool_call_streaming( else: if is_mistral_tokenizer(tokenizer): # Import mistral_common only if we need it. - from vllm.tool_parsers.mistral_tool_parser import MistralToolCall + from vllm.parser.mistral import MistralToolCall tool_call_id = MistralToolCall.generate_random_id() else: diff --git a/vllm/utils/mistral.py b/vllm/utils/mistral.py index 276ca8170f..469af7e603 100644 --- a/vllm/utils/mistral.py +++ b/vllm/utils/mistral.py @@ -12,10 +12,8 @@ from vllm.utils.import_utils import LazyLoader if TYPE_CHECKING: # if type checking, eagerly import the module import vllm.tokenizers.mistral as mt - import vllm.tool_parsers.mistral_tool_parser as mtp else: mt = LazyLoader("mt", globals(), "vllm.tokenizers.mistral") - mtp = LazyLoader("mtp", globals(), "vllm.tool_parsers.mistral_tool_parser") def is_mistral_tokenizer(obj: TokenizerLike | None) -> TypeGuard[mt.MistralTokenizer]: @@ -31,13 +29,10 @@ def is_mistral_tokenizer(obj: TokenizerLike | None) -> TypeGuard[mt.MistralToken def is_mistral_tool_parser(cls: type | None) -> bool: - """Return true if *cls* is (a subclass of) MistralToolParser. + """Return true if *cls* carries the ``IS_MISTRAL_TOOL_PARSER`` marker. - Uses a class attribute check so that importing - ``vllm.tool_parsers.mistral_tool_parser`` — and transitively - ``mistral_common`` — is not required. + The marker is set on the engine-adapter subclass registered under the + ``"mistral"`` tool-parser key. No import of the parser module is + required. """ - return bool( - getattr(cls, "IS_MISTRAL_TOOL_PARSER", False) - and issubclass(cls, mtp.MistralToolParser) # type: ignore[arg-type] - ) + return bool(getattr(cls, "IS_MISTRAL_TOOL_PARSER", False))