[PARSER][Mistral] unified engine-based parser for reasoning and tool calls (#48947)

This commit is contained in:
Julien Denize
2026-07-30 15:38:00 +02:00
committed by GitHub
parent e2efe79695
commit 1a20d23dab
30 changed files with 2934 additions and 1495 deletions
+1 -1
View File
@@ -31,7 +31,7 @@ partial-json-parser # used for parsing partial JSON outputs
jsonschema >= 4.23.0 # required for MiniMax M3 tool schema validation
pyzmq >= 25.0.0
msgspec
mistral_common[image] >= 1.11.5
mistral_common[image] >= 1.11.6
opencv-python-headless >= 4.13.0 # required for video IO
pyyaml
six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
+1 -1
View File
@@ -476,7 +476,7 @@ mcp==1.28.1
# via -r requirements/test/../common.txt
mdurl==0.1.2
# via markdown-it-py
mistral-common==1.11.5
mistral-common==1.11.6
# via
# -r requirements/test/../common.txt
# -r requirements/test/cuda.in
+1 -1
View File
@@ -33,7 +33,7 @@ torchaudio==2.11.0
torchvision==0.28.0
transformers_stream_generator # required for qwen-vl test
matplotlib # required for qwen-vl test
mistral_common[image,audio] >= 1.11.5 # required for voxtral test
mistral_common[image,audio] >= 1.11.6 # required for voxtral test
num2words # required for smolvlm test
open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
datamodel_code_generator # required for minicpm3 test
+1 -1
View File
@@ -502,7 +502,7 @@ mcp==1.28.1
# via -r requirements/test/../common.txt
mdurl==0.1.2
# via markdown-it-py
mistral-common==1.11.5
mistral-common==1.11.6
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
+1 -1
View File
@@ -23,7 +23,7 @@ jiwer # required for audio tests
timm # required for internvl test
transformers_stream_generator # required for qwen-vl test
matplotlib # required for qwen-vl test
mistral_common[image,audio] >= 1.11.5 # required for voxtral test
mistral_common[image,audio] >= 1.11.6 # required for voxtral test
num2words # required for smolvlm test
opencv-python-headless >= 4.13.0 # required for video test
datamodel_code_generator # required for minicpm3 test
+1 -1
View File
@@ -29,7 +29,7 @@ tblib # for pickling test exceptions
timm>=1.0.17 # required for internvl and gemma3n-mm test
transformers_stream_generator # required for qwen-vl test
matplotlib # required for qwen-vl test
mistral_common[image,audio]>=1.11.5 # required for voxtral test
mistral_common[image,audio]>=1.11.6 # required for voxtral test
num2words # required for smolvlm test
open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
datamodel_code_generator # required for minicpm3 test
+1 -1
View File
@@ -497,7 +497,7 @@ mcp==1.27.0
# via -r requirements/test/../common.txt
mdurl==0.1.2
# via markdown-it-py
mistral-common==1.11.5
mistral-common==1.11.6
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
+1 -1
View File
@@ -373,7 +373,7 @@ mcp==1.28.1
# via -r requirements/test/../common.txt
mdurl==0.1.2
# via markdown-it-py
mistral-common==1.11.5
mistral-common==1.11.6
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
@@ -5,12 +5,10 @@ import json
import pytest
from vllm.parser.mistral import MistralToolCall
from vllm.sampling_params import SamplingParams
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tool_parsers.mistral_tool_parser import (
MistralToolCall,
MistralToolParser,
)
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
from ...utils import check_logprobs_close
@@ -40,6 +40,7 @@ from vllm.parser.engine.adapters import (
ParserEngineReasoningAdapter,
ParserEngineToolAdapter,
)
from vllm.parser.mistral import MistralParser
_TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam])
@@ -84,6 +85,11 @@ def _discover_pairings() -> list[_PairingInfo]:
for engine_cls, adapters in engines.items():
if "tool" not in adapters or "reasoning" not in adapters:
continue
if engine_cls is MistralParser:
# Mistral uses brace-balanced JSON tool args with no TOOL_END
# token, so it does not fit this TOOL_END-based replay harness.
# It is covered by tests/parser/mistral/ instead.
continue
cfg = engine_cls(bare_tok, None).parser_engine_config
if cfg.name not in _BUILDERS:
missing_builders.append(f"{engine_cls.__name__} (config.name={cfg.name!r})")
+6
View File
@@ -34,6 +34,7 @@ from tests.parser.engine.trace_builder import _BUILDERS, build_samples
from vllm.parser.engine import registered_adapters as _adapters_mod
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.engine.parser_engine_config import ParserState
from vllm.parser.mistral import MistralParser
# ── Parser discovery ─────────────────────────────────────────────────
@@ -64,6 +65,11 @@ def _discover_parsers() -> list[_ParserInfo]:
and obj is not ParserEngine
):
continue
if obj is MistralParser:
# Mistral uses brace-balanced JSON tool args with no TOOL_END
# token, so it does not fit this TOOL_END-based replay harness.
# It is covered by tests/parser/mistral/ instead.
continue
cfg = obj(bare_tok, None).parser_engine_config
if cfg.name not in _BUILDERS:
missing_builders.append(f"{obj.__name__} (config.name={cfg.name!r})")
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+599
View File
@@ -0,0 +1,599 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from mistral_common.protocol.instruct.messages import (
AssistantMessage,
ThinkChunk,
ToolMessage,
UserMessage,
)
from mistral_common.protocol.instruct.request import InstructRequest
from mistral_common.protocol.instruct.tool_calls import FunctionCall, ToolCall
from tests.reasoning.utils import run_reasoning_extraction_mistral
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.parser.parser_manager import ParserManager
from vllm.reasoning import ReasoningParser, ReasoningParserManager
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
from vllm.tokenizers.mistral import MistralTokenizer
_PARSER_NAME = "mistral"
_MODEL_V13 = "mistralai/Magistral-Small-2509"
_MODEL_V11 = "mistralai/Magistral-Small-2506"
_SAMPLE_TOOLS = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a city.",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
]
@pytest.fixture(scope="module")
def mistral_tokenizer() -> MistralTokenizer:
"""v13 Magistral tokenizer with `[THINK]`/`[/THINK]` special tokens."""
return MistralTokenizer.from_pretrained(_MODEL_V13)
@pytest.fixture(scope="module")
def mistral_v11_tokenizer() -> MistralTokenizer:
"""v11 Magistral tokenizer using plain-text `<think>`/`</think>` tags."""
return MistralTokenizer.from_pretrained(_MODEL_V11)
def _encode_v13(tokenizer: MistralTokenizer, output: str) -> list[int]:
"""Encode output string with `[THINK]`/`[/THINK]` markers into token IDs.
`[THINK]` and `[/THINK]` placeholders in `output` are replaced by their
special-token IDs; all surrounding text is encoded with the base tokenizer.
Mirrors the encoding helper from the deleted
`tests/reasoning/test_mistral_reasoning_parser.py`.
"""
think_start = "[THINK]"
think_end = "[/THINK]"
len_start = len(think_start)
len_end = len(think_end)
index_start = output.find(think_start)
index_end = output.find(think_end)
out: list[int] = []
if index_start != -1:
out += tokenizer.tokenizer.encode(output[:index_start], False, False)
out += [tokenizer.instruct.BEGIN_THINK]
if index_end != -1:
middle = output[index_start + len_start : index_end]
suffix = output[index_end + len_end :]
out += tokenizer.tokenizer.encode(middle, False, False)
out += [tokenizer.instruct.END_THINK]
out += tokenizer.tokenizer.encode(suffix, False, False)
else:
out += tokenizer.tokenizer.encode(
output[index_start + len_start :], False, False
)
elif index_end != -1:
out += tokenizer.tokenizer.encode(output[:index_end], False, False)
out += [tokenizer.instruct.END_THINK]
out += tokenizer.tokenizer.encode(output[index_end + len_end :], False, False)
else:
out += tokenizer.tokenizer.encode(output, False, False)
return out
_TEST_CASES = [
# v13 special-token encoding: [THINK]…[/THINK] with trailing content.
pytest.param(
False,
"[THINK]r[/THINK]c",
"r",
"c",
id="v13_basic",
),
pytest.param(
True,
"[THINK]r[/THINK]c",
"r",
"c",
id="v13_basic_streaming",
),
# No trailing content after reasoning ends.
pytest.param(
False,
"[THINK]r[/THINK]",
"r",
None,
id="v13_no_trailing_content",
),
pytest.param(
True,
"[THINK]r[/THINK]",
"r",
None,
id="v13_no_trailing_content_streaming",
),
# Multi-line reasoning and multi-line content.
pytest.param(
False,
"[THINK]a\nb[/THINK]c\nd",
"a\nb",
"c\nd",
id="v13_multiline",
),
pytest.param(
True,
"[THINK]a\nb[/THINK]c\nd",
"a\nb",
"c\nd",
id="v13_multiline_streaming",
),
# No reasoning at all: output is pure content.
pytest.param(
False,
"hello world",
None,
"hello world",
id="v13_no_reasoning",
),
pytest.param(
True,
"hello world",
None,
"hello world",
id="v13_no_reasoning_streaming",
),
# [THINK] opened but never closed; reasoning accumulates, content is None.
pytest.param(
False,
"[THINK]r",
"r",
None,
id="v13_think_no_end",
),
pytest.param(
True,
"[THINK]r",
"r",
None,
id="v13_think_no_end_streaming",
),
# Stray [/THINK] with no matching [THINK]: no reasoning, content follows.
pytest.param(
False,
"[/THINK]This is the rest",
None,
"This is the rest",
id="v13_stray_think_end",
),
pytest.param(
True,
"[/THINK]This is the rest",
None,
"This is the rest",
id="v13_stray_think_end_streaming",
),
# Content before [THINK]: leading segment joins with trailing segment.
pytest.param(
False,
"Before\n[THINK]r[/THINK]\nAfter",
"r",
"Before\n\nAfter",
id="v13_leading_content",
),
pytest.param(
True,
"Before\n[THINK]r[/THINK]\nAfter",
"r",
"Before\n\nAfter",
id="v13_leading_content_streaming",
),
# Empty output: no reasoning, no content.
pytest.param(
False,
"",
None,
None,
id="v13_empty",
),
pytest.param(
True,
"",
None,
None,
id="v13_empty_streaming",
),
]
@pytest.mark.parametrize(
"streaming, output, expected_reasoning, expected_content",
_TEST_CASES,
)
def test_mistral_reasoning_v13(
streaming: bool,
output: str,
expected_reasoning: str | None,
expected_content: str | None,
mistral_tokenizer: MistralTokenizer,
) -> None:
output_tokens = _encode_v13(mistral_tokenizer, output)
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)(
mistral_tokenizer
)
reasoning, content = run_reasoning_extraction_mistral(
parser, output_tokens, streaming=streaming
)
assert reasoning == expected_reasoning
assert content == expected_content
def test_system_prompt_think_ignored(
mistral_tokenizer: MistralTokenizer,
) -> None:
"""Streaming content extraction must not
include `[THINK]`/`[/THINK]` examples from the system prompt.
"""
output = "[THINK]real[/THINK]answer"
output_tokens = _encode_v13(mistral_tokenizer, output)
for streaming in (False, True):
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(
_PARSER_NAME
)(mistral_tokenizer)
reasoning, content = run_reasoning_extraction_mistral(
parser, output_tokens, streaming=streaming
)
assert reasoning == "real"
assert content == "answer"
def test_reasoning_with_tool_calls(
mistral_tokenizer: MistralTokenizer,
) -> None:
"""Reasoning extraction with a tool call in the generated output."""
tool_calls_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]")
if tool_calls_token_id is None:
pytest.skip("Tokenizer does not have [TOOL_CALLS] in vocab")
# Token sequence representing: [THINK]r[/THINK][TOOL_CALLS]f{"a":1}
output_tokens = (
[mistral_tokenizer.instruct.BEGIN_THINK]
+ mistral_tokenizer.tokenizer.encode("r", False, False)
+ [mistral_tokenizer.instruct.END_THINK]
+ [tool_calls_token_id]
+ mistral_tokenizer.tokenizer.encode('f{"a":1}', False, False)
)
for streaming in (False, True):
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(
_PARSER_NAME
)(mistral_tokenizer)
reasoning, _content = run_reasoning_extraction_mistral(
parser, output_tokens, streaming=streaming
)
assert reasoning == "r"
assert "[TOOL_CALLS]" not in (reasoning or "")
@pytest.mark.parametrize("streaming", [False, True], ids=["nonstream", "stream"])
def test_reasoning_v11_plain_text_think(
mistral_v11_tokenizer: MistralTokenizer, streaming: bool
) -> None:
"""v11 tokenizers use plain-text `<think>`/`</think>` (no special tokens)."""
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)(
mistral_v11_tokenizer
)
tokens = mistral_v11_tokenizer.tokenizer.encode("<think>r</think>c", False, False)
reasoning, content = run_reasoning_extraction_mistral(
parser, tokens, streaming=streaming
)
assert reasoning == "r"
assert content == "c"
def test_is_reasoning_end(
mistral_tokenizer: MistralTokenizer,
) -> None:
"""is_reasoning_end returns True iff reasoning has finished."""
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)(
mistral_tokenizer
)
# Complete reasoning: END_THINK present → True.
complete_ids = _encode_v13(mistral_tokenizer, "[THINK]r[/THINK]c")
assert parser.is_reasoning_end(complete_ids)
# Open-only: no END_THINK, BEGIN_THINK present → False.
open_ids = _encode_v13(mistral_tokenizer, "[THINK]r")
assert not parser.is_reasoning_end(open_ids)
# No reasoning markers at all → False.
no_reasoning_ids = _encode_v13(mistral_tokenizer, "hello")
assert not parser.is_reasoning_end(no_reasoning_ids)
# Explicit END_THINK closes reasoning → True.
explicit_end_ids = (
[mistral_tokenizer.instruct.BEGIN_THINK]
+ mistral_tokenizer.tokenizer.encode("r", False, False)
+ [mistral_tokenizer.instruct.END_THINK]
)
assert parser.is_reasoning_end(explicit_end_ids)
# [TOOL_CALLS] acts as implicit reasoning-end marker.
tool_calls_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]")
if tool_calls_token_id is None:
pytest.skip("Tokenizer does not have [TOOL_CALLS] in vocab")
implicit_end_ids = (
[mistral_tokenizer.instruct.BEGIN_THINK]
+ mistral_tokenizer.tokenizer.encode("r", False, False)
+ [tool_calls_token_id]
)
assert parser.is_reasoning_end(implicit_end_ids)
def _stream_parse_delta(parser, tokenizer, gen_ids, request, prompt_ids):
"""Stream `gen_ids` through `parse_delta` and reconstruct reasoning/content."""
reasoning = ""
content = ""
previous_tokens: list[str] | None = None
prefix_offset = 0
read_offset = 0
for i, tok in enumerate(gen_ids):
new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally(
tokenizer=tokenizer,
all_input_ids=gen_ids[: i + 1],
prev_tokens=previous_tokens,
prefix_offset=prefix_offset,
read_offset=read_offset,
skip_special_tokens=True,
spaces_between_special_tokens=True,
)
previous_tokens = (
previous_tokens + new_tokens if previous_tokens else new_tokens
)
delta_message = parser.parse_delta(
delta_text=delta_text,
delta_token_ids=[tok],
request=request,
prompt_token_ids=prompt_ids,
finished=(i == len(gen_ids) - 1),
)
if delta_message is not None:
reasoning += delta_message.reasoning or ""
content += delta_message.content or ""
return reasoning, content
def test_parse_delta_keeps_reasoning_open_after_closed_think_prompt(
mistral_tokenizer: MistralTokenizer,
) -> None:
"""A closed `[THINK]` block in the prompt must not finalize reasoning.
Regression: reasoning system prompts end the prompt with a closed
`[THINK]...[/THINK]` block. With a parser-injected grammar the generation's
reasoning must still be captured as ``reasoning_content`` rather than
leaking into ``content`` (the prompt-based reasoning-end heuristic must be
skipped). This exercises ``DelegatingParser.parse_delta``'s prompt check,
which the isolated reasoning-parser tests do not cover.
"""
parser_cls = ParserManager.get_parser(
tool_parser_name="mistral",
reasoning_parser_name="mistral",
enable_auto_tools=True,
)
parser = parser_cls(mistral_tokenizer, None)
request = parser.adjust_request(
ChatCompletionRequest(
messages=[],
model="test",
tools=_SAMPLE_TOOLS,
tool_choice="auto",
)
)
assert getattr(request, "_grammar_from_parser", False)
prompt_ids = _encode_v13(mistral_tokenizer, "[THINK]system reasoning[/THINK]")
gen_ids = _encode_v13(
mistral_tokenizer,
"[THINK]because two plus two is four[/THINK]The answer is 4.",
)
reasoning, content = _stream_parse_delta(
parser, mistral_tokenizer, gen_ids, request, prompt_ids
)
assert "because two plus two is four" in reasoning
assert "because two plus two is four" not in content
assert "The answer is 4." in content
def _encode_gen(tokenizer: MistralTokenizer, text: str) -> list[int]:
"""Encode `text` mapping `[THINK]`, `[/THINK]`, `[TOOL_CALLS]`, `[ARGS]`
to their special-token IDs; all other text uses the base tokenizer.
"""
vocab = tokenizer.get_vocab()
markers: dict[str, int] = {
"[THINK]": tokenizer.instruct.BEGIN_THINK,
"[/THINK]": tokenizer.instruct.END_THINK,
"[TOOL_CALLS]": vocab["[TOOL_CALLS]"],
"[ARGS]": vocab["[ARGS]"],
}
out: list[int] = []
remaining = text
while remaining:
earliest: int | None = None
earliest_marker: str | None = None
for marker in markers:
idx = remaining.find(marker)
if idx != -1 and (earliest is None or idx < earliest):
earliest = idx
earliest_marker = marker
if earliest_marker is None:
out += tokenizer.tokenizer.encode(remaining, False, False)
break
assert earliest is not None # set together with earliest_marker
if earliest > 0:
out += tokenizer.tokenizer.encode(remaining[:earliest], False, False)
out.append(markers[earliest_marker])
remaining = remaining[earliest + len(earliest_marker) :]
return out
def _stream_turn(
parser,
tokenizer: MistralTokenizer,
gen_ids: list[int],
request,
prompt_ids: list[int],
) -> tuple[str, str, list[tuple[str, str]]]:
"""Stream `gen_ids` through `parse_delta`, returning `(reasoning, content,
tool_calls)`.
`tool_calls` is a list of `(name, arguments)` tuples ordered by
`DeltaToolCall.index`. Arguments are concatenated across deltas.
"""
reasoning = ""
content = ""
tool_call_acc: dict[int, dict[str, str]] = {}
previous_tokens: list[str] | None = None
prefix_offset = 0
read_offset = 0
for i, tok in enumerate(gen_ids):
new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally(
tokenizer=tokenizer,
all_input_ids=gen_ids[: i + 1],
prev_tokens=previous_tokens,
prefix_offset=prefix_offset,
read_offset=read_offset,
skip_special_tokens=True,
spaces_between_special_tokens=True,
)
previous_tokens = (
previous_tokens + new_tokens if previous_tokens else new_tokens
)
delta_message = parser.parse_delta(
delta_text=delta_text,
delta_token_ids=[tok],
request=request,
prompt_token_ids=prompt_ids,
finished=(i == len(gen_ids) - 1),
)
if delta_message is not None:
reasoning += delta_message.reasoning or ""
content += delta_message.content or ""
for tc in delta_message.tool_calls:
if tc.index not in tool_call_acc:
tool_call_acc[tc.index] = {"name": "", "args": ""}
if tc.function:
if tc.function.name:
tool_call_acc[tc.index]["name"] = tc.function.name
if tc.function.arguments:
tool_call_acc[tc.index]["args"] += tc.function.arguments
tool_calls = [
(tool_call_acc[k]["name"], tool_call_acc[k]["args"])
for k in sorted(tool_call_acc)
]
return reasoning, content, tool_calls
def test_parse_delta_multi_turn_reason_and_tool(
mistral_tokenizer: MistralTokenizer,
) -> None:
"""Two-turn loop: each assistant turn carries reasoning + a tool call.
For each turn, the mistral-common conversation is encoded into `prompt_ids`
before streaming the next generation. Turn 2's prompt therefore contains
a `[THINK]`/`[/THINK]` block and `[TOOL_CALLS]` from turn 1 — the
accumulated tool-call/result history that single-turn tests never exercise.
Asserts that across both turns the parser correctly classifies think tokens
as `reasoning_content` (never leaking into `content`), and that the tool
call name/arguments are reconstructed correctly from the streaming deltas.
"""
parser_cls = ParserManager.get_parser(
tool_parser_name="mistral",
reasoning_parser_name="mistral",
enable_auto_tools=True,
)
request = parser_cls(mistral_tokenizer, None).adjust_request(
ChatCompletionRequest(
messages=[],
model="test",
tools=_SAMPLE_TOOLS,
tool_choice="auto",
)
)
assert getattr(request, "_grammar_from_parser", False)
cities = ["Dallas", "Paris"]
conversation: list = [UserMessage(content="What is the weather in Dallas?")]
for i, city in enumerate(cities):
# Fresh parser per turn: streaming state must not bleed across turns.
parser = parser_cls(mistral_tokenizer, None)
prompt_ids = mistral_tokenizer.instruct.encode_instruct(
InstructRequest(messages=conversation)
).tokens
gen_str = (
f"[THINK]I should check {city} weather.[/THINK]"
f'[TOOL_CALLS]get_weather[ARGS]{{"city": "{city}"}}'
)
gen_ids = _encode_gen(mistral_tokenizer, gen_str)
reasoning, content, tool_calls = _stream_turn(
parser, mistral_tokenizer, gen_ids, request, prompt_ids
)
assert f"check {city} weather" in reasoning, f"turn {i}: reasoning not captured"
assert f"check {city} weather" not in content, (
f"turn {i}: reasoning leaked into content"
)
assert "[THINK]" not in content, f"turn {i}: [THINK] leaked"
assert "[TOOL_CALLS]" not in content, f"turn {i}: [TOOL_CALLS] leaked"
assert len(tool_calls) == 1, f"turn {i}: expected 1 tool call"
name, args = tool_calls[0]
assert name == "get_weather", f"turn {i}: wrong tool name {name!r}"
assert city in args, f"turn {i}: city not in args {args!r}"
# Extend conversation with this turn's history for the next turn.
tc_id = f"tc{i:07d}" # 9-char alphanumeric id: "tc0000000", "tc0000001"
conversation.extend(
[
AssistantMessage(
content=[
ThinkChunk(
thinking=f"I should check {city} weather.",
closed=True,
)
],
tool_calls=[
ToolCall(
id=tc_id,
function=FunctionCall(
name="get_weather",
arguments=json.dumps({"city": city}),
),
)
],
),
ToolMessage(
content=f"{city}: 70F sunny",
tool_call_id=tc_id,
),
]
)
if i < len(cities) - 1:
conversation.append(UserMessage(content=f"And in {cities[i + 1]}?"))
@@ -1,348 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.reasoning.utils import run_reasoning_extraction_mistral
from vllm.reasoning import ReasoningParser, ReasoningParserManager
from vllm.tokenizers.mistral import MistralTokenizer
parser_name = "mistral"
@pytest.fixture(scope="module")
def mistral_tokenizer():
mistral_tokenizer = MistralTokenizer.from_pretrained(
"mistralai/Magistral-Small-2509"
)
return mistral_tokenizer
INVALID_SIMPLE_REASONING = {
"output": "This is a reasoning section[/THINK]This is the rest",
"reasoning": None,
"content": "This is a reasoning sectionThis is the rest",
"is_reasoning_end": False,
}
INVALID_COMPLETE_REASONING = {
"output": "This is a reasoning section[/THINK]",
"reasoning": None,
"content": "This is a reasoning section",
"is_reasoning_end": False,
}
NO_CONTENT = {
"output": "[THINK]This is reasoning",
"reasoning": "This is reasoning",
"content": None,
"is_reasoning_end": False,
}
NO_REASONING = {
"output": "This is content",
"reasoning": None,
"content": "This is content",
"is_reasoning_end": False,
}
NO_REASONING_STREAMING = {
"output": "This is a reasoning section",
"reasoning": None,
"content": "This is a reasoning section",
"is_reasoning_end": False,
}
INVALID_MULTIPLE_LINES = {
"output": "This\nThat[/THINK]This is the rest\nThat",
"reasoning": None,
"content": "This\nThatThis is the rest\nThat",
"is_reasoning_end": False,
}
INVALID_SHORTEST_REASONING_NO_STREAMING = {
"output": "[/THINK]This is the rest",
"reasoning": None,
"content": "This is the rest",
"is_reasoning_end": False,
}
INVALID_SHORTEST_REASONING = {
"output": "[/THINK]This is the rest",
"reasoning": None,
"content": "This is the rest",
"is_reasoning_end": False,
}
REASONING_WITH_THINK = {
"output": "[THINK]This is a reasoning section[/THINK]This is the rest",
"reasoning": "This is a reasoning section",
"content": "This is the rest",
"is_reasoning_end": True,
}
COMPLETE_REASONING_WITH_THINK = {
"output": "[THINK]This is a reasoning section[/THINK]",
"reasoning": "This is a reasoning section",
"content": None,
"is_reasoning_end": True,
}
MULTIPLE_LINES_WITH_THINK = {
"output": "[THINK]This\nThat[/THINK]This is the rest\nThat",
"reasoning": "This\nThat",
"content": "This is the rest\nThat",
"is_reasoning_end": True,
}
INVALID_SHORTEST_REASONING_NO_STREAMING_WITH_THINK = {
"output": "[/THINK]This is the rest",
"reasoning": None,
"content": "This is the rest",
"is_reasoning_end": False,
}
INVALID_SHORTEST_REASONING_WITH_THINK = {
"output": "[/THINK]This is the rest",
"reasoning": None,
"content": "This is the rest",
"is_reasoning_end": False,
}
THINK_NO_END = {
"output": "[THINK]This is a reasoning section",
"reasoning": "This is a reasoning section",
"content": None,
"is_reasoning_end": False,
}
EMPTY = {
"output": "",
"reasoning": None,
"content": "",
"is_reasoning_end": False,
}
EMPTY_STREAMING = {
"output": "",
"reasoning": None,
"content": None,
"is_reasoning_end": False,
}
NEW_LINE = {
"output": "Before\n[THINK]This is a reasoning section[/THINK]\nThis is the rest",
"reasoning": "This is a reasoning section",
"content": "Before\n\nThis is the rest",
"is_reasoning_end": True,
}
NEW_LINE_STREAMING = {
"output": "Before\n[THINK]This is a reasoning section[/THINK]\nThis is the rest",
"reasoning": "This is a reasoning section",
"content": "Before\n\nThis is the rest",
"is_reasoning_end": True,
}
TEST_CASES = [
pytest.param(
False,
INVALID_SIMPLE_REASONING,
id="invalid_simple_reasoning",
),
pytest.param(
True,
INVALID_SIMPLE_REASONING,
id="invalid_simple_reasoning_streaming",
),
pytest.param(
False,
INVALID_COMPLETE_REASONING,
id="invalid_complete_reasoning",
),
pytest.param(
True,
INVALID_COMPLETE_REASONING,
id="invalid_complete_reasoning_streaming",
),
pytest.param(
False,
NO_CONTENT,
id="no_content",
),
pytest.param(
False,
NO_REASONING,
id="no_reasoning",
),
pytest.param(
True,
NO_REASONING_STREAMING,
id="no_reasoning_token_streaming",
),
pytest.param(
False,
INVALID_MULTIPLE_LINES,
id="invalid_multiple_lines",
),
pytest.param(
True,
INVALID_MULTIPLE_LINES,
id="invalid_multiple_lines_streaming",
),
pytest.param(
True,
INVALID_SHORTEST_REASONING,
id="invalid_shortest",
),
pytest.param(
False,
INVALID_SHORTEST_REASONING_NO_STREAMING,
id="invalid_shortest_streaming",
),
pytest.param(
False,
REASONING_WITH_THINK,
id="reasoning_with_think",
),
pytest.param(
True,
REASONING_WITH_THINK,
id="reasoning_with_think_streaming",
),
pytest.param(
False,
COMPLETE_REASONING_WITH_THINK,
id="complete_reasoning_with_think",
),
pytest.param(
True,
COMPLETE_REASONING_WITH_THINK,
id="complete_reasoning_with_think_streaming",
),
pytest.param(
False,
MULTIPLE_LINES_WITH_THINK,
id="multiple_lines_with_think",
),
pytest.param(
True,
MULTIPLE_LINES_WITH_THINK,
id="multiple_lines_with_think_streaming",
),
pytest.param(
False,
INVALID_SHORTEST_REASONING_NO_STREAMING_WITH_THINK,
id="invalid_shortest_with_think",
),
pytest.param(
True,
INVALID_SHORTEST_REASONING_WITH_THINK,
id="invalid_shortest_with_think_streaming",
),
pytest.param(
False,
THINK_NO_END,
id="think_no_end",
),
pytest.param(
True,
THINK_NO_END,
id="think_no_end_streaming",
),
pytest.param(
False,
EMPTY,
id="empty",
),
pytest.param(
True,
EMPTY_STREAMING,
id="empty_streaming",
),
pytest.param(
False,
NEW_LINE,
id="new_line",
),
pytest.param(
True,
NEW_LINE_STREAMING,
id="new_line_streaming",
),
]
@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)
def test_mistral_reasoning(
streaming: bool,
param_dict: dict,
mistral_tokenizer: MistralTokenizer,
):
output = param_dict["output"]
index_think = output.find("[THINK]")
len_think = len("[THINK]")
index_end_think = output.find("[/THINK]")
len_end_think = len("[/THINK]")
# encode everything to tokens ids
output_tokens = []
if index_think != -1:
output_before_think = output[:index_think]
output_tokens += mistral_tokenizer.tokenizer.encode(
output_before_think, False, False
)
output_tokens += [mistral_tokenizer.instruct.BEGIN_THINK]
if index_end_think != -1:
output_middle = output[index_think + len_think : index_end_think]
output_after_think = output[index_end_think + len_end_think :]
output_tokens += mistral_tokenizer.tokenizer.encode(
output_middle, False, False
)
output_tokens += [mistral_tokenizer.instruct.END_THINK]
output_tokens += mistral_tokenizer.tokenizer.encode(
output_after_think, False, False
)
else:
output_middle = output[index_think + len_think :]
output_tokens += mistral_tokenizer.tokenizer.encode(
output_middle, False, False
)
elif index_end_think != -1:
output_before_think = output[:index_end_think]
output_after_think = output[index_end_think + len_end_think :]
output_tokens += mistral_tokenizer.tokenizer.encode(
output_before_think, False, False
)
output_tokens += [mistral_tokenizer.instruct.END_THINK]
output_tokens += mistral_tokenizer.tokenizer.encode(
output_after_think, False, False
)
else:
output_tokens += mistral_tokenizer.tokenizer.encode(output, False, False)
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)(
mistral_tokenizer
)
reasoning, content = run_reasoning_extraction_mistral(
parser, output_tokens, streaming=streaming
)
assert reasoning == param_dict["reasoning"]
assert content == param_dict["content"]
# Test is_reasoning_end
is_reasoning_end = parser.is_reasoning_end(output_tokens)
assert is_reasoning_end == param_dict["is_reasoning_end"]
# Test extract_content
if param_dict["content"] is not None:
# Handle the case where there are tokens outputted before Thinking.
# This should not occur if the model is well trained and prompted.
if "[THINK]" in param_dict["output"] and not param_dict["output"].startswith(
"[THINK]"
):
before_content = param_dict["output"].split("[THINK]")[0]
before_token_ids = mistral_tokenizer.tokenizer.encode(
before_content, bos=False, eos=False
)
left_to_encode = param_dict["content"][len(before_content) :]
# Normal situation.
else:
before_token_ids = []
left_to_encode = param_dict["content"]
content_tokens = parser.extract_content_ids(output_tokens)
expected_token_ids = before_token_ids + mistral_tokenizer.tokenizer.encode(
left_to_encode, bos=False, eos=False
)
assert content_tokens == expected_token_ids
else:
content = parser.extract_content_ids(output_tokens)
assert content == []
+144 -2
View File
@@ -8,11 +8,13 @@ import llguidance
import pytest
from mistral_common.exceptions import InvalidMessageStructureException
from mistral_common.guidance.grammar_factory import GrammarFactory
from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy, SpecialTokens
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.tokenizers.mistral import (
MistralTokenizer,
_validate_apply_chat_template_args,
validate_request_params,
)
@@ -1247,7 +1249,9 @@ class TestMistralTokenizer:
expected_strings = (
'[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}] I am an AI\n\nHello world ![TOOL_CALLS][{"name": "get_weather", "arguments": {"city": "Paris"}, "id": "123456789"}] {"content": {"temperature": 20, "unit": "celsius"}, "call_id": "123456789"}', # noqa: E501
'I am an AI[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}]Hello world ![TOOL_CALLS]get_weather{"city": "Paris"}{"temperature": 20, "unit": "celsius"}', # noqa: E501
# v11+ tool-call decode emits the explicit [ARGS] separator between
# the function name and its JSON arguments (get_weather[ARGS]{...}).
'I am an AI[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}]Hello world ![TOOL_CALLS]get_weather[ARGS]{"city": "Paris"}{"temperature": 20, "unit": "celsius"}', # noqa: E501
)
assert (
@@ -1498,6 +1502,7 @@ class TestMistralTokenizer:
"get",
"_",
"weather",
"[ARGS]",
'{"',
"city",
'":',
@@ -2234,3 +2239,140 @@ class TestMistralTokenizer:
decoded = mistral_tokenizer.tokenizer.decode(output, SpecialTokenPolicy.KEEP)
assert "[THINK]2+2 equals 4[/THINK]" in decoded
def test_convert_ids_to_tokens_pre_args_tekken():
"""convert_ids_to_tokens must not raise on pre-[ARGS] Tekken tokenizers.
Tekken tokenizers before v11 (e.g. Ministral-8B-Instruct-2410) have no
[ARGS] special token. This guards the is_special([ARGS]) gate in
MistralTokenizer.convert_ids_to_tokens, which must skip [ARGS] rather than
call get_special_token(args) unconditionally (which raises on Tekken).
"""
tokenizer = MistralTokenizer.from_pretrained("mistralai/Ministral-8B-Instruct-2410")
assert tokenizer.is_tekken
assert not tokenizer.tokenizer.is_special(SpecialTokens.args)
ids = tokenizer.encode("Hello world !", add_special_tokens=False)
tokens = tokenizer.convert_ids_to_tokens(ids, skip_special_tokens=True)
assert len(tokens) > 0
# ---------------------------------------------------------------------------
# validate_request_params – reasoning_effort validation (no tokenizer needed)
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("reasoning_effort", [None, "none", "high"])
def test_validate_request_params_valid_effort(reasoning_effort: str | None) -> None:
request = ChatCompletionRequest(
model="test-model",
messages=[],
reasoning_effort=reasoning_effort,
)
assert validate_request_params(request) is None
def test_validate_request_params_rejects_unsupported_effort() -> None:
request = ChatCompletionRequest(
model="test-model",
messages=[],
reasoning_effort="medium",
)
with pytest.raises(ValueError, match="reasoning_effort"):
validate_request_params(request)
# ---------------------------------------------------------------------------
# apply_chat_template reasoning_effort passthrough (v15 tokenizer)
# ---------------------------------------------------------------------------
_PASSTHROUGH_MESSAGES = [{"role": "user", "content": "Hello"}]
@pytest.fixture(scope="module")
def v15_mistral_tokenizer() -> MistralTokenizer:
"""Load the v15 Mistral tokenizer; skip if unavailable."""
try:
return MistralTokenizer.from_pretrained("mistralai/Mistral-Small-4-119B-2603")
except Exception:
pytest.skip("v15 tokenizer unavailable")
@pytest.fixture(scope="module")
def v13_mistral_tokenizer() -> MistralTokenizer:
"""Load the v13 Mistral tokenizer; skip if unavailable."""
try:
return MistralTokenizer.from_pretrained("mistralai/Magistral-Small-2509")
except Exception:
pytest.skip("v13 tokenizer unavailable")
def test_v15_apply_chat_template_passes_reasoning_effort_high(
monkeypatch: pytest.MonkeyPatch,
v15_mistral_tokenizer: MistralTokenizer,
) -> None:
captured_kwargs: list[dict] = []
def fake_apply_chat_template(**kwargs):
captured_kwargs.append(kwargs)
return [0]
monkeypatch.setattr(
v15_mistral_tokenizer.transformers_tokenizer,
"apply_chat_template",
fake_apply_chat_template,
)
v15_mistral_tokenizer.apply_chat_template(
messages=_PASSTHROUGH_MESSAGES,
reasoning_effort="high",
)
assert captured_kwargs[-1]["reasoning_effort"] == "high"
def test_v15_apply_chat_template_passes_reasoning_effort_none_by_default(
monkeypatch: pytest.MonkeyPatch,
v15_mistral_tokenizer: MistralTokenizer,
) -> None:
captured_kwargs: list[dict] = []
def fake_apply_chat_template(**kwargs):
captured_kwargs.append(kwargs)
return [0]
monkeypatch.setattr(
v15_mistral_tokenizer.transformers_tokenizer,
"apply_chat_template",
fake_apply_chat_template,
)
v15_mistral_tokenizer.apply_chat_template(messages=_PASSTHROUGH_MESSAGES)
assert "reasoning_effort" in captured_kwargs[-1]
assert captured_kwargs[-1]["reasoning_effort"] is None
def test_pre_v15_apply_chat_template_omits_reasoning_effort(
monkeypatch: pytest.MonkeyPatch,
v13_mistral_tokenizer: MistralTokenizer,
) -> None:
captured_kwargs: list[dict] = []
def fake_apply_chat_template(**kwargs):
captured_kwargs.append(kwargs)
return [0]
monkeypatch.setattr(
v13_mistral_tokenizer.transformers_tokenizer,
"apply_chat_template",
fake_apply_chat_template,
)
v13_mistral_tokenizer.apply_chat_template(
messages=_PASSTHROUGH_MESSAGES,
reasoning_effort="high",
)
assert "reasoning_effort" not in captured_kwargs[-1]
@@ -219,7 +219,7 @@ _NOT_SET = object()
pytest.param(
[WEATHER_TOOL],
"required",
30,
9,
id="required",
),
],
@@ -533,8 +533,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
msg["tool_calls"] = list(tool_calls)
return self
_grammar_from_tool_parser: bool = PrivateAttr(default=False)
"""CAUTION: Should only be set by ``ToolParser.adjust_request``."""
_grammar_from_parser: bool = PrivateAttr(default=False)
"""CAUTION: Should only be set by the parser-engine adapter's adjust_request."""
def build_chat_params(
self,
@@ -65,7 +65,6 @@ from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.utils.collection_utils import as_list
from vllm.utils.mistral import is_mistral_tool_parser
logger = init_logger(__name__)
@@ -157,15 +156,6 @@ class OpenAIServingChat(GenerateBaseServing):
model_name=self.model_config.model,
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
)
if (
self.parser_cls is not None
and is_mistral_tool_parser(self.parser_cls.tool_parser_cls)
and self.parser_cls.reasoning_parser_cls is not None
):
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
MistralToolParser.model_can_reason = True
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
self.enable_prompt_tokens_details = enable_prompt_tokens_details
@@ -350,7 +340,7 @@ class OpenAIServingChat(GenerateBaseServing):
else:
if not request.include_reasoning:
reasoning_ended = True
elif request._grammar_from_tool_parser:
elif request._grammar_from_parser:
# The Mistral grammar already includes an optional
# `think?` rule that handles both reasoning and
# non-reasoning outputs.
@@ -284,7 +284,7 @@ class DerenderChatRequest(BaseModel):
Required by the parsing so that tool/reasoning parsers can receive the full
request context they expect (request.tools, request.tool_choice,
request._grammar_from_tool_parser, etc.).
request._grammar_from_parser, etc.).
"""
# --8<-- [end:derender-chat-request]
+7 -4
View File
@@ -396,6 +396,9 @@ class ParserEngine(Parser):
return True
return find_tool_name(self._tools, name)
def _accept_tool_name(self, name: str) -> bool:
return bool(name) and self._is_valid_tool_name(name)
# ── Private helpers ─────────────────────────────────────────────
def _check_skip_tool_parsing(
@@ -807,7 +810,7 @@ class ParserEngine(Parser):
deltas: list[DeltaToolCall],
name: str | None,
) -> None:
if not name or not self._is_valid_tool_name(name):
if name is None or not self._accept_tool_name(name):
return
slot = self._tool_slots[idx]
slot.name = name
@@ -866,8 +869,8 @@ class ParserEngine(Parser):
slot = self._tool_slots[idx]
if not slot.name_sent:
name = slot.name or self._try_extract_name(idx)
if name and self._is_valid_tool_name(name):
name = slot.name or self._try_extract_name(idx) or ""
if self._accept_tool_name(name):
slot.name = name
slot.name_sent = True
slot.string_keys = self._streamable_string_keys(
@@ -1042,7 +1045,7 @@ class ParserEngine(Parser):
else:
args_json = "{}"
if name and self._is_valid_tool_name(name):
if self._accept_tool_name(name):
self._ensure_tool_id(slot, name)
args_json = self._fix_arg_types(args_json, name)
tool_calls.append(
@@ -15,6 +15,7 @@ from vllm.parser.glm47_moe import Glm47MoeParser
from vllm.parser.inkling import InklingParser
from vllm.parser.kimi_k2 import KimiK2Parser
from vllm.parser.minimax_m2 import MinimaxM2Parser
from vllm.parser.mistral import MistralParser
from vllm.parser.nemotron_v3 import NemotronV3Parser
from vllm.parser.qwen3 import Qwen3Parser
from vllm.parser.seed_oss import SeedOssParser
@@ -68,3 +69,8 @@ from vllm.parser.seed_oss import SeedOssParser
InklingParserReasoningAdapter,
InklingParserToolAdapter,
) = make_adapters(InklingParser)
(
MistralParserReasoningAdapter,
MistralParserToolAdapter,
) = make_adapters(MistralParser)
+949 -57
View File
File diff suppressed because it is too large Load Diff
-9
View File
@@ -109,8 +109,6 @@ class ParserManager:
if reasoning_parser_cls is None and tool_parser_cls is None:
return None
from vllm.utils.mistral import is_mistral_tool_parser
if is_harmony:
from vllm.parser.harmony import HarmonyParser
@@ -118,13 +116,6 @@ class ParserManager:
HarmonyParser.tool_parser_cls = tool_parser_cls
return HarmonyParser
if is_mistral_tool_parser(tool_parser_cls):
from vllm.parser.mistral import MistralParser
MistralParser.reasoning_parser_cls = reasoning_parser_cls
MistralParser.tool_parser_cls = tool_parser_cls
return MistralParser
if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":
from vllm.parser.kimi_k3 import KimiK3Parser
+1 -1
View File
@@ -106,7 +106,7 @@ _REASONING_PARSERS_TO_REGISTER = {
),
"mistral": (
"mistral_reasoning_parser",
"MistralReasoningParser",
"MistralParserReasoningAdapter",
),
"nemotron_v3": (
"nemotron_v3_engine_reasoning_parser",
+2 -158
View File
@@ -1,162 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Sequence
from functools import cached_property
from typing import TYPE_CHECKING
from vllm.parser.engine.registered_adapters import MistralParserReasoningAdapter
from vllm.reasoning import ReasoningParser
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
from vllm.tokenizers.mistral import MistralTokenizer
if TYPE_CHECKING:
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
class MistralReasoningParser(BaseThinkingReasoningParser):
"""
Reasoning parser for Mistral models.
The Mistral models uses `[THINK]`...`[/THINK]` tokens to denote reasoning
text. This parser extracts the reasoning content from the model output.
A valid reasoning trace should always start with a `[THINK]` token and end with
a `[/THINK]` token.
If `[THINK]` token is not generated, then this parser only returns content.
"""
def __init__(self, tokenizer: MistralTokenizer, *args, **kwargs):
if not isinstance(tokenizer, MistralTokenizer):
raise ValueError("The tokenizer must be an instance of MistralTokenizer.")
ReasoningParser.__init__(self, tokenizer, *args, **kwargs)
if not self.model_tokenizer:
raise ValueError(
"The model tokenizer must be passed to the ReasoningParser "
"constructor during construction."
)
self.start_token_id = tokenizer.tokenizer.get_special_token(self.start_token)
self.end_token_id = tokenizer.tokenizer.get_special_token(self.end_token)
if self.start_token_id is None or self.end_token_id is None:
raise RuntimeError(
"Mistral reasoning parser could not locate think start/end "
"tokens in the tokenizer!"
)
@cached_property
def start_token(self) -> str:
"""The token that starts reasoning content."""
from mistral_common.tokens.tokenizers.base import SpecialTokens
return SpecialTokens.begin_think
@cached_property
def end_token(self) -> str:
"""The token that ends reasoning content."""
from mistral_common.tokens.tokenizers.base import SpecialTokens
return SpecialTokens.end_think
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
has_eot_token = False
for id in reversed(input_ids):
if id == self.start_token_id:
# Reasoning ends only if a BOT token is found before a EOT token.
return has_eot_token
elif id == self.end_token_id:
has_eot_token = True
return False
def is_reasoning_end_streaming(
self, input_ids: Sequence[int], delta_ids: Iterable[int]
) -> bool:
if self.end_token_id in delta_ids:
return True
# Grammar's think? is optional — if [THINK] was never generated,
# reasoning was skipped entirely.
return self.start_token_id not in input_ids
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
"""
Extract the content
"""
has_bot_token = False
has_eot_token = False
bot_token_index = -1
eot_token_index = -1
# One for loop instead of multiple lookups
for i, token_id in enumerate(input_ids):
# We filter that we have multiple BOT tokens which should not
# happen for a well prompted trained model
if token_id == self.start_token_id and not has_bot_token:
has_bot_token = True
bot_token_index = i
elif token_id == self.end_token_id:
has_eot_token = True
eot_token_index = i
break
# 1. Only BOT has been outputted
if has_bot_token and not has_eot_token:
# Should be = [] if model is well prompted and trained.
return input_ids[:bot_token_index]
# 2. Neither BOT or EOT have been outputted
elif not has_bot_token and not has_eot_token:
return input_ids
# 3. Both BOT and EOT have been outputted.
elif has_bot_token and has_eot_token:
return input_ids[:bot_token_index] + input_ids[eot_token_index + 1 :]
# 4. Only EOT has been outputted => this should not have occurred for a model
# well prompted and trained.
else:
return input_ids[:eot_token_index] + input_ids[eot_token_index + 1 :]
def extract_reasoning(
self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest"
) -> tuple[str | None, str | None]:
"""
Extract reasoning content from the model output.
"""
if not model_output:
return (None, "")
# Check if the start token is present in the model output, remove it
# if it is present.
prev_bot_token, bot_token, post_bot_token = model_output.partition(
self.start_token
)
has_bot_token = bool(bot_token)
# Valid EOT tokens should follow BOT token
has_valid_eot_token = has_bot_token and self.end_token in post_bot_token
# 1. If there is BOT token followed by EOT token
if has_bot_token and has_valid_eot_token:
prev_eot_token, _, post_eot_token = post_bot_token.partition(self.end_token)
# If model is well prompted and trained prev_bot_token should be ""
content = prev_bot_token + post_eot_token
return prev_eot_token, content if content else None
# 2. Only BOT token
elif has_bot_token:
# If model is well prompted and trained prev_bot_token should be ""
return post_bot_token, prev_bot_token if prev_bot_token else None
# 3. EOT token has been outputted without BOT or neither has been outputted
else:
has_non_valid_eot_token = self.end_token in prev_bot_token
# 3.a EOT token has been outputted without BOT
# If model is well prompted and trained `has_non_valid_eot_token` should
# be `False` and the parser outputs all tokens as 'content'
if has_non_valid_eot_token:
prev_eot_token, _, post_eot_token = prev_bot_token.partition(
self.end_token
)
return None, prev_eot_token + post_eot_token
# 3.b neither BOT or EOT have been outputted
else:
return None, prev_bot_token
__all__ = ["MistralParserReasoningAdapter"]
+15 -6
View File
@@ -21,7 +21,6 @@ from mistral_common.tokens.tokenizers.base import (
)
from mistral_common.tokens.tokenizers.instruct import (
InstructTokenizerBase,
InstructTokenizerV13,
)
from mistral_common.tokens.tokenizers.mistral import (
MistralTokenizer as MistralCommonTokenizer,
@@ -461,6 +460,7 @@ class MistralTokenizer(TokenizerLike):
def convert_tokens_to_string(self, tokens: list[str]) -> str:
to_decode_special_tokens = {
SpecialTokens.tool_calls,
SpecialTokens.args,
SpecialTokens.begin_think,
SpecialTokens.end_think,
}
@@ -517,11 +517,20 @@ class MistralTokenizer(TokenizerLike):
non_skip_special_tokens_ids = {
self.tokenizer.get_special_token(SpecialTokens.tool_calls),
}
if isinstance(self.instruct, InstructTokenizerV13):
if self.instruct.BEGIN_THINK:
non_skip_special_tokens_ids.add(self.instruct.BEGIN_THINK)
if self.instruct.END_THINK:
non_skip_special_tokens_ids.add(self.instruct.END_THINK)
# [ARGS] only exists in v11+ tool-call tokenizers; older tokenizers
# raise (Tekken) or return unk (SPM) for it.
if self.tokenizer.is_special(SpecialTokens.args):
non_skip_special_tokens_ids.add(
self.tokenizer.get_special_token(SpecialTokens.args)
)
# [THINK]/[/THINK] only exist in v13+ reasoning tokenizers; use the
# same is_special gate as [ARGS] above so newer versions are covered
# without an isinstance check on the instruct tokenizer.
for think_token in (SpecialTokens.begin_think, SpecialTokens.end_think):
if self.tokenizer.is_special(think_token):
non_skip_special_tokens_ids.add(
self.tokenizer.get_special_token(think_token)
)
ids_kept = [
i
+26 -756
View File
@@ -3,767 +3,37 @@
from __future__ import annotations
import json
from collections.abc import Sequence
from enum import Enum, auto
from random import choices
from string import ascii_letters, digits
from typing import Any
from typing import TYPE_CHECKING
import ijson
import regex as re
from mistral_common.protocol.instruct.tool_calls import (
NamedToolChoice as MistralNamedToolChoice,
)
from mistral_common.protocol.instruct.tool_calls import (
Tool as MistralTool,
)
from mistral_common.protocol.instruct.tool_calls import (
ToolChoice as MistralToolChoice,
)
from mistral_common.protocol.instruct.tool_calls import (
ToolChoiceEnum as MistralToolChoiceEnum,
)
from pydantic import Field
from vllm.parser.engine.registered_adapters import MistralParserToolAdapter
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.engine.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
ExtractedToolCallInformation,
FunctionCall,
ToolCall,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.logger import init_logger
from vllm.sampling_params import StructuredOutputsParams
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import (
Tool,
ToolParser,
)
from vllm.utils.mistral import is_mistral_tokenizer
logger = init_logger(__name__)
ALPHANUMERIC = ascii_letters + digits
_DEFAULT_JSON_SCHEMA = {"anyOf": [{"type": "object"}, {"type": "array"}]}
if TYPE_CHECKING:
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
class StreamingState(Enum):
"""Enum for tracking the current streaming parsing state."""
class MistralToolParser(MistralParserToolAdapter): # type: ignore[valid-type, misc]
# Marked so is_mistral_tool_parser() in vllm.utils.mistral recognises
# this adapter and lets adjust_request() fire for tool_choice="none"
# on grammar-capable Mistral tokenizers.
IS_MISTRAL_TOOL_PARSER = True
WAITING_FOR_TOOL_START = auto()
WAITING_FOR_TOOL_KEY = (
auto()
) # waiting for the "name" or "arguments" key to be complete
PARSING_NAME = auto()
PARSING_NAME_COMPLETED = auto()
WAITING_FOR_ARGUMENTS_START = auto()
PARSING_ARGUMENTS = auto()
PARSING_ARGUMENTS_COMPLETED = auto()
TOOL_COMPLETE = auto()
ALL_TOOLS_COMPLETE = auto()
class MistralToolCall(ToolCall):
id: str = Field(default_factory=lambda: MistralToolCall.generate_random_id())
@staticmethod
def generate_random_id():
# Mistral Tool Call Ids must be alphanumeric with a length of 9.
# https://github.com/mistralai/mistral-common/blob/21ee9f6cee3441e9bb1e6ed2d10173f90bd9b94b/src/mistral_common/protocol/instruct/validator.py#L299
return "".join(choices(ALPHANUMERIC, k=9))
@staticmethod
def is_valid_id(id: str) -> bool:
return id.isalnum() and len(id) == 9
def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool:
if is_mistral_tokenizer(model_tokenizer):
return model_tokenizer.version < 11
# For HF tokenizers, check if [ARGS] token exists in vocab
# which indicates a v11+ equivalent tokenizer
vocab: dict[str, int] = getattr(model_tokenizer, "get_vocab", lambda: {})()
return "[ARGS]" not in vocab
class MistralToolParser(ToolParser):
r"""Tool call parser for Mistral models, intended for use with either:
- `mistral_common <https://github.com/mistralai/mistral-common/>`_
(recommended)
- the `examples/tool_chat_template_mistral.jinja` template.
Used when `--enable-auto-tool-choice --tool-call-parser mistral` are all
set.
"""
IS_MISTRAL_TOOL_PARSER = True # used by vllm.utils.mistral
# Used to generate correct grammar in `adjust_request`
model_can_reason: bool = False
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
super().__init__(tokenizer, tools)
if not is_mistral_tokenizer(self.model_tokenizer):
logger.info("Non-Mistral tokenizer detected when using a Mistral model...")
# initialize properties used for state when parsing tool calls in
# streaming mode
self.prev_tool_call_arr: list[dict[str, Any]] = []
self.current_tool_id: int = -1
self.streaming_state: StreamingState = StreamingState.WAITING_FOR_TOOL_START
# For streaming pre v11 tokenizer tool calls
self.current_tool_name: str | None = None
self.current_tool_mistral_id: str | None = None
self.starting_new_tool = False
self._is_pre_v11 = _is_pre_v11_tokeniser(self.model_tokenizer)
if self._is_pre_v11:
self.parse_coro = ijson.parse_coro(
self.update_stream_state_pre_v11_tokenizer()
)
self.bot_token = "[TOOL_CALLS]"
self.bot_token_id = self.vocab.get(self.bot_token)
self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL)
if self.bot_token_id is None:
raise RuntimeError(
"Mistral Tool Parser could not locate the tool call token in "
"the tokenizer!"
)
# Mistral emits tool calls in the [TOOL_CALLS]name[ARGS]{...} format, not the
# plain JSON array / bare-args that the serving layer's generic required and
# named handlers expect. Route required/named through this parser's own
# extraction (treated like "auto") so the [TOOL_CALLS] format is parsed
# correctly instead of crashing or dumping the raw envelope into arguments.
supports_required_and_named = False
def adjust_request(
self, request: ChatCompletionRequest | ResponsesRequest
self,
request: ChatCompletionRequest | ResponsesRequest,
) -> ChatCompletionRequest | ResponsesRequest:
so_non_supported_attributes = [
"regex",
"choice",
"grammar",
# whitespace_pattern is not a constraint type but an option;
# Mistral grammar factory does not support it.
"whitespace_pattern",
"structural_tag",
]
any_so_non_supported_active = request.structured_outputs is not None and any(
getattr(request.structured_outputs, attribute) is not None
for attribute in so_non_supported_attributes
)
response_format_non_supported_active = (
isinstance(request, ResponsesRequest)
or request.response_format is not None
and request.response_format.type == "structural_tag"
)
if (
not is_mistral_tokenizer(self.model_tokenizer)
or isinstance(request, ResponsesRequest)
or not self.model_tokenizer.supports_grammar
or any_so_non_supported_active
or response_format_non_supported_active
):
request = super().adjust_request(request)
if request.tools and request.tool_choice != "none":
# Do not skip special tokens when using chat template
# with Mistral parser as TOOL_CALL token is needed
# for tool detection.
# Note: we don't want skip_special_tokens=False
# with MistralTokenizer as it is incompatible
request.skip_special_tokens = False
return request
json_schema: dict[str, Any] | None = None
if request.structured_outputs is not None:
if request.structured_outputs.json_object is not None:
json_schema = _DEFAULT_JSON_SCHEMA
elif request.structured_outputs.json is not None:
if isinstance(request.structured_outputs.json, str):
json_schema = json.loads(request.structured_outputs.json)
else:
json_schema = request.structured_outputs.json
else:
raise ValueError(
"Unsupported request.structured_outputs for MistralToolParser. "
"Only `json` and `json_object` are supported."
)
elif (
request.response_format is not None
and request.response_format.type != "text"
):
if request.response_format.type == "json_object":
json_schema = _DEFAULT_JSON_SCHEMA
elif request.response_format.type == "json_schema":
if request.response_format.json_schema is not None:
json_schema = request.response_format.json_schema.json_schema
else:
json_schema = _DEFAULT_JSON_SCHEMA
else:
raise ValueError(
"MistralToolParser only accepts `text`, `json_object` or "
f"`json_schema`, got {request.response_format=}"
)
# Structured Outputs will be defined.
request.response_format = None
grammar_factory = self.model_tokenizer.grammar_factory
# TODO: Once unified parser, improve this.
# The issue is figuring out when a model is a reasoning one or not.
template = grammar_factory.select_jinja_template(
reasoning=self.model_can_reason
)
mistral_tools = (
[MistralTool.from_openai(tool.model_dump()) for tool in request.tools]
if request.tools is not None
else None
)
tool_choice: MistralToolChoice
match request.tool_choice:
case "none" | "auto" | "required":
tool_choice = MistralToolChoiceEnum(request.tool_choice)
case None:
tool_choice = MistralToolChoiceEnum.auto
# _ == Named tool choice
case _:
tool_choice = MistralNamedToolChoice.model_validate(
{
"type": "function",
"function": {"name": request.tool_choice.function.name},
}
)
# Rendering grammar is cached in mistral-common given tools, template and mode.
match tool_choice, json_schema is not None:
case MistralToolChoiceEnum.none, True:
lark_grammar = grammar_factory.get_lark_for_json_schema(
template=template, json_schema=json_schema
)
case _, _:
lark_grammar = grammar_factory.get_lark_from_jinja(
template=template,
mode=tool_choice,
tools=mistral_tools,
json_schema=json_schema,
parallel_tool_calls=request.parallel_tool_calls,
json_only=False,
)
request.structured_outputs = StructuredOutputsParams(grammar=lark_grammar)
request._grammar_from_tool_parser = True
return request
def extract_tool_calls(
self,
model_output: str,
request: ChatCompletionRequest,
) -> ExtractedToolCallInformation:
"""
Extract the tool calls from a complete model response.
Content and tool calls formatting depends on the Mistral's tokenizer version
used to train the model:
- < v11: `content[BOT] [{tool_call1},{tool_call2}]`
- >= v11: `content[BOT]tool_name1{args_call1}[BOT]tool_name2{args_call2}`
with [BOT] the tool call token.
Note:
For tokenizer versions >= v11, tool calls with arguments wrongly formatted
are still returned as tool calls. This is to allow the model to know it
tried to make a tool call. It reduces chance of another failure and
prevents that the context is filled with tool calls wrongly placed in
assistant message contents.
"""
# If the tool call token is not present, return a text response
if self.bot_token not in model_output:
return ExtractedToolCallInformation(
tools_called=False, tool_calls=[], content=model_output
)
content_and_raw_tool_calls = model_output.split(self.bot_token)
content = content_and_raw_tool_calls[0]
raw_tool_calls = content_and_raw_tool_calls[1:]
# >= v11: content[BOT]tool_name1{args_call1}[BOT]tool_name2{args_call2}
if not self._is_pre_v11:
tool_calls = []
for raw_tool_call in raw_tool_calls:
if "{" not in raw_tool_call:
continue
end_name = raw_tool_call.find("{")
tool_name, args = (
raw_tool_call[:end_name],
raw_tool_call[end_name:],
)
# HF tokenizers may include [ARGS] in the text
tool_name = tool_name.replace("[ARGS]", "")
tool_calls.append({"name": tool_name, "arguments": args})
# < v11: content[BOT] [{tool_call1},{tool_call2}]
else:
if len(raw_tool_calls) != 1:
raise ValueError(
"Only one BOT token should have been outputted, "
f"but got {model_output}."
)
stringified_tool_calls = raw_tool_calls[0].strip()
try:
# Use raw_decode to parse the first valid JSON value,
# ignoring trailing tokens the model may emit after
# the tool call array.
tool_calls, _ = json.JSONDecoder().raw_decode(stringified_tool_calls)
except json.JSONDecodeError:
try:
raw_tool_call = self.tool_call_regex.findall(
stringified_tool_calls
)[0]
tool_calls = json.loads(raw_tool_call)
tool_calls = [
{
"name": tool_call["name"],
"arguments": json.dumps(
tool_call.get("arguments", {}),
ensure_ascii=False,
),
}
for tool_call in tool_calls
]
except (IndexError, json.JSONDecodeError):
logger.exception("Error in extracting tool call from response.")
return ExtractedToolCallInformation(
tools_called=False,
tool_calls=[],
content=stringified_tool_calls,
)
else:
tool_calls = [
{
"name": tool_call["name"],
"arguments": json.dumps(
tool_call.get("arguments", {}),
ensure_ascii=False,
),
}
for tool_call in tool_calls
]
mistral_tool_calls: list[MistralToolCall] = [
MistralToolCall(
type="function",
function=FunctionCall(
name=tool_call["name"],
arguments=tool_call.get("arguments", "{}"),
),
)
for tool_call in tool_calls
]
return ExtractedToolCallInformation(
tools_called=True,
tool_calls=mistral_tool_calls,
content=content if content.strip() else None,
)
def extract_tool_calls_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> DeltaMessage | None:
has_bot_token = (
self.bot_token_id in current_token_ids or self.bot_token in current_text
)
if not has_bot_token:
# if the tool call token is not in the tokens generated so far,
# append output to contents since it's not a tool
return DeltaMessage(content=delta_text)
# if the tool call token IS in the tokens generated so far, that
# means we're parsing as tool calls now
try:
if self._is_pre_v11:
return self._extract_tool_calls_streaming_pre_v11_tokenizer(
delta_text=delta_text,
delta_token_ids=delta_token_ids,
)
else:
return self._extract_tool_calls_streaming(
delta_text=delta_text, delta_token_ids=delta_token_ids
)
except Exception:
logger.exception("Error trying to handle streaming tool call.")
return None
def _extract_tool_calls_streaming(
self,
delta_text: str,
delta_token_ids: Sequence[int],
) -> DeltaMessage | None:
"""
Extracts tool calls for Mistral models
doing tool calls of the following format:
`[TOOL_CALLS]add{"a": 3.5, "b": 4}`
"""
additional_content: str = ""
if self.streaming_state == StreamingState.WAITING_FOR_TOOL_START:
# this is the first tool call
if self.bot_token not in delta_text:
return DeltaMessage(content=delta_text)
if not delta_text.startswith(self.bot_token):
additional_content += delta_text.split(self.bot_token)[0]
delta_text = self.bot_token + "".join(
delta_text.split(self.bot_token)[1:]
)
delta_tool_calls = self._generate_delta_tool_call(delta_text)
if not additional_content and len(delta_tool_calls) == 0:
if self.streaming_state in [
StreamingState.PARSING_ARGUMENTS,
StreamingState.PARSING_ARGUMENTS_COMPLETED,
StreamingState.TOOL_COMPLETE,
StreamingState.ALL_TOOLS_COMPLETE,
]:
# Return an empty DeltaMessage once the tool calls are all done
# so that finish_reason gets set.
return DeltaMessage()
else:
# return None when the tool is not likely to be finished
# This can occur when the name is being parsed for example
# and we wait for the name to be complete
# before sending the function name
return None
delta = DeltaMessage()
if additional_content:
delta.content = additional_content
if len(delta_tool_calls) > 0:
delta.tool_calls = delta_tool_calls
return delta
def _generate_delta_tool_call(self, delta_text: str) -> list[DeltaToolCall]:
if delta_text == "" or delta_text is None:
return []
delta_function_name = None
tool_id = None
if self.streaming_state not in [
StreamingState.PARSING_NAME,
StreamingState.PARSING_ARGUMENTS,
] and delta_text.startswith(self.bot_token):
self.current_tool_id += 1
self.streamed_args_for_tool.append("")
self.prev_tool_call_arr.append({})
self.streaming_state = StreamingState.PARSING_NAME
delta_text = delta_text.replace(self.bot_token, "", 1)
if self.streaming_state == StreamingState.PARSING_NAME:
if self.current_tool_name is None:
self.current_tool_name = ""
# The name stops where the arguments start
# And the arguments start with the `{` char
if "{" in delta_text:
tool_id = MistralToolCall.generate_random_id()
delta_function_name = delta_text.split("{")[0]
self.current_tool_name += delta_function_name
# HF tokenizers may include [ARGS] in the text
self.current_tool_name = self.current_tool_name.replace("[ARGS]", "")
self.prev_tool_call_arr[self.current_tool_id]["name"] = (
self.current_tool_name
)
delta_text = delta_text[len(delta_function_name) :]
self.streaming_state = StreamingState.PARSING_ARGUMENTS
else:
# we want to send the tool name once it's complete
self.current_tool_name += delta_text
return []
if self.streaming_state == StreamingState.PARSING_ARGUMENTS:
next_function_text = None
if self.bot_token in delta_text:
# current tool call is over
delta_arguments = ""
delta_arguments += delta_text.split(self.bot_token)[0]
next_function_text = delta_text[len(delta_arguments) :]
self.streaming_state = StreamingState.TOOL_COMPLETE
else:
delta_arguments = delta_text
self.streamed_args_for_tool[self.current_tool_id] += delta_arguments
self.prev_tool_call_arr[self.current_tool_id]["arguments"] = (
self.streamed_args_for_tool[self.current_tool_id]
)
ret = []
if self.current_tool_name or delta_arguments:
ret += [
DeltaToolCall(
index=self.current_tool_id,
type="function",
id=tool_id,
function=DeltaFunctionCall(
name=self.current_tool_name, arguments=delta_arguments
).model_dump(exclude_none=True),
)
]
self.current_tool_name = None
if next_function_text:
ret += self._generate_delta_tool_call(next_function_text)
return ret
# Should not happen
return []
@ijson.coroutine
def update_stream_state_pre_v11_tokenizer(self):
while True:
(prefix, event, value) = yield
if prefix == "item" and event == "start_map":
self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
self.starting_new_tool = True
if prefix == "item" and event == "map_key" and value == "name":
self.streaming_state = StreamingState.PARSING_NAME
if prefix == "item.name" and event == "string":
self.current_tool_name = value
self.streaming_state = StreamingState.PARSING_NAME_COMPLETED
if prefix == "item" and event == "map_key" and value == "arguments":
self.streaming_state = StreamingState.WAITING_FOR_ARGUMENTS_START
if prefix == "item.arguments" and event == "start_map":
self.streaming_state = StreamingState.PARSING_ARGUMENTS
if prefix == "item.arguments" and event == "end_map":
self.streaming_state = StreamingState.PARSING_ARGUMENTS_COMPLETED
if prefix == "item" and event == "end_map":
self.streaming_state = StreamingState.TOOL_COMPLETE
if prefix == "" and event == "end_array":
self.streaming_state = StreamingState.ALL_TOOLS_COMPLETE
def _extract_tool_calls_streaming_pre_v11_tokenizer(
self,
delta_text: str,
delta_token_ids: Sequence[int],
) -> DeltaMessage | None:
"""
Extracts tool calls for Mistral models
doing tool calls of the following format:
`[TOOL_CALLS][{"name": "add", "arguments":{"a": 3.5, "b": 4}}`
"""
assert self.parse_coro is not None
content = None
delta_tool_calls: list[DeltaToolCall] = []
current_tool_call: DeltaToolCall = DeltaToolCall(
index=self.current_tool_id, type="function"
)
current_tool_call_modified = False
if self.bot_token_id in delta_token_ids or self.bot_token in delta_text:
# this is the first tool call
if not delta_text.startswith(self.bot_token):
content = delta_text.split(self.bot_token)[0]
delta_text = "".join(delta_text.split(self.bot_token)[1:])
# Cut smartly the delta text to catch the ijson events
# as ijson does not give us the index in the text at each event.
# We need to cut so that we know
# where in the text the events are emitted from.
while len(delta_text) > 0:
streaming_state_before_parse = self.streaming_state
if self.streaming_state == StreamingState.WAITING_FOR_TOOL_START:
delta_to_be_parsed, delta_text = self._split_delta(
delta_text=delta_text,
stop_after_opening_curly_braces=1,
)
elif self.streaming_state == StreamingState.WAITING_FOR_TOOL_KEY:
# Wait until another key is sent
# or the current tool is completed
delta_to_be_parsed, delta_text = self._split_delta(
delta_text=delta_text,
stop_after_colon=1,
stop_after_opening_curly_braces=1,
# if the tool ends, we want to separate
# at the start of the next tool
)
elif self.streaming_state == StreamingState.PARSING_NAME:
delta_to_be_parsed, delta_text = self._split_delta(
delta_text=delta_text,
stop_after_comma=1,
stop_after_closing_brackets=1,
)
elif self.streaming_state == StreamingState.WAITING_FOR_ARGUMENTS_START:
delta_to_be_parsed, delta_text = self._split_delta(
delta_text=delta_text,
stop_after_opening_curly_braces=1,
)
elif self.streaming_state == StreamingState.PARSING_ARGUMENTS:
delta_to_be_parsed, delta_text = self._split_delta(
delta_text=delta_text,
stop_after_closing_curly_braces=1,
# we could be more clever
# by listening to item.arguments.* start_map events
# and know how many curly braces we can allow
)
elif self.streaming_state in [
StreamingState.PARSING_ARGUMENTS_COMPLETED,
StreamingState.PARSING_NAME_COMPLETED,
]:
delta_to_be_parsed, delta_text = self._split_delta(
delta_text=delta_text,
stop_after_closing_curly_braces=1,
stop_after_closing_brackets=1,
)
elif self.streaming_state == StreamingState.TOOL_COMPLETE:
delta_to_be_parsed, delta_text = self._split_delta(
delta_text=delta_text,
stop_after_opening_curly_braces=1,
stop_after_closing_brackets=1,
)
elif self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE:
content = delta_text
delta_text = ""
else:
delta_to_be_parsed = delta_text
delta_text = ""
if self.streaming_state != StreamingState.ALL_TOOLS_COMPLETE:
self.parse_coro.send(delta_to_be_parsed.encode("utf-8"))
# Given the parsed text and the possible streaming state change,
# let's add to the tool delta
# start_map is the authoritative new-tool signal and survives
# batched deltas, unlike comparing pre/post streaming states
if self.starting_new_tool:
self.starting_new_tool = False
if current_tool_call_modified:
if self.current_tool_mistral_id is not None:
current_tool_call.id = self.current_tool_mistral_id
self.current_tool_mistral_id = None
self._track_streamed_args_pre_v11(current_tool_call)
delta_tool_calls.append(current_tool_call)
current_tool_call_modified = False
self.current_tool_id += 1
self.streamed_args_for_tool.append("")
self.prev_tool_call_arr.append({})
self.current_tool_mistral_id = MistralToolCall.generate_random_id()
current_tool_call = DeltaToolCall(
index=self.current_tool_id,
type="function",
)
if current_tool_call.function is None:
current_tool_call.function = DeltaFunctionCall()
if self.current_tool_name is not None:
# we have the complete tool name
current_tool_call_modified = True
current_tool_call.function.name = self.current_tool_name
self.prev_tool_call_arr[self.current_tool_id]["name"] = (
self.current_tool_name
)
self.current_tool_name = None
if self.streaming_state == StreamingState.PARSING_NAME_COMPLETED:
self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
if self.streaming_state in [
StreamingState.PARSING_ARGUMENTS,
StreamingState.PARSING_ARGUMENTS_COMPLETED,
]:
if self.streaming_state == StreamingState.PARSING_ARGUMENTS_COMPLETED:
self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
# the delta_to_be_parsed is part of arguments.
current_tool_call_modified = True
if current_tool_call.function.arguments is None:
current_tool_call.function.arguments = delta_to_be_parsed
else:
current_tool_call.function.arguments += delta_to_be_parsed
if streaming_state_before_parse != StreamingState.PARSING_ARGUMENTS:
# It's the first chunk of arg. let's lstrip it
current_tool_call.function.arguments = (
current_tool_call.function.arguments.lstrip()
)
if current_tool_call_modified:
if self.current_tool_mistral_id is not None:
current_tool_call.id = self.current_tool_mistral_id
self.current_tool_mistral_id = None
self._track_streamed_args_pre_v11(current_tool_call)
delta_tool_calls.append(current_tool_call)
if content or len(delta_tool_calls) > 0:
delta_message = DeltaMessage()
if content:
delta_message.content = content
if len(delta_tool_calls) > 0:
delta_message.tool_calls = delta_tool_calls
return delta_message
else:
if self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE:
return DeltaMessage()
else:
return None
def _track_streamed_args_pre_v11(self, tool_call: DeltaToolCall) -> None:
r"""Accumulate `tool_call` arguments into the streaming state."""
if tool_call.function is not None and tool_call.function.arguments is not None:
self.streamed_args_for_tool[self.current_tool_id] += (
tool_call.function.arguments
)
self.prev_tool_call_arr[self.current_tool_id]["arguments"] = (
self.streamed_args_for_tool[self.current_tool_id]
)
def _split_delta(
self,
delta_text: str,
stop_after_quotes: int = -1,
stop_after_opening_curly_braces: int = -1,
stop_after_closing_curly_braces: int = -1,
stop_after_closing_brackets: int = -1,
stop_after_colon: int = -1,
stop_after_comma=-1,
) -> tuple[str, str]:
delta_to_be_parsed = ""
for i, c in enumerate(delta_text):
if c in ['"', "'"]:
delta_to_be_parsed += c
stop_after_quotes -= 1
if stop_after_quotes == 0:
return (delta_to_be_parsed, delta_text[i + 1 :])
elif c == "{":
delta_to_be_parsed += c
stop_after_opening_curly_braces -= 1
if stop_after_opening_curly_braces == 0:
return (delta_to_be_parsed, delta_text[i + 1 :])
elif c == "}":
delta_to_be_parsed += c
stop_after_closing_curly_braces -= 1
if stop_after_closing_curly_braces == 0:
return (delta_to_be_parsed, delta_text[i + 1 :])
elif c == "]":
delta_to_be_parsed += c
stop_after_closing_brackets -= 1
if stop_after_closing_brackets == 0:
return (delta_to_be_parsed, delta_text[i + 1 :])
elif c == ":":
delta_to_be_parsed += c
stop_after_colon -= 1
if stop_after_colon == 0:
return (delta_to_be_parsed, delta_text[i + 1 :])
elif c == ",":
delta_to_be_parsed += c
stop_after_comma -= 1
if stop_after_comma == 0:
return (delta_to_be_parsed, delta_text[i + 1 :])
else:
delta_to_be_parsed += c
return (delta_to_be_parsed, "")
# Skip the base ToolParser tool_choice -> structured_outputs
# conversion. Mistral enforces tool_choice via the grammar ``mode``
# (auto/none/required/named); forwarding a tool-derived json_schema
# would be unioned into the grammar by mistral-common, allowing raw
# JSON in place of a real [TOOL_CALLS] call. Genuine user-provided
# structured output on the request is still honored by the engine's
# adjust_request.
return self._parser_engine.adjust_request(request)
+1 -1
View File
@@ -99,7 +99,7 @@ def extract_named_tool_call_streaming(
else:
if is_mistral_tokenizer(tokenizer):
# Import mistral_common only if we need it.
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
from vllm.parser.mistral import MistralToolCall
tool_call_id = MistralToolCall.generate_random_id()
else:
+5 -10
View File
@@ -12,10 +12,8 @@ from vllm.utils.import_utils import LazyLoader
if TYPE_CHECKING:
# if type checking, eagerly import the module
import vllm.tokenizers.mistral as mt
import vllm.tool_parsers.mistral_tool_parser as mtp
else:
mt = LazyLoader("mt", globals(), "vllm.tokenizers.mistral")
mtp = LazyLoader("mtp", globals(), "vllm.tool_parsers.mistral_tool_parser")
def is_mistral_tokenizer(obj: TokenizerLike | None) -> TypeGuard[mt.MistralTokenizer]:
@@ -31,13 +29,10 @@ def is_mistral_tokenizer(obj: TokenizerLike | None) -> TypeGuard[mt.MistralToken
def is_mistral_tool_parser(cls: type | None) -> bool:
"""Return true if *cls* is (a subclass of) MistralToolParser.
"""Return true if *cls* carries the ``IS_MISTRAL_TOOL_PARSER`` marker.
Uses a class attribute check so that importing
``vllm.tool_parsers.mistral_tool_parser`` — and transitively
``mistral_common`` — is not required.
The marker is set on the engine-adapter subclass registered under the
``"mistral"`` tool-parser key. No import of the parser module is
required.
"""
return bool(
getattr(cls, "IS_MISTRAL_TOOL_PARSER", False)
and issubclass(cls, mtp.MistralToolParser) # type: ignore[arg-type]
)
return bool(getattr(cls, "IS_MISTRAL_TOOL_PARSER", False))