[PARSER][Mistral] unified engine-based parser for reasoning and tool calls (#48947)
This commit is contained in:
@@ -31,7 +31,7 @@ partial-json-parser # used for parsing partial JSON outputs
|
||||
jsonschema >= 4.23.0 # required for MiniMax M3 tool schema validation
|
||||
pyzmq >= 25.0.0
|
||||
msgspec
|
||||
mistral_common[image] >= 1.11.5
|
||||
mistral_common[image] >= 1.11.6
|
||||
opencv-python-headless >= 4.13.0 # required for video IO
|
||||
pyyaml
|
||||
six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
|
||||
|
||||
@@ -476,7 +476,7 @@ mcp==1.28.1
|
||||
# via -r requirements/test/../common.txt
|
||||
mdurl==0.1.2
|
||||
# via markdown-it-py
|
||||
mistral-common==1.11.5
|
||||
mistral-common==1.11.6
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
# -r requirements/test/cuda.in
|
||||
|
||||
@@ -33,7 +33,7 @@ torchaudio==2.11.0
|
||||
torchvision==0.28.0
|
||||
transformers_stream_generator # required for qwen-vl test
|
||||
matplotlib # required for qwen-vl test
|
||||
mistral_common[image,audio] >= 1.11.5 # required for voxtral test
|
||||
mistral_common[image,audio] >= 1.11.6 # required for voxtral test
|
||||
num2words # required for smolvlm test
|
||||
open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
|
||||
datamodel_code_generator # required for minicpm3 test
|
||||
|
||||
@@ -502,7 +502,7 @@ mcp==1.28.1
|
||||
# via -r requirements/test/../common.txt
|
||||
mdurl==0.1.2
|
||||
# via markdown-it-py
|
||||
mistral-common==1.11.5
|
||||
mistral-common==1.11.6
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
|
||||
@@ -23,7 +23,7 @@ jiwer # required for audio tests
|
||||
timm # required for internvl test
|
||||
transformers_stream_generator # required for qwen-vl test
|
||||
matplotlib # required for qwen-vl test
|
||||
mistral_common[image,audio] >= 1.11.5 # required for voxtral test
|
||||
mistral_common[image,audio] >= 1.11.6 # required for voxtral test
|
||||
num2words # required for smolvlm test
|
||||
opencv-python-headless >= 4.13.0 # required for video test
|
||||
datamodel_code_generator # required for minicpm3 test
|
||||
|
||||
@@ -29,7 +29,7 @@ tblib # for pickling test exceptions
|
||||
timm>=1.0.17 # required for internvl and gemma3n-mm test
|
||||
transformers_stream_generator # required for qwen-vl test
|
||||
matplotlib # required for qwen-vl test
|
||||
mistral_common[image,audio]>=1.11.5 # required for voxtral test
|
||||
mistral_common[image,audio]>=1.11.6 # required for voxtral test
|
||||
num2words # required for smolvlm test
|
||||
open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
|
||||
datamodel_code_generator # required for minicpm3 test
|
||||
|
||||
@@ -497,7 +497,7 @@ mcp==1.27.0
|
||||
# via -r requirements/test/../common.txt
|
||||
mdurl==0.1.2
|
||||
# via markdown-it-py
|
||||
mistral-common==1.11.5
|
||||
mistral-common==1.11.6
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
|
||||
@@ -373,7 +373,7 @@ mcp==1.28.1
|
||||
# via -r requirements/test/../common.txt
|
||||
mdurl==0.1.2
|
||||
# via markdown-it-py
|
||||
mistral-common==1.11.5
|
||||
mistral-common==1.11.6
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
|
||||
@@ -5,12 +5,10 @@ import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.parser.mistral import MistralToolCall
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.tool_parsers.mistral_tool_parser import (
|
||||
MistralToolCall,
|
||||
MistralToolParser,
|
||||
)
|
||||
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
|
||||
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.parser.engine.adapters import (
|
||||
ParserEngineReasoningAdapter,
|
||||
ParserEngineToolAdapter,
|
||||
)
|
||||
from vllm.parser.mistral import MistralParser
|
||||
|
||||
_TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam])
|
||||
|
||||
@@ -84,6 +85,11 @@ def _discover_pairings() -> list[_PairingInfo]:
|
||||
for engine_cls, adapters in engines.items():
|
||||
if "tool" not in adapters or "reasoning" not in adapters:
|
||||
continue
|
||||
if engine_cls is MistralParser:
|
||||
# Mistral uses brace-balanced JSON tool args with no TOOL_END
|
||||
# token, so it does not fit this TOOL_END-based replay harness.
|
||||
# It is covered by tests/parser/mistral/ instead.
|
||||
continue
|
||||
cfg = engine_cls(bare_tok, None).parser_engine_config
|
||||
if cfg.name not in _BUILDERS:
|
||||
missing_builders.append(f"{engine_cls.__name__} (config.name={cfg.name!r})")
|
||||
|
||||
@@ -34,6 +34,7 @@ from tests.parser.engine.trace_builder import _BUILDERS, build_samples
|
||||
from vllm.parser.engine import registered_adapters as _adapters_mod
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import ParserState
|
||||
from vllm.parser.mistral import MistralParser
|
||||
|
||||
# ── Parser discovery ─────────────────────────────────────────────────
|
||||
|
||||
@@ -64,6 +65,11 @@ def _discover_parsers() -> list[_ParserInfo]:
|
||||
and obj is not ParserEngine
|
||||
):
|
||||
continue
|
||||
if obj is MistralParser:
|
||||
# Mistral uses brace-balanced JSON tool args with no TOOL_END
|
||||
# token, so it does not fit this TOOL_END-based replay harness.
|
||||
# It is covered by tests/parser/mistral/ instead.
|
||||
continue
|
||||
cfg = obj(bare_tok, None).parser_engine_config
|
||||
if cfg.name not in _BUILDERS:
|
||||
missing_builders.append(f"{obj.__name__} (config.name={cfg.name!r})")
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,599 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from mistral_common.protocol.instruct.messages import (
|
||||
AssistantMessage,
|
||||
ThinkChunk,
|
||||
ToolMessage,
|
||||
UserMessage,
|
||||
)
|
||||
from mistral_common.protocol.instruct.request import InstructRequest
|
||||
from mistral_common.protocol.instruct.tool_calls import FunctionCall, ToolCall
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction_mistral
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
_PARSER_NAME = "mistral"
|
||||
_MODEL_V13 = "mistralai/Magistral-Small-2509"
|
||||
_MODEL_V11 = "mistralai/Magistral-Small-2506"
|
||||
|
||||
_SAMPLE_TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather in a city.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def mistral_tokenizer() -> MistralTokenizer:
|
||||
"""v13 Magistral tokenizer with `[THINK]`/`[/THINK]` special tokens."""
|
||||
return MistralTokenizer.from_pretrained(_MODEL_V13)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def mistral_v11_tokenizer() -> MistralTokenizer:
|
||||
"""v11 Magistral tokenizer using plain-text `<think>`/`</think>` tags."""
|
||||
return MistralTokenizer.from_pretrained(_MODEL_V11)
|
||||
|
||||
|
||||
def _encode_v13(tokenizer: MistralTokenizer, output: str) -> list[int]:
|
||||
"""Encode output string with `[THINK]`/`[/THINK]` markers into token IDs.
|
||||
|
||||
`[THINK]` and `[/THINK]` placeholders in `output` are replaced by their
|
||||
special-token IDs; all surrounding text is encoded with the base tokenizer.
|
||||
Mirrors the encoding helper from the deleted
|
||||
`tests/reasoning/test_mistral_reasoning_parser.py`.
|
||||
"""
|
||||
think_start = "[THINK]"
|
||||
think_end = "[/THINK]"
|
||||
len_start = len(think_start)
|
||||
len_end = len(think_end)
|
||||
|
||||
index_start = output.find(think_start)
|
||||
index_end = output.find(think_end)
|
||||
|
||||
out: list[int] = []
|
||||
if index_start != -1:
|
||||
out += tokenizer.tokenizer.encode(output[:index_start], False, False)
|
||||
out += [tokenizer.instruct.BEGIN_THINK]
|
||||
|
||||
if index_end != -1:
|
||||
middle = output[index_start + len_start : index_end]
|
||||
suffix = output[index_end + len_end :]
|
||||
out += tokenizer.tokenizer.encode(middle, False, False)
|
||||
out += [tokenizer.instruct.END_THINK]
|
||||
out += tokenizer.tokenizer.encode(suffix, False, False)
|
||||
else:
|
||||
out += tokenizer.tokenizer.encode(
|
||||
output[index_start + len_start :], False, False
|
||||
)
|
||||
elif index_end != -1:
|
||||
out += tokenizer.tokenizer.encode(output[:index_end], False, False)
|
||||
out += [tokenizer.instruct.END_THINK]
|
||||
out += tokenizer.tokenizer.encode(output[index_end + len_end :], False, False)
|
||||
else:
|
||||
out += tokenizer.tokenizer.encode(output, False, False)
|
||||
return out
|
||||
|
||||
|
||||
_TEST_CASES = [
|
||||
# v13 special-token encoding: [THINK]…[/THINK] with trailing content.
|
||||
pytest.param(
|
||||
False,
|
||||
"[THINK]r[/THINK]c",
|
||||
"r",
|
||||
"c",
|
||||
id="v13_basic",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"[THINK]r[/THINK]c",
|
||||
"r",
|
||||
"c",
|
||||
id="v13_basic_streaming",
|
||||
),
|
||||
# No trailing content after reasoning ends.
|
||||
pytest.param(
|
||||
False,
|
||||
"[THINK]r[/THINK]",
|
||||
"r",
|
||||
None,
|
||||
id="v13_no_trailing_content",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"[THINK]r[/THINK]",
|
||||
"r",
|
||||
None,
|
||||
id="v13_no_trailing_content_streaming",
|
||||
),
|
||||
# Multi-line reasoning and multi-line content.
|
||||
pytest.param(
|
||||
False,
|
||||
"[THINK]a\nb[/THINK]c\nd",
|
||||
"a\nb",
|
||||
"c\nd",
|
||||
id="v13_multiline",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"[THINK]a\nb[/THINK]c\nd",
|
||||
"a\nb",
|
||||
"c\nd",
|
||||
id="v13_multiline_streaming",
|
||||
),
|
||||
# No reasoning at all: output is pure content.
|
||||
pytest.param(
|
||||
False,
|
||||
"hello world",
|
||||
None,
|
||||
"hello world",
|
||||
id="v13_no_reasoning",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"hello world",
|
||||
None,
|
||||
"hello world",
|
||||
id="v13_no_reasoning_streaming",
|
||||
),
|
||||
# [THINK] opened but never closed; reasoning accumulates, content is None.
|
||||
pytest.param(
|
||||
False,
|
||||
"[THINK]r",
|
||||
"r",
|
||||
None,
|
||||
id="v13_think_no_end",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"[THINK]r",
|
||||
"r",
|
||||
None,
|
||||
id="v13_think_no_end_streaming",
|
||||
),
|
||||
# Stray [/THINK] with no matching [THINK]: no reasoning, content follows.
|
||||
pytest.param(
|
||||
False,
|
||||
"[/THINK]This is the rest",
|
||||
None,
|
||||
"This is the rest",
|
||||
id="v13_stray_think_end",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"[/THINK]This is the rest",
|
||||
None,
|
||||
"This is the rest",
|
||||
id="v13_stray_think_end_streaming",
|
||||
),
|
||||
# Content before [THINK]: leading segment joins with trailing segment.
|
||||
pytest.param(
|
||||
False,
|
||||
"Before\n[THINK]r[/THINK]\nAfter",
|
||||
"r",
|
||||
"Before\n\nAfter",
|
||||
id="v13_leading_content",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"Before\n[THINK]r[/THINK]\nAfter",
|
||||
"r",
|
||||
"Before\n\nAfter",
|
||||
id="v13_leading_content_streaming",
|
||||
),
|
||||
# Empty output: no reasoning, no content.
|
||||
pytest.param(
|
||||
False,
|
||||
"",
|
||||
None,
|
||||
None,
|
||||
id="v13_empty",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"",
|
||||
None,
|
||||
None,
|
||||
id="v13_empty_streaming",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"streaming, output, expected_reasoning, expected_content",
|
||||
_TEST_CASES,
|
||||
)
|
||||
def test_mistral_reasoning_v13(
|
||||
streaming: bool,
|
||||
output: str,
|
||||
expected_reasoning: str | None,
|
||||
expected_content: str | None,
|
||||
mistral_tokenizer: MistralTokenizer,
|
||||
) -> None:
|
||||
output_tokens = _encode_v13(mistral_tokenizer, output)
|
||||
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)(
|
||||
mistral_tokenizer
|
||||
)
|
||||
reasoning, content = run_reasoning_extraction_mistral(
|
||||
parser, output_tokens, streaming=streaming
|
||||
)
|
||||
assert reasoning == expected_reasoning
|
||||
assert content == expected_content
|
||||
|
||||
|
||||
def test_system_prompt_think_ignored(
|
||||
mistral_tokenizer: MistralTokenizer,
|
||||
) -> None:
|
||||
"""Streaming content extraction must not
|
||||
include `[THINK]`/`[/THINK]` examples from the system prompt.
|
||||
"""
|
||||
output = "[THINK]real[/THINK]answer"
|
||||
output_tokens = _encode_v13(mistral_tokenizer, output)
|
||||
for streaming in (False, True):
|
||||
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(
|
||||
_PARSER_NAME
|
||||
)(mistral_tokenizer)
|
||||
reasoning, content = run_reasoning_extraction_mistral(
|
||||
parser, output_tokens, streaming=streaming
|
||||
)
|
||||
assert reasoning == "real"
|
||||
assert content == "answer"
|
||||
|
||||
|
||||
def test_reasoning_with_tool_calls(
|
||||
mistral_tokenizer: MistralTokenizer,
|
||||
) -> None:
|
||||
"""Reasoning extraction with a tool call in the generated output."""
|
||||
tool_calls_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]")
|
||||
if tool_calls_token_id is None:
|
||||
pytest.skip("Tokenizer does not have [TOOL_CALLS] in vocab")
|
||||
|
||||
# Token sequence representing: [THINK]r[/THINK][TOOL_CALLS]f{"a":1}
|
||||
output_tokens = (
|
||||
[mistral_tokenizer.instruct.BEGIN_THINK]
|
||||
+ mistral_tokenizer.tokenizer.encode("r", False, False)
|
||||
+ [mistral_tokenizer.instruct.END_THINK]
|
||||
+ [tool_calls_token_id]
|
||||
+ mistral_tokenizer.tokenizer.encode('f{"a":1}', False, False)
|
||||
)
|
||||
for streaming in (False, True):
|
||||
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(
|
||||
_PARSER_NAME
|
||||
)(mistral_tokenizer)
|
||||
reasoning, _content = run_reasoning_extraction_mistral(
|
||||
parser, output_tokens, streaming=streaming
|
||||
)
|
||||
assert reasoning == "r"
|
||||
assert "[TOOL_CALLS]" not in (reasoning or "")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [False, True], ids=["nonstream", "stream"])
|
||||
def test_reasoning_v11_plain_text_think(
|
||||
mistral_v11_tokenizer: MistralTokenizer, streaming: bool
|
||||
) -> None:
|
||||
"""v11 tokenizers use plain-text `<think>`/`</think>` (no special tokens)."""
|
||||
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)(
|
||||
mistral_v11_tokenizer
|
||||
)
|
||||
|
||||
tokens = mistral_v11_tokenizer.tokenizer.encode("<think>r</think>c", False, False)
|
||||
reasoning, content = run_reasoning_extraction_mistral(
|
||||
parser, tokens, streaming=streaming
|
||||
)
|
||||
assert reasoning == "r"
|
||||
assert content == "c"
|
||||
|
||||
|
||||
def test_is_reasoning_end(
|
||||
mistral_tokenizer: MistralTokenizer,
|
||||
) -> None:
|
||||
"""is_reasoning_end returns True iff reasoning has finished."""
|
||||
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)(
|
||||
mistral_tokenizer
|
||||
)
|
||||
|
||||
# Complete reasoning: END_THINK present → True.
|
||||
complete_ids = _encode_v13(mistral_tokenizer, "[THINK]r[/THINK]c")
|
||||
assert parser.is_reasoning_end(complete_ids)
|
||||
|
||||
# Open-only: no END_THINK, BEGIN_THINK present → False.
|
||||
open_ids = _encode_v13(mistral_tokenizer, "[THINK]r")
|
||||
assert not parser.is_reasoning_end(open_ids)
|
||||
|
||||
# No reasoning markers at all → False.
|
||||
no_reasoning_ids = _encode_v13(mistral_tokenizer, "hello")
|
||||
assert not parser.is_reasoning_end(no_reasoning_ids)
|
||||
|
||||
# Explicit END_THINK closes reasoning → True.
|
||||
explicit_end_ids = (
|
||||
[mistral_tokenizer.instruct.BEGIN_THINK]
|
||||
+ mistral_tokenizer.tokenizer.encode("r", False, False)
|
||||
+ [mistral_tokenizer.instruct.END_THINK]
|
||||
)
|
||||
assert parser.is_reasoning_end(explicit_end_ids)
|
||||
|
||||
# [TOOL_CALLS] acts as implicit reasoning-end marker.
|
||||
tool_calls_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]")
|
||||
if tool_calls_token_id is None:
|
||||
pytest.skip("Tokenizer does not have [TOOL_CALLS] in vocab")
|
||||
|
||||
implicit_end_ids = (
|
||||
[mistral_tokenizer.instruct.BEGIN_THINK]
|
||||
+ mistral_tokenizer.tokenizer.encode("r", False, False)
|
||||
+ [tool_calls_token_id]
|
||||
)
|
||||
assert parser.is_reasoning_end(implicit_end_ids)
|
||||
|
||||
|
||||
def _stream_parse_delta(parser, tokenizer, gen_ids, request, prompt_ids):
|
||||
"""Stream `gen_ids` through `parse_delta` and reconstruct reasoning/content."""
|
||||
reasoning = ""
|
||||
content = ""
|
||||
previous_tokens: list[str] | None = None
|
||||
prefix_offset = 0
|
||||
read_offset = 0
|
||||
for i, tok in enumerate(gen_ids):
|
||||
new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally(
|
||||
tokenizer=tokenizer,
|
||||
all_input_ids=gen_ids[: i + 1],
|
||||
prev_tokens=previous_tokens,
|
||||
prefix_offset=prefix_offset,
|
||||
read_offset=read_offset,
|
||||
skip_special_tokens=True,
|
||||
spaces_between_special_tokens=True,
|
||||
)
|
||||
previous_tokens = (
|
||||
previous_tokens + new_tokens if previous_tokens else new_tokens
|
||||
)
|
||||
delta_message = parser.parse_delta(
|
||||
delta_text=delta_text,
|
||||
delta_token_ids=[tok],
|
||||
request=request,
|
||||
prompt_token_ids=prompt_ids,
|
||||
finished=(i == len(gen_ids) - 1),
|
||||
)
|
||||
if delta_message is not None:
|
||||
reasoning += delta_message.reasoning or ""
|
||||
content += delta_message.content or ""
|
||||
return reasoning, content
|
||||
|
||||
|
||||
def test_parse_delta_keeps_reasoning_open_after_closed_think_prompt(
|
||||
mistral_tokenizer: MistralTokenizer,
|
||||
) -> None:
|
||||
"""A closed `[THINK]` block in the prompt must not finalize reasoning.
|
||||
|
||||
Regression: reasoning system prompts end the prompt with a closed
|
||||
`[THINK]...[/THINK]` block. With a parser-injected grammar the generation's
|
||||
reasoning must still be captured as ``reasoning_content`` rather than
|
||||
leaking into ``content`` (the prompt-based reasoning-end heuristic must be
|
||||
skipped). This exercises ``DelegatingParser.parse_delta``'s prompt check,
|
||||
which the isolated reasoning-parser tests do not cover.
|
||||
"""
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="mistral",
|
||||
reasoning_parser_name="mistral",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
parser = parser_cls(mistral_tokenizer, None)
|
||||
request = parser.adjust_request(
|
||||
ChatCompletionRequest(
|
||||
messages=[],
|
||||
model="test",
|
||||
tools=_SAMPLE_TOOLS,
|
||||
tool_choice="auto",
|
||||
)
|
||||
)
|
||||
assert getattr(request, "_grammar_from_parser", False)
|
||||
|
||||
prompt_ids = _encode_v13(mistral_tokenizer, "[THINK]system reasoning[/THINK]")
|
||||
gen_ids = _encode_v13(
|
||||
mistral_tokenizer,
|
||||
"[THINK]because two plus two is four[/THINK]The answer is 4.",
|
||||
)
|
||||
|
||||
reasoning, content = _stream_parse_delta(
|
||||
parser, mistral_tokenizer, gen_ids, request, prompt_ids
|
||||
)
|
||||
|
||||
assert "because two plus two is four" in reasoning
|
||||
assert "because two plus two is four" not in content
|
||||
assert "The answer is 4." in content
|
||||
|
||||
|
||||
def _encode_gen(tokenizer: MistralTokenizer, text: str) -> list[int]:
|
||||
"""Encode `text` mapping `[THINK]`, `[/THINK]`, `[TOOL_CALLS]`, `[ARGS]`
|
||||
to their special-token IDs; all other text uses the base tokenizer.
|
||||
"""
|
||||
vocab = tokenizer.get_vocab()
|
||||
markers: dict[str, int] = {
|
||||
"[THINK]": tokenizer.instruct.BEGIN_THINK,
|
||||
"[/THINK]": tokenizer.instruct.END_THINK,
|
||||
"[TOOL_CALLS]": vocab["[TOOL_CALLS]"],
|
||||
"[ARGS]": vocab["[ARGS]"],
|
||||
}
|
||||
out: list[int] = []
|
||||
remaining = text
|
||||
while remaining:
|
||||
earliest: int | None = None
|
||||
earliest_marker: str | None = None
|
||||
for marker in markers:
|
||||
idx = remaining.find(marker)
|
||||
if idx != -1 and (earliest is None or idx < earliest):
|
||||
earliest = idx
|
||||
earliest_marker = marker
|
||||
if earliest_marker is None:
|
||||
out += tokenizer.tokenizer.encode(remaining, False, False)
|
||||
break
|
||||
assert earliest is not None # set together with earliest_marker
|
||||
if earliest > 0:
|
||||
out += tokenizer.tokenizer.encode(remaining[:earliest], False, False)
|
||||
out.append(markers[earliest_marker])
|
||||
remaining = remaining[earliest + len(earliest_marker) :]
|
||||
return out
|
||||
|
||||
|
||||
def _stream_turn(
|
||||
parser,
|
||||
tokenizer: MistralTokenizer,
|
||||
gen_ids: list[int],
|
||||
request,
|
||||
prompt_ids: list[int],
|
||||
) -> tuple[str, str, list[tuple[str, str]]]:
|
||||
"""Stream `gen_ids` through `parse_delta`, returning `(reasoning, content,
|
||||
tool_calls)`.
|
||||
|
||||
`tool_calls` is a list of `(name, arguments)` tuples ordered by
|
||||
`DeltaToolCall.index`. Arguments are concatenated across deltas.
|
||||
"""
|
||||
reasoning = ""
|
||||
content = ""
|
||||
tool_call_acc: dict[int, dict[str, str]] = {}
|
||||
previous_tokens: list[str] | None = None
|
||||
prefix_offset = 0
|
||||
read_offset = 0
|
||||
for i, tok in enumerate(gen_ids):
|
||||
new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally(
|
||||
tokenizer=tokenizer,
|
||||
all_input_ids=gen_ids[: i + 1],
|
||||
prev_tokens=previous_tokens,
|
||||
prefix_offset=prefix_offset,
|
||||
read_offset=read_offset,
|
||||
skip_special_tokens=True,
|
||||
spaces_between_special_tokens=True,
|
||||
)
|
||||
previous_tokens = (
|
||||
previous_tokens + new_tokens if previous_tokens else new_tokens
|
||||
)
|
||||
delta_message = parser.parse_delta(
|
||||
delta_text=delta_text,
|
||||
delta_token_ids=[tok],
|
||||
request=request,
|
||||
prompt_token_ids=prompt_ids,
|
||||
finished=(i == len(gen_ids) - 1),
|
||||
)
|
||||
if delta_message is not None:
|
||||
reasoning += delta_message.reasoning or ""
|
||||
content += delta_message.content or ""
|
||||
for tc in delta_message.tool_calls:
|
||||
if tc.index not in tool_call_acc:
|
||||
tool_call_acc[tc.index] = {"name": "", "args": ""}
|
||||
if tc.function:
|
||||
if tc.function.name:
|
||||
tool_call_acc[tc.index]["name"] = tc.function.name
|
||||
if tc.function.arguments:
|
||||
tool_call_acc[tc.index]["args"] += tc.function.arguments
|
||||
tool_calls = [
|
||||
(tool_call_acc[k]["name"], tool_call_acc[k]["args"])
|
||||
for k in sorted(tool_call_acc)
|
||||
]
|
||||
return reasoning, content, tool_calls
|
||||
|
||||
|
||||
def test_parse_delta_multi_turn_reason_and_tool(
|
||||
mistral_tokenizer: MistralTokenizer,
|
||||
) -> None:
|
||||
"""Two-turn loop: each assistant turn carries reasoning + a tool call.
|
||||
|
||||
For each turn, the mistral-common conversation is encoded into `prompt_ids`
|
||||
before streaming the next generation. Turn 2's prompt therefore contains
|
||||
a `[THINK]`/`[/THINK]` block and `[TOOL_CALLS]` from turn 1 — the
|
||||
accumulated tool-call/result history that single-turn tests never exercise.
|
||||
|
||||
Asserts that across both turns the parser correctly classifies think tokens
|
||||
as `reasoning_content` (never leaking into `content`), and that the tool
|
||||
call name/arguments are reconstructed correctly from the streaming deltas.
|
||||
"""
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="mistral",
|
||||
reasoning_parser_name="mistral",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
request = parser_cls(mistral_tokenizer, None).adjust_request(
|
||||
ChatCompletionRequest(
|
||||
messages=[],
|
||||
model="test",
|
||||
tools=_SAMPLE_TOOLS,
|
||||
tool_choice="auto",
|
||||
)
|
||||
)
|
||||
assert getattr(request, "_grammar_from_parser", False)
|
||||
|
||||
cities = ["Dallas", "Paris"]
|
||||
conversation: list = [UserMessage(content="What is the weather in Dallas?")]
|
||||
|
||||
for i, city in enumerate(cities):
|
||||
# Fresh parser per turn: streaming state must not bleed across turns.
|
||||
parser = parser_cls(mistral_tokenizer, None)
|
||||
|
||||
prompt_ids = mistral_tokenizer.instruct.encode_instruct(
|
||||
InstructRequest(messages=conversation)
|
||||
).tokens
|
||||
|
||||
gen_str = (
|
||||
f"[THINK]I should check {city} weather.[/THINK]"
|
||||
f'[TOOL_CALLS]get_weather[ARGS]{{"city": "{city}"}}'
|
||||
)
|
||||
gen_ids = _encode_gen(mistral_tokenizer, gen_str)
|
||||
|
||||
reasoning, content, tool_calls = _stream_turn(
|
||||
parser, mistral_tokenizer, gen_ids, request, prompt_ids
|
||||
)
|
||||
|
||||
assert f"check {city} weather" in reasoning, f"turn {i}: reasoning not captured"
|
||||
assert f"check {city} weather" not in content, (
|
||||
f"turn {i}: reasoning leaked into content"
|
||||
)
|
||||
assert "[THINK]" not in content, f"turn {i}: [THINK] leaked"
|
||||
assert "[TOOL_CALLS]" not in content, f"turn {i}: [TOOL_CALLS] leaked"
|
||||
assert len(tool_calls) == 1, f"turn {i}: expected 1 tool call"
|
||||
name, args = tool_calls[0]
|
||||
assert name == "get_weather", f"turn {i}: wrong tool name {name!r}"
|
||||
assert city in args, f"turn {i}: city not in args {args!r}"
|
||||
|
||||
# Extend conversation with this turn's history for the next turn.
|
||||
tc_id = f"tc{i:07d}" # 9-char alphanumeric id: "tc0000000", "tc0000001"
|
||||
conversation.extend(
|
||||
[
|
||||
AssistantMessage(
|
||||
content=[
|
||||
ThinkChunk(
|
||||
thinking=f"I should check {city} weather.",
|
||||
closed=True,
|
||||
)
|
||||
],
|
||||
tool_calls=[
|
||||
ToolCall(
|
||||
id=tc_id,
|
||||
function=FunctionCall(
|
||||
name="get_weather",
|
||||
arguments=json.dumps({"city": city}),
|
||||
),
|
||||
)
|
||||
],
|
||||
),
|
||||
ToolMessage(
|
||||
content=f"{city}: 70F sunny",
|
||||
tool_call_id=tc_id,
|
||||
),
|
||||
]
|
||||
)
|
||||
if i < len(cities) - 1:
|
||||
conversation.append(UserMessage(content=f"And in {cities[i + 1]}?"))
|
||||
+1150
-116
File diff suppressed because it is too large
Load Diff
@@ -1,348 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction_mistral
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
parser_name = "mistral"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def mistral_tokenizer():
|
||||
mistral_tokenizer = MistralTokenizer.from_pretrained(
|
||||
"mistralai/Magistral-Small-2509"
|
||||
)
|
||||
return mistral_tokenizer
|
||||
|
||||
|
||||
INVALID_SIMPLE_REASONING = {
|
||||
"output": "This is a reasoning section[/THINK]This is the rest",
|
||||
"reasoning": None,
|
||||
"content": "This is a reasoning sectionThis is the rest",
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
INVALID_COMPLETE_REASONING = {
|
||||
"output": "This is a reasoning section[/THINK]",
|
||||
"reasoning": None,
|
||||
"content": "This is a reasoning section",
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
NO_CONTENT = {
|
||||
"output": "[THINK]This is reasoning",
|
||||
"reasoning": "This is reasoning",
|
||||
"content": None,
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
NO_REASONING = {
|
||||
"output": "This is content",
|
||||
"reasoning": None,
|
||||
"content": "This is content",
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
NO_REASONING_STREAMING = {
|
||||
"output": "This is a reasoning section",
|
||||
"reasoning": None,
|
||||
"content": "This is a reasoning section",
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
INVALID_MULTIPLE_LINES = {
|
||||
"output": "This\nThat[/THINK]This is the rest\nThat",
|
||||
"reasoning": None,
|
||||
"content": "This\nThatThis is the rest\nThat",
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
INVALID_SHORTEST_REASONING_NO_STREAMING = {
|
||||
"output": "[/THINK]This is the rest",
|
||||
"reasoning": None,
|
||||
"content": "This is the rest",
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
INVALID_SHORTEST_REASONING = {
|
||||
"output": "[/THINK]This is the rest",
|
||||
"reasoning": None,
|
||||
"content": "This is the rest",
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
REASONING_WITH_THINK = {
|
||||
"output": "[THINK]This is a reasoning section[/THINK]This is the rest",
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": "This is the rest",
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
COMPLETE_REASONING_WITH_THINK = {
|
||||
"output": "[THINK]This is a reasoning section[/THINK]",
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": None,
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
MULTIPLE_LINES_WITH_THINK = {
|
||||
"output": "[THINK]This\nThat[/THINK]This is the rest\nThat",
|
||||
"reasoning": "This\nThat",
|
||||
"content": "This is the rest\nThat",
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
INVALID_SHORTEST_REASONING_NO_STREAMING_WITH_THINK = {
|
||||
"output": "[/THINK]This is the rest",
|
||||
"reasoning": None,
|
||||
"content": "This is the rest",
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
INVALID_SHORTEST_REASONING_WITH_THINK = {
|
||||
"output": "[/THINK]This is the rest",
|
||||
"reasoning": None,
|
||||
"content": "This is the rest",
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
THINK_NO_END = {
|
||||
"output": "[THINK]This is a reasoning section",
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": None,
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
EMPTY = {
|
||||
"output": "",
|
||||
"reasoning": None,
|
||||
"content": "",
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
EMPTY_STREAMING = {
|
||||
"output": "",
|
||||
"reasoning": None,
|
||||
"content": None,
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
NEW_LINE = {
|
||||
"output": "Before\n[THINK]This is a reasoning section[/THINK]\nThis is the rest",
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": "Before\n\nThis is the rest",
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
NEW_LINE_STREAMING = {
|
||||
"output": "Before\n[THINK]This is a reasoning section[/THINK]\nThis is the rest",
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": "Before\n\nThis is the rest",
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
|
||||
TEST_CASES = [
|
||||
pytest.param(
|
||||
False,
|
||||
INVALID_SIMPLE_REASONING,
|
||||
id="invalid_simple_reasoning",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
INVALID_SIMPLE_REASONING,
|
||||
id="invalid_simple_reasoning_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
INVALID_COMPLETE_REASONING,
|
||||
id="invalid_complete_reasoning",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
INVALID_COMPLETE_REASONING,
|
||||
id="invalid_complete_reasoning_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
NO_CONTENT,
|
||||
id="no_content",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
NO_REASONING,
|
||||
id="no_reasoning",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
NO_REASONING_STREAMING,
|
||||
id="no_reasoning_token_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
INVALID_MULTIPLE_LINES,
|
||||
id="invalid_multiple_lines",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
INVALID_MULTIPLE_LINES,
|
||||
id="invalid_multiple_lines_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
INVALID_SHORTEST_REASONING,
|
||||
id="invalid_shortest",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
INVALID_SHORTEST_REASONING_NO_STREAMING,
|
||||
id="invalid_shortest_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
REASONING_WITH_THINK,
|
||||
id="reasoning_with_think",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
REASONING_WITH_THINK,
|
||||
id="reasoning_with_think_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
COMPLETE_REASONING_WITH_THINK,
|
||||
id="complete_reasoning_with_think",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
COMPLETE_REASONING_WITH_THINK,
|
||||
id="complete_reasoning_with_think_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
MULTIPLE_LINES_WITH_THINK,
|
||||
id="multiple_lines_with_think",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
MULTIPLE_LINES_WITH_THINK,
|
||||
id="multiple_lines_with_think_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
INVALID_SHORTEST_REASONING_NO_STREAMING_WITH_THINK,
|
||||
id="invalid_shortest_with_think",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
INVALID_SHORTEST_REASONING_WITH_THINK,
|
||||
id="invalid_shortest_with_think_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
THINK_NO_END,
|
||||
id="think_no_end",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
THINK_NO_END,
|
||||
id="think_no_end_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
EMPTY,
|
||||
id="empty",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
EMPTY_STREAMING,
|
||||
id="empty_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
NEW_LINE,
|
||||
id="new_line",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
NEW_LINE_STREAMING,
|
||||
id="new_line_streaming",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)
|
||||
def test_mistral_reasoning(
|
||||
streaming: bool,
|
||||
param_dict: dict,
|
||||
mistral_tokenizer: MistralTokenizer,
|
||||
):
|
||||
output = param_dict["output"]
|
||||
|
||||
index_think = output.find("[THINK]")
|
||||
len_think = len("[THINK]")
|
||||
index_end_think = output.find("[/THINK]")
|
||||
len_end_think = len("[/THINK]")
|
||||
|
||||
# encode everything to tokens ids
|
||||
output_tokens = []
|
||||
if index_think != -1:
|
||||
output_before_think = output[:index_think]
|
||||
output_tokens += mistral_tokenizer.tokenizer.encode(
|
||||
output_before_think, False, False
|
||||
)
|
||||
output_tokens += [mistral_tokenizer.instruct.BEGIN_THINK]
|
||||
|
||||
if index_end_think != -1:
|
||||
output_middle = output[index_think + len_think : index_end_think]
|
||||
output_after_think = output[index_end_think + len_end_think :]
|
||||
output_tokens += mistral_tokenizer.tokenizer.encode(
|
||||
output_middle, False, False
|
||||
)
|
||||
output_tokens += [mistral_tokenizer.instruct.END_THINK]
|
||||
output_tokens += mistral_tokenizer.tokenizer.encode(
|
||||
output_after_think, False, False
|
||||
)
|
||||
else:
|
||||
output_middle = output[index_think + len_think :]
|
||||
output_tokens += mistral_tokenizer.tokenizer.encode(
|
||||
output_middle, False, False
|
||||
)
|
||||
elif index_end_think != -1:
|
||||
output_before_think = output[:index_end_think]
|
||||
output_after_think = output[index_end_think + len_end_think :]
|
||||
output_tokens += mistral_tokenizer.tokenizer.encode(
|
||||
output_before_think, False, False
|
||||
)
|
||||
output_tokens += [mistral_tokenizer.instruct.END_THINK]
|
||||
output_tokens += mistral_tokenizer.tokenizer.encode(
|
||||
output_after_think, False, False
|
||||
)
|
||||
else:
|
||||
output_tokens += mistral_tokenizer.tokenizer.encode(output, False, False)
|
||||
|
||||
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)(
|
||||
mistral_tokenizer
|
||||
)
|
||||
|
||||
reasoning, content = run_reasoning_extraction_mistral(
|
||||
parser, output_tokens, streaming=streaming
|
||||
)
|
||||
|
||||
assert reasoning == param_dict["reasoning"]
|
||||
assert content == param_dict["content"]
|
||||
|
||||
# Test is_reasoning_end
|
||||
is_reasoning_end = parser.is_reasoning_end(output_tokens)
|
||||
assert is_reasoning_end == param_dict["is_reasoning_end"]
|
||||
|
||||
# Test extract_content
|
||||
if param_dict["content"] is not None:
|
||||
# Handle the case where there are tokens outputted before Thinking.
|
||||
# This should not occur if the model is well trained and prompted.
|
||||
if "[THINK]" in param_dict["output"] and not param_dict["output"].startswith(
|
||||
"[THINK]"
|
||||
):
|
||||
before_content = param_dict["output"].split("[THINK]")[0]
|
||||
before_token_ids = mistral_tokenizer.tokenizer.encode(
|
||||
before_content, bos=False, eos=False
|
||||
)
|
||||
left_to_encode = param_dict["content"][len(before_content) :]
|
||||
# Normal situation.
|
||||
else:
|
||||
before_token_ids = []
|
||||
left_to_encode = param_dict["content"]
|
||||
|
||||
content_tokens = parser.extract_content_ids(output_tokens)
|
||||
expected_token_ids = before_token_ids + mistral_tokenizer.tokenizer.encode(
|
||||
left_to_encode, bos=False, eos=False
|
||||
)
|
||||
assert content_tokens == expected_token_ids
|
||||
else:
|
||||
content = parser.extract_content_ids(output_tokens)
|
||||
assert content == []
|
||||
@@ -8,11 +8,13 @@ import llguidance
|
||||
import pytest
|
||||
from mistral_common.exceptions import InvalidMessageStructureException
|
||||
from mistral_common.guidance.grammar_factory import GrammarFactory
|
||||
from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
|
||||
from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy, SpecialTokens
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.tokenizers.mistral import (
|
||||
MistralTokenizer,
|
||||
_validate_apply_chat_template_args,
|
||||
validate_request_params,
|
||||
)
|
||||
|
||||
|
||||
@@ -1247,7 +1249,9 @@ class TestMistralTokenizer:
|
||||
|
||||
expected_strings = (
|
||||
'[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}] I am an AI\n\nHello world ![TOOL_CALLS][{"name": "get_weather", "arguments": {"city": "Paris"}, "id": "123456789"}] {"content": {"temperature": 20, "unit": "celsius"}, "call_id": "123456789"}', # noqa: E501
|
||||
'I am an AI[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}]Hello world ![TOOL_CALLS]get_weather{"city": "Paris"}{"temperature": 20, "unit": "celsius"}', # noqa: E501
|
||||
# v11+ tool-call decode emits the explicit [ARGS] separator between
|
||||
# the function name and its JSON arguments (get_weather[ARGS]{...}).
|
||||
'I am an AI[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}]Hello world ![TOOL_CALLS]get_weather[ARGS]{"city": "Paris"}{"temperature": 20, "unit": "celsius"}', # noqa: E501
|
||||
)
|
||||
|
||||
assert (
|
||||
@@ -1498,6 +1502,7 @@ class TestMistralTokenizer:
|
||||
"get",
|
||||
"_",
|
||||
"weather",
|
||||
"[ARGS]",
|
||||
'{"',
|
||||
"city",
|
||||
'":',
|
||||
@@ -2234,3 +2239,140 @@ class TestMistralTokenizer:
|
||||
decoded = mistral_tokenizer.tokenizer.decode(output, SpecialTokenPolicy.KEEP)
|
||||
|
||||
assert "[THINK]2+2 equals 4[/THINK]" in decoded
|
||||
|
||||
|
||||
def test_convert_ids_to_tokens_pre_args_tekken():
|
||||
"""convert_ids_to_tokens must not raise on pre-[ARGS] Tekken tokenizers.
|
||||
|
||||
Tekken tokenizers before v11 (e.g. Ministral-8B-Instruct-2410) have no
|
||||
[ARGS] special token. This guards the is_special([ARGS]) gate in
|
||||
MistralTokenizer.convert_ids_to_tokens, which must skip [ARGS] rather than
|
||||
call get_special_token(args) unconditionally (which raises on Tekken).
|
||||
"""
|
||||
tokenizer = MistralTokenizer.from_pretrained("mistralai/Ministral-8B-Instruct-2410")
|
||||
assert tokenizer.is_tekken
|
||||
assert not tokenizer.tokenizer.is_special(SpecialTokens.args)
|
||||
|
||||
ids = tokenizer.encode("Hello world !", add_special_tokens=False)
|
||||
tokens = tokenizer.convert_ids_to_tokens(ids, skip_special_tokens=True)
|
||||
assert len(tokens) > 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# validate_request_params – reasoning_effort validation (no tokenizer needed)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reasoning_effort", [None, "none", "high"])
|
||||
def test_validate_request_params_valid_effort(reasoning_effort: str | None) -> None:
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
reasoning_effort=reasoning_effort,
|
||||
)
|
||||
assert validate_request_params(request) is None
|
||||
|
||||
|
||||
def test_validate_request_params_rejects_unsupported_effort() -> None:
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
reasoning_effort="medium",
|
||||
)
|
||||
with pytest.raises(ValueError, match="reasoning_effort"):
|
||||
validate_request_params(request)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# apply_chat_template reasoning_effort passthrough (v15 tokenizer)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PASSTHROUGH_MESSAGES = [{"role": "user", "content": "Hello"}]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def v15_mistral_tokenizer() -> MistralTokenizer:
|
||||
"""Load the v15 Mistral tokenizer; skip if unavailable."""
|
||||
try:
|
||||
return MistralTokenizer.from_pretrained("mistralai/Mistral-Small-4-119B-2603")
|
||||
except Exception:
|
||||
pytest.skip("v15 tokenizer unavailable")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def v13_mistral_tokenizer() -> MistralTokenizer:
|
||||
"""Load the v13 Mistral tokenizer; skip if unavailable."""
|
||||
try:
|
||||
return MistralTokenizer.from_pretrained("mistralai/Magistral-Small-2509")
|
||||
except Exception:
|
||||
pytest.skip("v13 tokenizer unavailable")
|
||||
|
||||
|
||||
def test_v15_apply_chat_template_passes_reasoning_effort_high(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
v15_mistral_tokenizer: MistralTokenizer,
|
||||
) -> None:
|
||||
captured_kwargs: list[dict] = []
|
||||
|
||||
def fake_apply_chat_template(**kwargs):
|
||||
captured_kwargs.append(kwargs)
|
||||
return [0]
|
||||
|
||||
monkeypatch.setattr(
|
||||
v15_mistral_tokenizer.transformers_tokenizer,
|
||||
"apply_chat_template",
|
||||
fake_apply_chat_template,
|
||||
)
|
||||
|
||||
v15_mistral_tokenizer.apply_chat_template(
|
||||
messages=_PASSTHROUGH_MESSAGES,
|
||||
reasoning_effort="high",
|
||||
)
|
||||
|
||||
assert captured_kwargs[-1]["reasoning_effort"] == "high"
|
||||
|
||||
|
||||
def test_v15_apply_chat_template_passes_reasoning_effort_none_by_default(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
v15_mistral_tokenizer: MistralTokenizer,
|
||||
) -> None:
|
||||
captured_kwargs: list[dict] = []
|
||||
|
||||
def fake_apply_chat_template(**kwargs):
|
||||
captured_kwargs.append(kwargs)
|
||||
return [0]
|
||||
|
||||
monkeypatch.setattr(
|
||||
v15_mistral_tokenizer.transformers_tokenizer,
|
||||
"apply_chat_template",
|
||||
fake_apply_chat_template,
|
||||
)
|
||||
|
||||
v15_mistral_tokenizer.apply_chat_template(messages=_PASSTHROUGH_MESSAGES)
|
||||
|
||||
assert "reasoning_effort" in captured_kwargs[-1]
|
||||
assert captured_kwargs[-1]["reasoning_effort"] is None
|
||||
|
||||
|
||||
def test_pre_v15_apply_chat_template_omits_reasoning_effort(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
v13_mistral_tokenizer: MistralTokenizer,
|
||||
) -> None:
|
||||
captured_kwargs: list[dict] = []
|
||||
|
||||
def fake_apply_chat_template(**kwargs):
|
||||
captured_kwargs.append(kwargs)
|
||||
return [0]
|
||||
|
||||
monkeypatch.setattr(
|
||||
v13_mistral_tokenizer.transformers_tokenizer,
|
||||
"apply_chat_template",
|
||||
fake_apply_chat_template,
|
||||
)
|
||||
|
||||
v13_mistral_tokenizer.apply_chat_template(
|
||||
messages=_PASSTHROUGH_MESSAGES,
|
||||
reasoning_effort="high",
|
||||
)
|
||||
|
||||
assert "reasoning_effort" not in captured_kwargs[-1]
|
||||
|
||||
@@ -219,7 +219,7 @@ _NOT_SET = object()
|
||||
pytest.param(
|
||||
[WEATHER_TOOL],
|
||||
"required",
|
||||
30,
|
||||
9,
|
||||
id="required",
|
||||
),
|
||||
],
|
||||
|
||||
@@ -533,8 +533,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
msg["tool_calls"] = list(tool_calls)
|
||||
return self
|
||||
|
||||
_grammar_from_tool_parser: bool = PrivateAttr(default=False)
|
||||
"""CAUTION: Should only be set by ``ToolParser.adjust_request``."""
|
||||
_grammar_from_parser: bool = PrivateAttr(default=False)
|
||||
"""CAUTION: Should only be set by the parser-engine adapter's adjust_request."""
|
||||
|
||||
def build_chat_params(
|
||||
self,
|
||||
|
||||
@@ -65,7 +65,6 @@ from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.collection_utils import as_list
|
||||
from vllm.utils.mistral import is_mistral_tool_parser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -157,15 +156,6 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
model_name=self.model_config.model,
|
||||
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
|
||||
)
|
||||
if (
|
||||
self.parser_cls is not None
|
||||
and is_mistral_tool_parser(self.parser_cls.tool_parser_cls)
|
||||
and self.parser_cls.reasoning_parser_cls is not None
|
||||
):
|
||||
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
|
||||
|
||||
MistralToolParser.model_can_reason = True
|
||||
|
||||
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
|
||||
|
||||
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
||||
@@ -350,7 +340,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
else:
|
||||
if not request.include_reasoning:
|
||||
reasoning_ended = True
|
||||
elif request._grammar_from_tool_parser:
|
||||
elif request._grammar_from_parser:
|
||||
# The Mistral grammar already includes an optional
|
||||
# `think?` rule that handles both reasoning and
|
||||
# non-reasoning outputs.
|
||||
|
||||
@@ -284,7 +284,7 @@ class DerenderChatRequest(BaseModel):
|
||||
|
||||
Required by the parsing so that tool/reasoning parsers can receive the full
|
||||
request context they expect (request.tools, request.tool_choice,
|
||||
request._grammar_from_tool_parser, etc.).
|
||||
request._grammar_from_parser, etc.).
|
||||
"""
|
||||
# --8<-- [end:derender-chat-request]
|
||||
|
||||
|
||||
@@ -396,6 +396,9 @@ class ParserEngine(Parser):
|
||||
return True
|
||||
return find_tool_name(self._tools, name)
|
||||
|
||||
def _accept_tool_name(self, name: str) -> bool:
|
||||
return bool(name) and self._is_valid_tool_name(name)
|
||||
|
||||
# ── Private helpers ─────────────────────────────────────────────
|
||||
|
||||
def _check_skip_tool_parsing(
|
||||
@@ -807,7 +810,7 @@ class ParserEngine(Parser):
|
||||
deltas: list[DeltaToolCall],
|
||||
name: str | None,
|
||||
) -> None:
|
||||
if not name or not self._is_valid_tool_name(name):
|
||||
if name is None or not self._accept_tool_name(name):
|
||||
return
|
||||
slot = self._tool_slots[idx]
|
||||
slot.name = name
|
||||
@@ -866,8 +869,8 @@ class ParserEngine(Parser):
|
||||
slot = self._tool_slots[idx]
|
||||
|
||||
if not slot.name_sent:
|
||||
name = slot.name or self._try_extract_name(idx)
|
||||
if name and self._is_valid_tool_name(name):
|
||||
name = slot.name or self._try_extract_name(idx) or ""
|
||||
if self._accept_tool_name(name):
|
||||
slot.name = name
|
||||
slot.name_sent = True
|
||||
slot.string_keys = self._streamable_string_keys(
|
||||
@@ -1042,7 +1045,7 @@ class ParserEngine(Parser):
|
||||
else:
|
||||
args_json = "{}"
|
||||
|
||||
if name and self._is_valid_tool_name(name):
|
||||
if self._accept_tool_name(name):
|
||||
self._ensure_tool_id(slot, name)
|
||||
args_json = self._fix_arg_types(args_json, name)
|
||||
tool_calls.append(
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.parser.glm47_moe import Glm47MoeParser
|
||||
from vllm.parser.inkling import InklingParser
|
||||
from vllm.parser.kimi_k2 import KimiK2Parser
|
||||
from vllm.parser.minimax_m2 import MinimaxM2Parser
|
||||
from vllm.parser.mistral import MistralParser
|
||||
from vllm.parser.nemotron_v3 import NemotronV3Parser
|
||||
from vllm.parser.qwen3 import Qwen3Parser
|
||||
from vllm.parser.seed_oss import SeedOssParser
|
||||
@@ -68,3 +69,8 @@ from vllm.parser.seed_oss import SeedOssParser
|
||||
InklingParserReasoningAdapter,
|
||||
InklingParserToolAdapter,
|
||||
) = make_adapters(InklingParser)
|
||||
|
||||
(
|
||||
MistralParserReasoningAdapter,
|
||||
MistralParserToolAdapter,
|
||||
) = make_adapters(MistralParser)
|
||||
|
||||
+949
-57
File diff suppressed because it is too large
Load Diff
@@ -109,8 +109,6 @@ class ParserManager:
|
||||
if reasoning_parser_cls is None and tool_parser_cls is None:
|
||||
return None
|
||||
|
||||
from vllm.utils.mistral import is_mistral_tool_parser
|
||||
|
||||
if is_harmony:
|
||||
from vllm.parser.harmony import HarmonyParser
|
||||
|
||||
@@ -118,13 +116,6 @@ class ParserManager:
|
||||
HarmonyParser.tool_parser_cls = tool_parser_cls
|
||||
return HarmonyParser
|
||||
|
||||
if is_mistral_tool_parser(tool_parser_cls):
|
||||
from vllm.parser.mistral import MistralParser
|
||||
|
||||
MistralParser.reasoning_parser_cls = reasoning_parser_cls
|
||||
MistralParser.tool_parser_cls = tool_parser_cls
|
||||
return MistralParser
|
||||
|
||||
if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":
|
||||
from vllm.parser.kimi_k3 import KimiK3Parser
|
||||
|
||||
|
||||
@@ -106,7 +106,7 @@ _REASONING_PARSERS_TO_REGISTER = {
|
||||
),
|
||||
"mistral": (
|
||||
"mistral_reasoning_parser",
|
||||
"MistralReasoningParser",
|
||||
"MistralParserReasoningAdapter",
|
||||
),
|
||||
"nemotron_v3": (
|
||||
"nemotron_v3_engine_reasoning_parser",
|
||||
|
||||
@@ -1,162 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING
|
||||
from vllm.parser.engine.registered_adapters import MistralParserReasoningAdapter
|
||||
|
||||
from vllm.reasoning import ReasoningParser
|
||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
|
||||
class MistralReasoningParser(BaseThinkingReasoningParser):
|
||||
"""
|
||||
Reasoning parser for Mistral models.
|
||||
|
||||
The Mistral models uses `[THINK]`...`[/THINK]` tokens to denote reasoning
|
||||
text. This parser extracts the reasoning content from the model output.
|
||||
|
||||
A valid reasoning trace should always start with a `[THINK]` token and end with
|
||||
a `[/THINK]` token.
|
||||
|
||||
If `[THINK]` token is not generated, then this parser only returns content.
|
||||
"""
|
||||
|
||||
def __init__(self, tokenizer: MistralTokenizer, *args, **kwargs):
|
||||
if not isinstance(tokenizer, MistralTokenizer):
|
||||
raise ValueError("The tokenizer must be an instance of MistralTokenizer.")
|
||||
|
||||
ReasoningParser.__init__(self, tokenizer, *args, **kwargs)
|
||||
|
||||
if not self.model_tokenizer:
|
||||
raise ValueError(
|
||||
"The model tokenizer must be passed to the ReasoningParser "
|
||||
"constructor during construction."
|
||||
)
|
||||
|
||||
self.start_token_id = tokenizer.tokenizer.get_special_token(self.start_token)
|
||||
self.end_token_id = tokenizer.tokenizer.get_special_token(self.end_token)
|
||||
|
||||
if self.start_token_id is None or self.end_token_id is None:
|
||||
raise RuntimeError(
|
||||
"Mistral reasoning parser could not locate think start/end "
|
||||
"tokens in the tokenizer!"
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def start_token(self) -> str:
|
||||
"""The token that starts reasoning content."""
|
||||
from mistral_common.tokens.tokenizers.base import SpecialTokens
|
||||
|
||||
return SpecialTokens.begin_think
|
||||
|
||||
@cached_property
|
||||
def end_token(self) -> str:
|
||||
"""The token that ends reasoning content."""
|
||||
from mistral_common.tokens.tokenizers.base import SpecialTokens
|
||||
|
||||
return SpecialTokens.end_think
|
||||
|
||||
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
|
||||
has_eot_token = False
|
||||
|
||||
for id in reversed(input_ids):
|
||||
if id == self.start_token_id:
|
||||
# Reasoning ends only if a BOT token is found before a EOT token.
|
||||
return has_eot_token
|
||||
elif id == self.end_token_id:
|
||||
has_eot_token = True
|
||||
return False
|
||||
|
||||
def is_reasoning_end_streaming(
|
||||
self, input_ids: Sequence[int], delta_ids: Iterable[int]
|
||||
) -> bool:
|
||||
if self.end_token_id in delta_ids:
|
||||
return True
|
||||
# Grammar's think? is optional — if [THINK] was never generated,
|
||||
# reasoning was skipped entirely.
|
||||
return self.start_token_id not in input_ids
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
"""
|
||||
Extract the content
|
||||
"""
|
||||
has_bot_token = False
|
||||
has_eot_token = False
|
||||
bot_token_index = -1
|
||||
eot_token_index = -1
|
||||
# One for loop instead of multiple lookups
|
||||
for i, token_id in enumerate(input_ids):
|
||||
# We filter that we have multiple BOT tokens which should not
|
||||
# happen for a well prompted trained model
|
||||
if token_id == self.start_token_id and not has_bot_token:
|
||||
has_bot_token = True
|
||||
bot_token_index = i
|
||||
elif token_id == self.end_token_id:
|
||||
has_eot_token = True
|
||||
eot_token_index = i
|
||||
break
|
||||
|
||||
# 1. Only BOT has been outputted
|
||||
if has_bot_token and not has_eot_token:
|
||||
# Should be = [] if model is well prompted and trained.
|
||||
return input_ids[:bot_token_index]
|
||||
# 2. Neither BOT or EOT have been outputted
|
||||
elif not has_bot_token and not has_eot_token:
|
||||
return input_ids
|
||||
# 3. Both BOT and EOT have been outputted.
|
||||
elif has_bot_token and has_eot_token:
|
||||
return input_ids[:bot_token_index] + input_ids[eot_token_index + 1 :]
|
||||
# 4. Only EOT has been outputted => this should not have occurred for a model
|
||||
# well prompted and trained.
|
||||
else:
|
||||
return input_ids[:eot_token_index] + input_ids[eot_token_index + 1 :]
|
||||
|
||||
def extract_reasoning(
|
||||
self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest"
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""
|
||||
Extract reasoning content from the model output.
|
||||
"""
|
||||
if not model_output:
|
||||
return (None, "")
|
||||
|
||||
# Check if the start token is present in the model output, remove it
|
||||
# if it is present.
|
||||
prev_bot_token, bot_token, post_bot_token = model_output.partition(
|
||||
self.start_token
|
||||
)
|
||||
|
||||
has_bot_token = bool(bot_token)
|
||||
# Valid EOT tokens should follow BOT token
|
||||
has_valid_eot_token = has_bot_token and self.end_token in post_bot_token
|
||||
|
||||
# 1. If there is BOT token followed by EOT token
|
||||
if has_bot_token and has_valid_eot_token:
|
||||
prev_eot_token, _, post_eot_token = post_bot_token.partition(self.end_token)
|
||||
# If model is well prompted and trained prev_bot_token should be ""
|
||||
content = prev_bot_token + post_eot_token
|
||||
return prev_eot_token, content if content else None
|
||||
# 2. Only BOT token
|
||||
elif has_bot_token:
|
||||
# If model is well prompted and trained prev_bot_token should be ""
|
||||
return post_bot_token, prev_bot_token if prev_bot_token else None
|
||||
# 3. EOT token has been outputted without BOT or neither has been outputted
|
||||
else:
|
||||
has_non_valid_eot_token = self.end_token in prev_bot_token
|
||||
# 3.a EOT token has been outputted without BOT
|
||||
# If model is well prompted and trained `has_non_valid_eot_token` should
|
||||
# be `False` and the parser outputs all tokens as 'content'
|
||||
if has_non_valid_eot_token:
|
||||
prev_eot_token, _, post_eot_token = prev_bot_token.partition(
|
||||
self.end_token
|
||||
)
|
||||
return None, prev_eot_token + post_eot_token
|
||||
# 3.b neither BOT or EOT have been outputted
|
||||
else:
|
||||
return None, prev_bot_token
|
||||
__all__ = ["MistralParserReasoningAdapter"]
|
||||
|
||||
@@ -21,7 +21,6 @@ from mistral_common.tokens.tokenizers.base import (
|
||||
)
|
||||
from mistral_common.tokens.tokenizers.instruct import (
|
||||
InstructTokenizerBase,
|
||||
InstructTokenizerV13,
|
||||
)
|
||||
from mistral_common.tokens.tokenizers.mistral import (
|
||||
MistralTokenizer as MistralCommonTokenizer,
|
||||
@@ -461,6 +460,7 @@ class MistralTokenizer(TokenizerLike):
|
||||
def convert_tokens_to_string(self, tokens: list[str]) -> str:
|
||||
to_decode_special_tokens = {
|
||||
SpecialTokens.tool_calls,
|
||||
SpecialTokens.args,
|
||||
SpecialTokens.begin_think,
|
||||
SpecialTokens.end_think,
|
||||
}
|
||||
@@ -517,11 +517,20 @@ class MistralTokenizer(TokenizerLike):
|
||||
non_skip_special_tokens_ids = {
|
||||
self.tokenizer.get_special_token(SpecialTokens.tool_calls),
|
||||
}
|
||||
if isinstance(self.instruct, InstructTokenizerV13):
|
||||
if self.instruct.BEGIN_THINK:
|
||||
non_skip_special_tokens_ids.add(self.instruct.BEGIN_THINK)
|
||||
if self.instruct.END_THINK:
|
||||
non_skip_special_tokens_ids.add(self.instruct.END_THINK)
|
||||
# [ARGS] only exists in v11+ tool-call tokenizers; older tokenizers
|
||||
# raise (Tekken) or return unk (SPM) for it.
|
||||
if self.tokenizer.is_special(SpecialTokens.args):
|
||||
non_skip_special_tokens_ids.add(
|
||||
self.tokenizer.get_special_token(SpecialTokens.args)
|
||||
)
|
||||
# [THINK]/[/THINK] only exist in v13+ reasoning tokenizers; use the
|
||||
# same is_special gate as [ARGS] above so newer versions are covered
|
||||
# without an isinstance check on the instruct tokenizer.
|
||||
for think_token in (SpecialTokens.begin_think, SpecialTokens.end_think):
|
||||
if self.tokenizer.is_special(think_token):
|
||||
non_skip_special_tokens_ids.add(
|
||||
self.tokenizer.get_special_token(think_token)
|
||||
)
|
||||
|
||||
ids_kept = [
|
||||
i
|
||||
|
||||
@@ -3,767 +3,37 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from enum import Enum, auto
|
||||
from random import choices
|
||||
from string import ascii_letters, digits
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import ijson
|
||||
import regex as re
|
||||
from mistral_common.protocol.instruct.tool_calls import (
|
||||
NamedToolChoice as MistralNamedToolChoice,
|
||||
)
|
||||
from mistral_common.protocol.instruct.tool_calls import (
|
||||
Tool as MistralTool,
|
||||
)
|
||||
from mistral_common.protocol.instruct.tool_calls import (
|
||||
ToolChoice as MistralToolChoice,
|
||||
)
|
||||
from mistral_common.protocol.instruct.tool_calls import (
|
||||
ToolChoiceEnum as MistralToolChoiceEnum,
|
||||
)
|
||||
from pydantic import Field
|
||||
from vllm.parser.engine.registered_adapters import MistralParserToolAdapter
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
ExtractedToolCallInformation,
|
||||
FunctionCall,
|
||||
ToolCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import (
|
||||
Tool,
|
||||
ToolParser,
|
||||
)
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
ALPHANUMERIC = ascii_letters + digits
|
||||
|
||||
_DEFAULT_JSON_SCHEMA = {"anyOf": [{"type": "object"}, {"type": "array"}]}
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
|
||||
class StreamingState(Enum):
|
||||
"""Enum for tracking the current streaming parsing state."""
|
||||
class MistralToolParser(MistralParserToolAdapter): # type: ignore[valid-type, misc]
|
||||
# Marked so is_mistral_tool_parser() in vllm.utils.mistral recognises
|
||||
# this adapter and lets adjust_request() fire for tool_choice="none"
|
||||
# on grammar-capable Mistral tokenizers.
|
||||
IS_MISTRAL_TOOL_PARSER = True
|
||||
|
||||
WAITING_FOR_TOOL_START = auto()
|
||||
WAITING_FOR_TOOL_KEY = (
|
||||
auto()
|
||||
) # waiting for the "name" or "arguments" key to be complete
|
||||
PARSING_NAME = auto()
|
||||
PARSING_NAME_COMPLETED = auto()
|
||||
WAITING_FOR_ARGUMENTS_START = auto()
|
||||
PARSING_ARGUMENTS = auto()
|
||||
PARSING_ARGUMENTS_COMPLETED = auto()
|
||||
TOOL_COMPLETE = auto()
|
||||
ALL_TOOLS_COMPLETE = auto()
|
||||
|
||||
|
||||
class MistralToolCall(ToolCall):
|
||||
id: str = Field(default_factory=lambda: MistralToolCall.generate_random_id())
|
||||
|
||||
@staticmethod
|
||||
def generate_random_id():
|
||||
# Mistral Tool Call Ids must be alphanumeric with a length of 9.
|
||||
# https://github.com/mistralai/mistral-common/blob/21ee9f6cee3441e9bb1e6ed2d10173f90bd9b94b/src/mistral_common/protocol/instruct/validator.py#L299
|
||||
return "".join(choices(ALPHANUMERIC, k=9))
|
||||
|
||||
@staticmethod
|
||||
def is_valid_id(id: str) -> bool:
|
||||
return id.isalnum() and len(id) == 9
|
||||
|
||||
|
||||
def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool:
|
||||
if is_mistral_tokenizer(model_tokenizer):
|
||||
return model_tokenizer.version < 11
|
||||
# For HF tokenizers, check if [ARGS] token exists in vocab
|
||||
# which indicates a v11+ equivalent tokenizer
|
||||
vocab: dict[str, int] = getattr(model_tokenizer, "get_vocab", lambda: {})()
|
||||
return "[ARGS]" not in vocab
|
||||
|
||||
|
||||
class MistralToolParser(ToolParser):
|
||||
r"""Tool call parser for Mistral models, intended for use with either:
|
||||
|
||||
- `mistral_common <https://github.com/mistralai/mistral-common/>`_
|
||||
(recommended)
|
||||
- the `examples/tool_chat_template_mistral.jinja` template.
|
||||
|
||||
Used when `--enable-auto-tool-choice --tool-call-parser mistral` are all
|
||||
set.
|
||||
"""
|
||||
|
||||
IS_MISTRAL_TOOL_PARSER = True # used by vllm.utils.mistral
|
||||
|
||||
# Used to generate correct grammar in `adjust_request`
|
||||
model_can_reason: bool = False
|
||||
|
||||
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
|
||||
super().__init__(tokenizer, tools)
|
||||
|
||||
if not is_mistral_tokenizer(self.model_tokenizer):
|
||||
logger.info("Non-Mistral tokenizer detected when using a Mistral model...")
|
||||
|
||||
# initialize properties used for state when parsing tool calls in
|
||||
# streaming mode
|
||||
self.prev_tool_call_arr: list[dict[str, Any]] = []
|
||||
self.current_tool_id: int = -1
|
||||
self.streaming_state: StreamingState = StreamingState.WAITING_FOR_TOOL_START
|
||||
|
||||
# For streaming pre v11 tokenizer tool calls
|
||||
self.current_tool_name: str | None = None
|
||||
self.current_tool_mistral_id: str | None = None
|
||||
self.starting_new_tool = False
|
||||
self._is_pre_v11 = _is_pre_v11_tokeniser(self.model_tokenizer)
|
||||
if self._is_pre_v11:
|
||||
self.parse_coro = ijson.parse_coro(
|
||||
self.update_stream_state_pre_v11_tokenizer()
|
||||
)
|
||||
|
||||
self.bot_token = "[TOOL_CALLS]"
|
||||
self.bot_token_id = self.vocab.get(self.bot_token)
|
||||
self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL)
|
||||
|
||||
if self.bot_token_id is None:
|
||||
raise RuntimeError(
|
||||
"Mistral Tool Parser could not locate the tool call token in "
|
||||
"the tokenizer!"
|
||||
)
|
||||
# Mistral emits tool calls in the [TOOL_CALLS]name[ARGS]{...} format, not the
|
||||
# plain JSON array / bare-args that the serving layer's generic required and
|
||||
# named handlers expect. Route required/named through this parser's own
|
||||
# extraction (treated like "auto") so the [TOOL_CALLS] format is parsed
|
||||
# correctly instead of crashing or dumping the raw envelope into arguments.
|
||||
supports_required_and_named = False
|
||||
|
||||
def adjust_request(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
so_non_supported_attributes = [
|
||||
"regex",
|
||||
"choice",
|
||||
"grammar",
|
||||
# whitespace_pattern is not a constraint type but an option;
|
||||
# Mistral grammar factory does not support it.
|
||||
"whitespace_pattern",
|
||||
"structural_tag",
|
||||
]
|
||||
any_so_non_supported_active = request.structured_outputs is not None and any(
|
||||
getattr(request.structured_outputs, attribute) is not None
|
||||
for attribute in so_non_supported_attributes
|
||||
)
|
||||
response_format_non_supported_active = (
|
||||
isinstance(request, ResponsesRequest)
|
||||
or request.response_format is not None
|
||||
and request.response_format.type == "structural_tag"
|
||||
)
|
||||
|
||||
if (
|
||||
not is_mistral_tokenizer(self.model_tokenizer)
|
||||
or isinstance(request, ResponsesRequest)
|
||||
or not self.model_tokenizer.supports_grammar
|
||||
or any_so_non_supported_active
|
||||
or response_format_non_supported_active
|
||||
):
|
||||
request = super().adjust_request(request)
|
||||
if request.tools and request.tool_choice != "none":
|
||||
# Do not skip special tokens when using chat template
|
||||
# with Mistral parser as TOOL_CALL token is needed
|
||||
# for tool detection.
|
||||
# Note: we don't want skip_special_tokens=False
|
||||
# with MistralTokenizer as it is incompatible
|
||||
request.skip_special_tokens = False
|
||||
return request
|
||||
|
||||
json_schema: dict[str, Any] | None = None
|
||||
if request.structured_outputs is not None:
|
||||
if request.structured_outputs.json_object is not None:
|
||||
json_schema = _DEFAULT_JSON_SCHEMA
|
||||
elif request.structured_outputs.json is not None:
|
||||
if isinstance(request.structured_outputs.json, str):
|
||||
json_schema = json.loads(request.structured_outputs.json)
|
||||
else:
|
||||
json_schema = request.structured_outputs.json
|
||||
else:
|
||||
raise ValueError(
|
||||
"Unsupported request.structured_outputs for MistralToolParser. "
|
||||
"Only `json` and `json_object` are supported."
|
||||
)
|
||||
elif (
|
||||
request.response_format is not None
|
||||
and request.response_format.type != "text"
|
||||
):
|
||||
if request.response_format.type == "json_object":
|
||||
json_schema = _DEFAULT_JSON_SCHEMA
|
||||
elif request.response_format.type == "json_schema":
|
||||
if request.response_format.json_schema is not None:
|
||||
json_schema = request.response_format.json_schema.json_schema
|
||||
else:
|
||||
json_schema = _DEFAULT_JSON_SCHEMA
|
||||
else:
|
||||
raise ValueError(
|
||||
"MistralToolParser only accepts `text`, `json_object` or "
|
||||
f"`json_schema`, got {request.response_format=}"
|
||||
)
|
||||
# Structured Outputs will be defined.
|
||||
request.response_format = None
|
||||
|
||||
grammar_factory = self.model_tokenizer.grammar_factory
|
||||
|
||||
# TODO: Once unified parser, improve this.
|
||||
# The issue is figuring out when a model is a reasoning one or not.
|
||||
template = grammar_factory.select_jinja_template(
|
||||
reasoning=self.model_can_reason
|
||||
)
|
||||
|
||||
mistral_tools = (
|
||||
[MistralTool.from_openai(tool.model_dump()) for tool in request.tools]
|
||||
if request.tools is not None
|
||||
else None
|
||||
)
|
||||
|
||||
tool_choice: MistralToolChoice
|
||||
match request.tool_choice:
|
||||
case "none" | "auto" | "required":
|
||||
tool_choice = MistralToolChoiceEnum(request.tool_choice)
|
||||
case None:
|
||||
tool_choice = MistralToolChoiceEnum.auto
|
||||
# _ == Named tool choice
|
||||
case _:
|
||||
tool_choice = MistralNamedToolChoice.model_validate(
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": request.tool_choice.function.name},
|
||||
}
|
||||
)
|
||||
|
||||
# Rendering grammar is cached in mistral-common given tools, template and mode.
|
||||
match tool_choice, json_schema is not None:
|
||||
case MistralToolChoiceEnum.none, True:
|
||||
lark_grammar = grammar_factory.get_lark_for_json_schema(
|
||||
template=template, json_schema=json_schema
|
||||
)
|
||||
case _, _:
|
||||
lark_grammar = grammar_factory.get_lark_from_jinja(
|
||||
template=template,
|
||||
mode=tool_choice,
|
||||
tools=mistral_tools,
|
||||
json_schema=json_schema,
|
||||
parallel_tool_calls=request.parallel_tool_calls,
|
||||
json_only=False,
|
||||
)
|
||||
|
||||
request.structured_outputs = StructuredOutputsParams(grammar=lark_grammar)
|
||||
request._grammar_from_tool_parser = True
|
||||
return request
|
||||
|
||||
def extract_tool_calls(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest,
|
||||
) -> ExtractedToolCallInformation:
|
||||
"""
|
||||
Extract the tool calls from a complete model response.
|
||||
|
||||
Content and tool calls formatting depends on the Mistral's tokenizer version
|
||||
used to train the model:
|
||||
|
||||
- < v11: `content[BOT] [{tool_call1},{tool_call2}]`
|
||||
- >= v11: `content[BOT]tool_name1{args_call1}[BOT]tool_name2{args_call2}`
|
||||
|
||||
with [BOT] the tool call token.
|
||||
|
||||
Note:
|
||||
For tokenizer versions >= v11, tool calls with arguments wrongly formatted
|
||||
are still returned as tool calls. This is to allow the model to know it
|
||||
tried to make a tool call. It reduces chance of another failure and
|
||||
prevents that the context is filled with tool calls wrongly placed in
|
||||
assistant message contents.
|
||||
"""
|
||||
|
||||
# If the tool call token is not present, return a text response
|
||||
if self.bot_token not in model_output:
|
||||
return ExtractedToolCallInformation(
|
||||
tools_called=False, tool_calls=[], content=model_output
|
||||
)
|
||||
|
||||
content_and_raw_tool_calls = model_output.split(self.bot_token)
|
||||
content = content_and_raw_tool_calls[0]
|
||||
raw_tool_calls = content_and_raw_tool_calls[1:]
|
||||
|
||||
# >= v11: content[BOT]tool_name1{args_call1}[BOT]tool_name2{args_call2}
|
||||
if not self._is_pre_v11:
|
||||
tool_calls = []
|
||||
for raw_tool_call in raw_tool_calls:
|
||||
if "{" not in raw_tool_call:
|
||||
continue
|
||||
|
||||
end_name = raw_tool_call.find("{")
|
||||
tool_name, args = (
|
||||
raw_tool_call[:end_name],
|
||||
raw_tool_call[end_name:],
|
||||
)
|
||||
|
||||
# HF tokenizers may include [ARGS] in the text
|
||||
tool_name = tool_name.replace("[ARGS]", "")
|
||||
tool_calls.append({"name": tool_name, "arguments": args})
|
||||
|
||||
# < v11: content[BOT] [{tool_call1},{tool_call2}]
|
||||
else:
|
||||
if len(raw_tool_calls) != 1:
|
||||
raise ValueError(
|
||||
"Only one BOT token should have been outputted, "
|
||||
f"but got {model_output}."
|
||||
)
|
||||
stringified_tool_calls = raw_tool_calls[0].strip()
|
||||
try:
|
||||
# Use raw_decode to parse the first valid JSON value,
|
||||
# ignoring trailing tokens the model may emit after
|
||||
# the tool call array.
|
||||
tool_calls, _ = json.JSONDecoder().raw_decode(stringified_tool_calls)
|
||||
except json.JSONDecodeError:
|
||||
try:
|
||||
raw_tool_call = self.tool_call_regex.findall(
|
||||
stringified_tool_calls
|
||||
)[0]
|
||||
tool_calls = json.loads(raw_tool_call)
|
||||
tool_calls = [
|
||||
{
|
||||
"name": tool_call["name"],
|
||||
"arguments": json.dumps(
|
||||
tool_call.get("arguments", {}),
|
||||
ensure_ascii=False,
|
||||
),
|
||||
}
|
||||
for tool_call in tool_calls
|
||||
]
|
||||
except (IndexError, json.JSONDecodeError):
|
||||
logger.exception("Error in extracting tool call from response.")
|
||||
return ExtractedToolCallInformation(
|
||||
tools_called=False,
|
||||
tool_calls=[],
|
||||
content=stringified_tool_calls,
|
||||
)
|
||||
else:
|
||||
tool_calls = [
|
||||
{
|
||||
"name": tool_call["name"],
|
||||
"arguments": json.dumps(
|
||||
tool_call.get("arguments", {}),
|
||||
ensure_ascii=False,
|
||||
),
|
||||
}
|
||||
for tool_call in tool_calls
|
||||
]
|
||||
|
||||
mistral_tool_calls: list[MistralToolCall] = [
|
||||
MistralToolCall(
|
||||
type="function",
|
||||
function=FunctionCall(
|
||||
name=tool_call["name"],
|
||||
arguments=tool_call.get("arguments", "{}"),
|
||||
),
|
||||
)
|
||||
for tool_call in tool_calls
|
||||
]
|
||||
|
||||
return ExtractedToolCallInformation(
|
||||
tools_called=True,
|
||||
tool_calls=mistral_tool_calls,
|
||||
content=content if content.strip() else None,
|
||||
)
|
||||
|
||||
def extract_tool_calls_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
current_text: str,
|
||||
delta_text: str,
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> DeltaMessage | None:
|
||||
has_bot_token = (
|
||||
self.bot_token_id in current_token_ids or self.bot_token in current_text
|
||||
)
|
||||
if not has_bot_token:
|
||||
# if the tool call token is not in the tokens generated so far,
|
||||
# append output to contents since it's not a tool
|
||||
return DeltaMessage(content=delta_text)
|
||||
|
||||
# if the tool call token IS in the tokens generated so far, that
|
||||
# means we're parsing as tool calls now
|
||||
try:
|
||||
if self._is_pre_v11:
|
||||
return self._extract_tool_calls_streaming_pre_v11_tokenizer(
|
||||
delta_text=delta_text,
|
||||
delta_token_ids=delta_token_ids,
|
||||
)
|
||||
else:
|
||||
return self._extract_tool_calls_streaming(
|
||||
delta_text=delta_text, delta_token_ids=delta_token_ids
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Error trying to handle streaming tool call.")
|
||||
return None
|
||||
|
||||
def _extract_tool_calls_streaming(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: Sequence[int],
|
||||
) -> DeltaMessage | None:
|
||||
"""
|
||||
Extracts tool calls for Mistral models
|
||||
doing tool calls of the following format:
|
||||
`[TOOL_CALLS]add{"a": 3.5, "b": 4}`
|
||||
"""
|
||||
additional_content: str = ""
|
||||
if self.streaming_state == StreamingState.WAITING_FOR_TOOL_START:
|
||||
# this is the first tool call
|
||||
if self.bot_token not in delta_text:
|
||||
return DeltaMessage(content=delta_text)
|
||||
if not delta_text.startswith(self.bot_token):
|
||||
additional_content += delta_text.split(self.bot_token)[0]
|
||||
delta_text = self.bot_token + "".join(
|
||||
delta_text.split(self.bot_token)[1:]
|
||||
)
|
||||
|
||||
delta_tool_calls = self._generate_delta_tool_call(delta_text)
|
||||
if not additional_content and len(delta_tool_calls) == 0:
|
||||
if self.streaming_state in [
|
||||
StreamingState.PARSING_ARGUMENTS,
|
||||
StreamingState.PARSING_ARGUMENTS_COMPLETED,
|
||||
StreamingState.TOOL_COMPLETE,
|
||||
StreamingState.ALL_TOOLS_COMPLETE,
|
||||
]:
|
||||
# Return an empty DeltaMessage once the tool calls are all done
|
||||
# so that finish_reason gets set.
|
||||
return DeltaMessage()
|
||||
else:
|
||||
# return None when the tool is not likely to be finished
|
||||
# This can occur when the name is being parsed for example
|
||||
# and we wait for the name to be complete
|
||||
# before sending the function name
|
||||
return None
|
||||
|
||||
delta = DeltaMessage()
|
||||
if additional_content:
|
||||
delta.content = additional_content
|
||||
if len(delta_tool_calls) > 0:
|
||||
delta.tool_calls = delta_tool_calls
|
||||
|
||||
return delta
|
||||
|
||||
def _generate_delta_tool_call(self, delta_text: str) -> list[DeltaToolCall]:
|
||||
if delta_text == "" or delta_text is None:
|
||||
return []
|
||||
delta_function_name = None
|
||||
tool_id = None
|
||||
if self.streaming_state not in [
|
||||
StreamingState.PARSING_NAME,
|
||||
StreamingState.PARSING_ARGUMENTS,
|
||||
] and delta_text.startswith(self.bot_token):
|
||||
self.current_tool_id += 1
|
||||
self.streamed_args_for_tool.append("")
|
||||
self.prev_tool_call_arr.append({})
|
||||
self.streaming_state = StreamingState.PARSING_NAME
|
||||
delta_text = delta_text.replace(self.bot_token, "", 1)
|
||||
if self.streaming_state == StreamingState.PARSING_NAME:
|
||||
if self.current_tool_name is None:
|
||||
self.current_tool_name = ""
|
||||
# The name stops where the arguments start
|
||||
# And the arguments start with the `{` char
|
||||
if "{" in delta_text:
|
||||
tool_id = MistralToolCall.generate_random_id()
|
||||
delta_function_name = delta_text.split("{")[0]
|
||||
self.current_tool_name += delta_function_name
|
||||
# HF tokenizers may include [ARGS] in the text
|
||||
self.current_tool_name = self.current_tool_name.replace("[ARGS]", "")
|
||||
self.prev_tool_call_arr[self.current_tool_id]["name"] = (
|
||||
self.current_tool_name
|
||||
)
|
||||
delta_text = delta_text[len(delta_function_name) :]
|
||||
self.streaming_state = StreamingState.PARSING_ARGUMENTS
|
||||
else:
|
||||
# we want to send the tool name once it's complete
|
||||
self.current_tool_name += delta_text
|
||||
return []
|
||||
if self.streaming_state == StreamingState.PARSING_ARGUMENTS:
|
||||
next_function_text = None
|
||||
if self.bot_token in delta_text:
|
||||
# current tool call is over
|
||||
delta_arguments = ""
|
||||
delta_arguments += delta_text.split(self.bot_token)[0]
|
||||
next_function_text = delta_text[len(delta_arguments) :]
|
||||
self.streaming_state = StreamingState.TOOL_COMPLETE
|
||||
else:
|
||||
delta_arguments = delta_text
|
||||
self.streamed_args_for_tool[self.current_tool_id] += delta_arguments
|
||||
self.prev_tool_call_arr[self.current_tool_id]["arguments"] = (
|
||||
self.streamed_args_for_tool[self.current_tool_id]
|
||||
)
|
||||
ret = []
|
||||
if self.current_tool_name or delta_arguments:
|
||||
ret += [
|
||||
DeltaToolCall(
|
||||
index=self.current_tool_id,
|
||||
type="function",
|
||||
id=tool_id,
|
||||
function=DeltaFunctionCall(
|
||||
name=self.current_tool_name, arguments=delta_arguments
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
]
|
||||
self.current_tool_name = None
|
||||
if next_function_text:
|
||||
ret += self._generate_delta_tool_call(next_function_text)
|
||||
return ret
|
||||
# Should not happen
|
||||
return []
|
||||
|
||||
@ijson.coroutine
|
||||
def update_stream_state_pre_v11_tokenizer(self):
|
||||
while True:
|
||||
(prefix, event, value) = yield
|
||||
|
||||
if prefix == "item" and event == "start_map":
|
||||
self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
|
||||
self.starting_new_tool = True
|
||||
if prefix == "item" and event == "map_key" and value == "name":
|
||||
self.streaming_state = StreamingState.PARSING_NAME
|
||||
if prefix == "item.name" and event == "string":
|
||||
self.current_tool_name = value
|
||||
self.streaming_state = StreamingState.PARSING_NAME_COMPLETED
|
||||
if prefix == "item" and event == "map_key" and value == "arguments":
|
||||
self.streaming_state = StreamingState.WAITING_FOR_ARGUMENTS_START
|
||||
if prefix == "item.arguments" and event == "start_map":
|
||||
self.streaming_state = StreamingState.PARSING_ARGUMENTS
|
||||
if prefix == "item.arguments" and event == "end_map":
|
||||
self.streaming_state = StreamingState.PARSING_ARGUMENTS_COMPLETED
|
||||
if prefix == "item" and event == "end_map":
|
||||
self.streaming_state = StreamingState.TOOL_COMPLETE
|
||||
if prefix == "" and event == "end_array":
|
||||
self.streaming_state = StreamingState.ALL_TOOLS_COMPLETE
|
||||
|
||||
def _extract_tool_calls_streaming_pre_v11_tokenizer(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: Sequence[int],
|
||||
) -> DeltaMessage | None:
|
||||
"""
|
||||
Extracts tool calls for Mistral models
|
||||
doing tool calls of the following format:
|
||||
`[TOOL_CALLS][{"name": "add", "arguments":{"a": 3.5, "b": 4}}`
|
||||
"""
|
||||
assert self.parse_coro is not None
|
||||
content = None
|
||||
delta_tool_calls: list[DeltaToolCall] = []
|
||||
current_tool_call: DeltaToolCall = DeltaToolCall(
|
||||
index=self.current_tool_id, type="function"
|
||||
)
|
||||
current_tool_call_modified = False
|
||||
if self.bot_token_id in delta_token_ids or self.bot_token in delta_text:
|
||||
# this is the first tool call
|
||||
if not delta_text.startswith(self.bot_token):
|
||||
content = delta_text.split(self.bot_token)[0]
|
||||
delta_text = "".join(delta_text.split(self.bot_token)[1:])
|
||||
|
||||
# Cut smartly the delta text to catch the ijson events
|
||||
# as ijson does not give us the index in the text at each event.
|
||||
# We need to cut so that we know
|
||||
# where in the text the events are emitted from.
|
||||
while len(delta_text) > 0:
|
||||
streaming_state_before_parse = self.streaming_state
|
||||
|
||||
if self.streaming_state == StreamingState.WAITING_FOR_TOOL_START:
|
||||
delta_to_be_parsed, delta_text = self._split_delta(
|
||||
delta_text=delta_text,
|
||||
stop_after_opening_curly_braces=1,
|
||||
)
|
||||
elif self.streaming_state == StreamingState.WAITING_FOR_TOOL_KEY:
|
||||
# Wait until another key is sent
|
||||
# or the current tool is completed
|
||||
delta_to_be_parsed, delta_text = self._split_delta(
|
||||
delta_text=delta_text,
|
||||
stop_after_colon=1,
|
||||
stop_after_opening_curly_braces=1,
|
||||
# if the tool ends, we want to separate
|
||||
# at the start of the next tool
|
||||
)
|
||||
elif self.streaming_state == StreamingState.PARSING_NAME:
|
||||
delta_to_be_parsed, delta_text = self._split_delta(
|
||||
delta_text=delta_text,
|
||||
stop_after_comma=1,
|
||||
stop_after_closing_brackets=1,
|
||||
)
|
||||
elif self.streaming_state == StreamingState.WAITING_FOR_ARGUMENTS_START:
|
||||
delta_to_be_parsed, delta_text = self._split_delta(
|
||||
delta_text=delta_text,
|
||||
stop_after_opening_curly_braces=1,
|
||||
)
|
||||
elif self.streaming_state == StreamingState.PARSING_ARGUMENTS:
|
||||
delta_to_be_parsed, delta_text = self._split_delta(
|
||||
delta_text=delta_text,
|
||||
stop_after_closing_curly_braces=1,
|
||||
# we could be more clever
|
||||
# by listening to item.arguments.* start_map events
|
||||
# and know how many curly braces we can allow
|
||||
)
|
||||
elif self.streaming_state in [
|
||||
StreamingState.PARSING_ARGUMENTS_COMPLETED,
|
||||
StreamingState.PARSING_NAME_COMPLETED,
|
||||
]:
|
||||
delta_to_be_parsed, delta_text = self._split_delta(
|
||||
delta_text=delta_text,
|
||||
stop_after_closing_curly_braces=1,
|
||||
stop_after_closing_brackets=1,
|
||||
)
|
||||
elif self.streaming_state == StreamingState.TOOL_COMPLETE:
|
||||
delta_to_be_parsed, delta_text = self._split_delta(
|
||||
delta_text=delta_text,
|
||||
stop_after_opening_curly_braces=1,
|
||||
stop_after_closing_brackets=1,
|
||||
)
|
||||
elif self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE:
|
||||
content = delta_text
|
||||
delta_text = ""
|
||||
else:
|
||||
delta_to_be_parsed = delta_text
|
||||
delta_text = ""
|
||||
|
||||
if self.streaming_state != StreamingState.ALL_TOOLS_COMPLETE:
|
||||
self.parse_coro.send(delta_to_be_parsed.encode("utf-8"))
|
||||
|
||||
# Given the parsed text and the possible streaming state change,
|
||||
# let's add to the tool delta
|
||||
# start_map is the authoritative new-tool signal and survives
|
||||
# batched deltas, unlike comparing pre/post streaming states
|
||||
if self.starting_new_tool:
|
||||
self.starting_new_tool = False
|
||||
if current_tool_call_modified:
|
||||
if self.current_tool_mistral_id is not None:
|
||||
current_tool_call.id = self.current_tool_mistral_id
|
||||
self.current_tool_mistral_id = None
|
||||
self._track_streamed_args_pre_v11(current_tool_call)
|
||||
delta_tool_calls.append(current_tool_call)
|
||||
current_tool_call_modified = False
|
||||
self.current_tool_id += 1
|
||||
self.streamed_args_for_tool.append("")
|
||||
self.prev_tool_call_arr.append({})
|
||||
self.current_tool_mistral_id = MistralToolCall.generate_random_id()
|
||||
current_tool_call = DeltaToolCall(
|
||||
index=self.current_tool_id,
|
||||
type="function",
|
||||
)
|
||||
if current_tool_call.function is None:
|
||||
current_tool_call.function = DeltaFunctionCall()
|
||||
|
||||
if self.current_tool_name is not None:
|
||||
# we have the complete tool name
|
||||
current_tool_call_modified = True
|
||||
current_tool_call.function.name = self.current_tool_name
|
||||
self.prev_tool_call_arr[self.current_tool_id]["name"] = (
|
||||
self.current_tool_name
|
||||
)
|
||||
self.current_tool_name = None
|
||||
if self.streaming_state == StreamingState.PARSING_NAME_COMPLETED:
|
||||
self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
|
||||
if self.streaming_state in [
|
||||
StreamingState.PARSING_ARGUMENTS,
|
||||
StreamingState.PARSING_ARGUMENTS_COMPLETED,
|
||||
]:
|
||||
if self.streaming_state == StreamingState.PARSING_ARGUMENTS_COMPLETED:
|
||||
self.streaming_state = StreamingState.WAITING_FOR_TOOL_KEY
|
||||
# the delta_to_be_parsed is part of arguments.
|
||||
current_tool_call_modified = True
|
||||
if current_tool_call.function.arguments is None:
|
||||
current_tool_call.function.arguments = delta_to_be_parsed
|
||||
else:
|
||||
current_tool_call.function.arguments += delta_to_be_parsed
|
||||
if streaming_state_before_parse != StreamingState.PARSING_ARGUMENTS:
|
||||
# It's the first chunk of arg. let's lstrip it
|
||||
current_tool_call.function.arguments = (
|
||||
current_tool_call.function.arguments.lstrip()
|
||||
)
|
||||
|
||||
if current_tool_call_modified:
|
||||
if self.current_tool_mistral_id is not None:
|
||||
current_tool_call.id = self.current_tool_mistral_id
|
||||
self.current_tool_mistral_id = None
|
||||
self._track_streamed_args_pre_v11(current_tool_call)
|
||||
delta_tool_calls.append(current_tool_call)
|
||||
|
||||
if content or len(delta_tool_calls) > 0:
|
||||
delta_message = DeltaMessage()
|
||||
if content:
|
||||
delta_message.content = content
|
||||
if len(delta_tool_calls) > 0:
|
||||
delta_message.tool_calls = delta_tool_calls
|
||||
return delta_message
|
||||
else:
|
||||
if self.streaming_state == StreamingState.ALL_TOOLS_COMPLETE:
|
||||
return DeltaMessage()
|
||||
else:
|
||||
return None
|
||||
|
||||
def _track_streamed_args_pre_v11(self, tool_call: DeltaToolCall) -> None:
|
||||
r"""Accumulate `tool_call` arguments into the streaming state."""
|
||||
if tool_call.function is not None and tool_call.function.arguments is not None:
|
||||
self.streamed_args_for_tool[self.current_tool_id] += (
|
||||
tool_call.function.arguments
|
||||
)
|
||||
self.prev_tool_call_arr[self.current_tool_id]["arguments"] = (
|
||||
self.streamed_args_for_tool[self.current_tool_id]
|
||||
)
|
||||
|
||||
def _split_delta(
|
||||
self,
|
||||
delta_text: str,
|
||||
stop_after_quotes: int = -1,
|
||||
stop_after_opening_curly_braces: int = -1,
|
||||
stop_after_closing_curly_braces: int = -1,
|
||||
stop_after_closing_brackets: int = -1,
|
||||
stop_after_colon: int = -1,
|
||||
stop_after_comma=-1,
|
||||
) -> tuple[str, str]:
|
||||
delta_to_be_parsed = ""
|
||||
for i, c in enumerate(delta_text):
|
||||
if c in ['"', "'"]:
|
||||
delta_to_be_parsed += c
|
||||
stop_after_quotes -= 1
|
||||
if stop_after_quotes == 0:
|
||||
return (delta_to_be_parsed, delta_text[i + 1 :])
|
||||
elif c == "{":
|
||||
delta_to_be_parsed += c
|
||||
stop_after_opening_curly_braces -= 1
|
||||
if stop_after_opening_curly_braces == 0:
|
||||
return (delta_to_be_parsed, delta_text[i + 1 :])
|
||||
elif c == "}":
|
||||
delta_to_be_parsed += c
|
||||
stop_after_closing_curly_braces -= 1
|
||||
if stop_after_closing_curly_braces == 0:
|
||||
return (delta_to_be_parsed, delta_text[i + 1 :])
|
||||
elif c == "]":
|
||||
delta_to_be_parsed += c
|
||||
stop_after_closing_brackets -= 1
|
||||
if stop_after_closing_brackets == 0:
|
||||
return (delta_to_be_parsed, delta_text[i + 1 :])
|
||||
elif c == ":":
|
||||
delta_to_be_parsed += c
|
||||
stop_after_colon -= 1
|
||||
if stop_after_colon == 0:
|
||||
return (delta_to_be_parsed, delta_text[i + 1 :])
|
||||
elif c == ",":
|
||||
delta_to_be_parsed += c
|
||||
stop_after_comma -= 1
|
||||
if stop_after_comma == 0:
|
||||
return (delta_to_be_parsed, delta_text[i + 1 :])
|
||||
else:
|
||||
delta_to_be_parsed += c
|
||||
|
||||
return (delta_to_be_parsed, "")
|
||||
# Skip the base ToolParser tool_choice -> structured_outputs
|
||||
# conversion. Mistral enforces tool_choice via the grammar ``mode``
|
||||
# (auto/none/required/named); forwarding a tool-derived json_schema
|
||||
# would be unioned into the grammar by mistral-common, allowing raw
|
||||
# JSON in place of a real [TOOL_CALLS] call. Genuine user-provided
|
||||
# structured output on the request is still honored by the engine's
|
||||
# adjust_request.
|
||||
return self._parser_engine.adjust_request(request)
|
||||
|
||||
@@ -99,7 +99,7 @@ def extract_named_tool_call_streaming(
|
||||
else:
|
||||
if is_mistral_tokenizer(tokenizer):
|
||||
# Import mistral_common only if we need it.
|
||||
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
|
||||
from vllm.parser.mistral import MistralToolCall
|
||||
|
||||
tool_call_id = MistralToolCall.generate_random_id()
|
||||
else:
|
||||
|
||||
+5
-10
@@ -12,10 +12,8 @@ from vllm.utils.import_utils import LazyLoader
|
||||
if TYPE_CHECKING:
|
||||
# if type checking, eagerly import the module
|
||||
import vllm.tokenizers.mistral as mt
|
||||
import vllm.tool_parsers.mistral_tool_parser as mtp
|
||||
else:
|
||||
mt = LazyLoader("mt", globals(), "vllm.tokenizers.mistral")
|
||||
mtp = LazyLoader("mtp", globals(), "vllm.tool_parsers.mistral_tool_parser")
|
||||
|
||||
|
||||
def is_mistral_tokenizer(obj: TokenizerLike | None) -> TypeGuard[mt.MistralTokenizer]:
|
||||
@@ -31,13 +29,10 @@ def is_mistral_tokenizer(obj: TokenizerLike | None) -> TypeGuard[mt.MistralToken
|
||||
|
||||
|
||||
def is_mistral_tool_parser(cls: type | None) -> bool:
|
||||
"""Return true if *cls* is (a subclass of) MistralToolParser.
|
||||
"""Return true if *cls* carries the ``IS_MISTRAL_TOOL_PARSER`` marker.
|
||||
|
||||
Uses a class attribute check so that importing
|
||||
``vllm.tool_parsers.mistral_tool_parser`` — and transitively
|
||||
``mistral_common`` — is not required.
|
||||
The marker is set on the engine-adapter subclass registered under the
|
||||
``"mistral"`` tool-parser key. No import of the parser module is
|
||||
required.
|
||||
"""
|
||||
return bool(
|
||||
getattr(cls, "IS_MISTRAL_TOOL_PARSER", False)
|
||||
and issubclass(cls, mtp.MistralToolParser) # type: ignore[arg-type]
|
||||
)
|
||||
return bool(getattr(cls, "IS_MISTRAL_TOOL_PARSER", False))
|
||||
|
||||
Reference in New Issue
Block a user