[PARSER][Mistral] unified engine-based parser for reasoning and tool calls (#48947)

This commit is contained in:
Julien Denize
2026-07-30 15:38:00 +02:00
committed by GitHub
parent e2efe79695
commit 1a20d23dab
30 changed files with 2934 additions and 1495 deletions
+144 -2
View File
@@ -8,11 +8,13 @@ import llguidance
import pytest
from mistral_common.exceptions import InvalidMessageStructureException
from mistral_common.guidance.grammar_factory import GrammarFactory
from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy, SpecialTokens
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.tokenizers.mistral import (
MistralTokenizer,
_validate_apply_chat_template_args,
validate_request_params,
)
@@ -1247,7 +1249,9 @@ class TestMistralTokenizer:
expected_strings = (
'[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}] I am an AI\n\nHello world ![TOOL_CALLS][{"name": "get_weather", "arguments": {"city": "Paris"}, "id": "123456789"}] {"content": {"temperature": 20, "unit": "celsius"}, "call_id": "123456789"}', # noqa: E501
'I am an AI[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}]Hello world ![TOOL_CALLS]get_weather{"city": "Paris"}{"temperature": 20, "unit": "celsius"}', # noqa: E501
# v11+ tool-call decode emits the explicit [ARGS] separator between
# the function name and its JSON arguments (get_weather[ARGS]{...}).
'I am an AI[{"type": "function", "function": {"name": "get_weather", "description": "Gets the current weather in a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city name"}}, "required": ["city"]}}}]Hello world ![TOOL_CALLS]get_weather[ARGS]{"city": "Paris"}{"temperature": 20, "unit": "celsius"}', # noqa: E501
)
assert (
@@ -1498,6 +1502,7 @@ class TestMistralTokenizer:
"get",
"_",
"weather",
"[ARGS]",
'{"',
"city",
'":',
@@ -2234,3 +2239,140 @@ class TestMistralTokenizer:
decoded = mistral_tokenizer.tokenizer.decode(output, SpecialTokenPolicy.KEEP)
assert "[THINK]2+2 equals 4[/THINK]" in decoded
def test_convert_ids_to_tokens_pre_args_tekken():
"""convert_ids_to_tokens must not raise on pre-[ARGS] Tekken tokenizers.
Tekken tokenizers before v11 (e.g. Ministral-8B-Instruct-2410) have no
[ARGS] special token. This guards the is_special([ARGS]) gate in
MistralTokenizer.convert_ids_to_tokens, which must skip [ARGS] rather than
call get_special_token(args) unconditionally (which raises on Tekken).
"""
tokenizer = MistralTokenizer.from_pretrained("mistralai/Ministral-8B-Instruct-2410")
assert tokenizer.is_tekken
assert not tokenizer.tokenizer.is_special(SpecialTokens.args)
ids = tokenizer.encode("Hello world !", add_special_tokens=False)
tokens = tokenizer.convert_ids_to_tokens(ids, skip_special_tokens=True)
assert len(tokens) > 0
# ---------------------------------------------------------------------------
# validate_request_params – reasoning_effort validation (no tokenizer needed)
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("reasoning_effort", [None, "none", "high"])
def test_validate_request_params_valid_effort(reasoning_effort: str | None) -> None:
request = ChatCompletionRequest(
model="test-model",
messages=[],
reasoning_effort=reasoning_effort,
)
assert validate_request_params(request) is None
def test_validate_request_params_rejects_unsupported_effort() -> None:
request = ChatCompletionRequest(
model="test-model",
messages=[],
reasoning_effort="medium",
)
with pytest.raises(ValueError, match="reasoning_effort"):
validate_request_params(request)
# ---------------------------------------------------------------------------
# apply_chat_template reasoning_effort passthrough (v15 tokenizer)
# ---------------------------------------------------------------------------
_PASSTHROUGH_MESSAGES = [{"role": "user", "content": "Hello"}]
@pytest.fixture(scope="module")
def v15_mistral_tokenizer() -> MistralTokenizer:
"""Load the v15 Mistral tokenizer; skip if unavailable."""
try:
return MistralTokenizer.from_pretrained("mistralai/Mistral-Small-4-119B-2603")
except Exception:
pytest.skip("v15 tokenizer unavailable")
@pytest.fixture(scope="module")
def v13_mistral_tokenizer() -> MistralTokenizer:
"""Load the v13 Mistral tokenizer; skip if unavailable."""
try:
return MistralTokenizer.from_pretrained("mistralai/Magistral-Small-2509")
except Exception:
pytest.skip("v13 tokenizer unavailable")
def test_v15_apply_chat_template_passes_reasoning_effort_high(
monkeypatch: pytest.MonkeyPatch,
v15_mistral_tokenizer: MistralTokenizer,
) -> None:
captured_kwargs: list[dict] = []
def fake_apply_chat_template(**kwargs):
captured_kwargs.append(kwargs)
return [0]
monkeypatch.setattr(
v15_mistral_tokenizer.transformers_tokenizer,
"apply_chat_template",
fake_apply_chat_template,
)
v15_mistral_tokenizer.apply_chat_template(
messages=_PASSTHROUGH_MESSAGES,
reasoning_effort="high",
)
assert captured_kwargs[-1]["reasoning_effort"] == "high"
def test_v15_apply_chat_template_passes_reasoning_effort_none_by_default(
monkeypatch: pytest.MonkeyPatch,
v15_mistral_tokenizer: MistralTokenizer,
) -> None:
captured_kwargs: list[dict] = []
def fake_apply_chat_template(**kwargs):
captured_kwargs.append(kwargs)
return [0]
monkeypatch.setattr(
v15_mistral_tokenizer.transformers_tokenizer,
"apply_chat_template",
fake_apply_chat_template,
)
v15_mistral_tokenizer.apply_chat_template(messages=_PASSTHROUGH_MESSAGES)
assert "reasoning_effort" in captured_kwargs[-1]
assert captured_kwargs[-1]["reasoning_effort"] is None
def test_pre_v15_apply_chat_template_omits_reasoning_effort(
monkeypatch: pytest.MonkeyPatch,
v13_mistral_tokenizer: MistralTokenizer,
) -> None:
captured_kwargs: list[dict] = []
def fake_apply_chat_template(**kwargs):
captured_kwargs.append(kwargs)
return [0]
monkeypatch.setattr(
v13_mistral_tokenizer.transformers_tokenizer,
"apply_chat_template",
fake_apply_chat_template,
)
v13_mistral_tokenizer.apply_chat_template(
messages=_PASSTHROUGH_MESSAGES,
reasoning_effort="high",
)
assert "reasoning_effort" not in captured_kwargs[-1]