[Mistral Grammar] Fix tool and reasoning parsing (#39217)
Signed-off-by: juliendenize <julien.denize@mistral.ai>
This commit is contained in:
@@ -1,12 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from random import choices
|
||||
from string import ascii_letters, digits
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import ijson
|
||||
import regex as re
|
||||
@@ -37,14 +40,19 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning.mistral_reasoning_parser import MistralReasoningParser
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers.mistral import MistralTokenizer, adapt_inplace_to_mistral_tool
|
||||
from vllm.tool_parsers.abstract_tool_parser import (
|
||||
Tool,
|
||||
ToolParser,
|
||||
)
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.reasoning import ReasoningParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
ALPHANUMERIC = ascii_letters + digits
|
||||
@@ -86,13 +94,28 @@ def _is_pre_v11_tokeniser(model_tokenizer: TokenizerLike) -> bool:
|
||||
return not (is_mistral_tokenizer(model_tokenizer) and model_tokenizer.version >= 11)
|
||||
|
||||
|
||||
class MistralToolParser(ToolParser):
|
||||
@dataclass
|
||||
class MistralStreamingResult:
|
||||
r"""Encapsulates the mutable state returned from
|
||||
`MistralToolParser.extract_maybe_reasoning_and_tool_streaming`.
|
||||
"""
|
||||
Tool call parser for Mistral 7B Instruct v0.3, intended for use with
|
||||
- [`mistral_common`](https://github.com/mistralai/mistral-common/)
|
||||
- the examples/tool_chat_template_mistral.jinja template.
|
||||
|
||||
Used when --enable-auto-tool-choice --tool-call-parser mistral are all set
|
||||
delta_message: DeltaMessage | None
|
||||
reasoning_ended: bool
|
||||
tools_called: bool
|
||||
current_text: str
|
||||
current_token_ids: list[int]
|
||||
|
||||
|
||||
class MistralToolParser(ToolParser):
|
||||
r"""Tool call parser for Mistral models, intended for use with either:
|
||||
|
||||
- `mistral_common <https://github.com/mistralai/mistral-common/>`_
|
||||
(recommended)
|
||||
- the `examples/tool_chat_template_mistral.jinja` template.
|
||||
|
||||
Used when `--enable-auto-tool-choice --tool-call-parser mistral` are all
|
||||
set.
|
||||
"""
|
||||
|
||||
# Used to generate correct grammar in `adjust_request`
|
||||
@@ -210,9 +233,11 @@ class MistralToolParser(ToolParser):
|
||||
reasoning=self.model_can_reason
|
||||
)
|
||||
|
||||
tools = (
|
||||
mistral_tools = (
|
||||
[
|
||||
MistralTool.from_openai(openai_tool=tool.model_dump())
|
||||
MistralTool.model_validate(
|
||||
adapt_inplace_to_mistral_tool(tool.model_dump())
|
||||
)
|
||||
for tool in request.tools
|
||||
]
|
||||
if request.tools is not None
|
||||
@@ -244,15 +269,158 @@ class MistralToolParser(ToolParser):
|
||||
lark_grammar = grammar_factory.get_lark_from_jinja(
|
||||
template=template,
|
||||
mode=tool_choice,
|
||||
tools=tools,
|
||||
tools=mistral_tools,
|
||||
json_schema=json_schema,
|
||||
parallel_tool_calls=request.parallel_tool_calls,
|
||||
json_only=False,
|
||||
)
|
||||
|
||||
request.structured_outputs = StructuredOutputsParams(grammar=lark_grammar)
|
||||
request._grammar_from_tool_parser = True
|
||||
return request
|
||||
|
||||
def extract_maybe_reasoning_and_tool_streaming(
|
||||
self,
|
||||
*,
|
||||
reasoning_parser: ReasoningParser | None,
|
||||
previous_text: str,
|
||||
current_text: str,
|
||||
delta_text: str,
|
||||
previous_token_ids: list[int],
|
||||
current_token_ids: list[int],
|
||||
output_token_ids: Sequence[int],
|
||||
reasoning_ended: bool,
|
||||
prompt_is_reasoning_end: bool | None,
|
||||
request: ChatCompletionRequest,
|
||||
) -> MistralStreamingResult:
|
||||
r"""Streaming extraction with reasoning followed by tool-call parsing.
|
||||
|
||||
This method encapsulates the combined reasoning extraction and
|
||||
tool-call streaming logic so that the serving layer only needs a
|
||||
thin routing branch.
|
||||
|
||||
The flow is:
|
||||
|
||||
1. If a *reasoning_parser* is present and reasoning has **not** ended,
|
||||
extract reasoning tokens. Pre-v15 models may have pre-filled
|
||||
`[THINK]...[/THINK]` in system prompts, so we skip the
|
||||
prompt-level reasoning-end check for those.
|
||||
2. Once reasoning ends (or if there is no reasoning parser), delegate
|
||||
to `extract_tool_calls_streaming` and track whether tools were
|
||||
called.
|
||||
|
||||
Args:
|
||||
reasoning_parser: Optional reasoning parser instance.
|
||||
previous_text: Accumulated text from prior chunks.
|
||||
current_text: Full accumulated text including current chunk.
|
||||
delta_text: New text in this chunk.
|
||||
previous_token_ids: Token ids from prior chunks.
|
||||
current_token_ids: Full token ids including current chunk.
|
||||
output_token_ids: Raw output token ids from the engine.
|
||||
reasoning_ended: Whether reasoning has already ended.
|
||||
prompt_is_reasoning_end: Whether the prompt itself ends reasoning.
|
||||
request: The originating chat completion request.
|
||||
"""
|
||||
delta_message: DeltaMessage | None = None
|
||||
tools_called = False
|
||||
reasoning_ended_at_entry = reasoning_ended
|
||||
|
||||
# For MistralReasoningParser, only enter the reasoning block when
|
||||
# the model has actually emitted a [THINK] token. Other reasoning
|
||||
# parsers always expect thinking to be present.
|
||||
expect_thinking = (
|
||||
not isinstance(reasoning_parser, MistralReasoningParser)
|
||||
or reasoning_parser.start_token_id in current_token_ids
|
||||
)
|
||||
if reasoning_parser is not None and not reasoning_ended and expect_thinking:
|
||||
# Pre-v15 models may have pre-filled [THINK]...[/THINK] in
|
||||
# system prompts, so skip the prompt-level reasoning-end
|
||||
# check and wait for the output's own end-of-think.
|
||||
is_pre_v15 = (
|
||||
isinstance(self.model_tokenizer, MistralTokenizer)
|
||||
and self.model_tokenizer.version < 15
|
||||
)
|
||||
|
||||
if not is_pre_v15 and prompt_is_reasoning_end:
|
||||
reasoning_ended = True
|
||||
current_token_ids = list(output_token_ids)
|
||||
else:
|
||||
delta_message = reasoning_parser.extract_reasoning_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
output_token_ids,
|
||||
)
|
||||
if reasoning_parser.is_reasoning_end_streaming(
|
||||
current_token_ids, output_token_ids
|
||||
):
|
||||
reasoning_ended = True
|
||||
current_token_ids = reasoning_parser.extract_content_ids(
|
||||
list(output_token_ids)
|
||||
)
|
||||
if delta_message and delta_message.content:
|
||||
current_text = delta_message.content
|
||||
delta_message.content = None
|
||||
else:
|
||||
current_text = ""
|
||||
|
||||
if not reasoning_ended:
|
||||
return MistralStreamingResult(
|
||||
delta_message=delta_message,
|
||||
reasoning_ended=False,
|
||||
tools_called=False,
|
||||
current_text=current_text,
|
||||
current_token_ids=current_token_ids,
|
||||
)
|
||||
|
||||
delta_token_ids = list(output_token_ids)
|
||||
|
||||
# On the iteration where reasoning just ended, reset the text/token
|
||||
# state so the tool parser sees a clean history instead of the
|
||||
# accumulated reasoning text.
|
||||
if not reasoning_ended_at_entry and reasoning_ended:
|
||||
previous_text = ""
|
||||
previous_token_ids = []
|
||||
delta_text = current_text
|
||||
delta_token_ids = current_token_ids
|
||||
|
||||
delta_message = self.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=previous_token_ids,
|
||||
current_token_ids=current_token_ids,
|
||||
delta_token_ids=delta_token_ids,
|
||||
request=request,
|
||||
)
|
||||
if delta_message and delta_message.tool_calls:
|
||||
tools_called = True
|
||||
|
||||
return MistralStreamingResult(
|
||||
delta_message=delta_message,
|
||||
reasoning_ended=reasoning_ended,
|
||||
tools_called=tools_called,
|
||||
current_text=current_text,
|
||||
current_token_ids=current_token_ids,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def build_non_streaming_tool_calls(
|
||||
tool_calls: list[FunctionCall] | None,
|
||||
) -> list[ToolCall]:
|
||||
r"""Build `MistralToolCall` items for non-streaming responses."""
|
||||
if not tool_calls:
|
||||
return []
|
||||
|
||||
return [
|
||||
MistralToolCall(id=tc.id, function=tc)
|
||||
if tc.id
|
||||
else MistralToolCall(function=tc)
|
||||
for tc in tool_calls
|
||||
]
|
||||
|
||||
def extract_tool_calls(
|
||||
self,
|
||||
model_output: str,
|
||||
@@ -323,7 +491,7 @@ class MistralToolParser(ToolParser):
|
||||
)[0]
|
||||
tool_calls = json.loads(raw_tool_call)
|
||||
except (IndexError, json.JSONDecodeError):
|
||||
logger.exception("Error in extracting tool call from response: {e}")
|
||||
logger.exception("Error in extracting tool call from response.")
|
||||
# If raw decoding and decoding post regex rule fails, then just
|
||||
# return content.
|
||||
return ExtractedToolCallInformation(
|
||||
|
||||
Reference in New Issue
Block a user