[Refactor][Mistral] Extract parsing logic into MistralParser (#44596)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -43,7 +43,6 @@ from vllm.tool_parsers.streaming import (
|
||||
extract_required_tool_call_streaming,
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.mistral import is_mistral_tool_parser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -546,14 +545,6 @@ class DelegatingParser(Parser):
|
||||
if tool_parser is None:
|
||||
return [], content
|
||||
|
||||
# When the Mistral grammar factory injected structured outputs,
|
||||
# let the parser handle the output.
|
||||
use_mistral_tool_parser = (
|
||||
is_mistral_tool_parser(type(tool_parser))
|
||||
and isinstance(request, ChatCompletionRequest)
|
||||
and request._grammar_from_tool_parser
|
||||
)
|
||||
|
||||
supports_required_and_named = tool_parser.supports_required_and_named
|
||||
is_named_tool_choice = request.tool_choice and isinstance(
|
||||
request.tool_choice,
|
||||
@@ -570,11 +561,7 @@ class DelegatingParser(Parser):
|
||||
)
|
||||
|
||||
tool_calls = list[FunctionCall]()
|
||||
if (
|
||||
is_named_tool_choice
|
||||
and supports_required_and_named
|
||||
and not use_mistral_tool_parser
|
||||
):
|
||||
if is_named_tool_choice and supports_required_and_named:
|
||||
if content is None:
|
||||
return [], None
|
||||
tool_calls.append(
|
||||
@@ -584,11 +571,7 @@ class DelegatingParser(Parser):
|
||||
)
|
||||
)
|
||||
content = None
|
||||
elif (
|
||||
is_required_tool_choice
|
||||
and supports_required_and_named
|
||||
and not use_mistral_tool_parser
|
||||
):
|
||||
elif is_required_tool_choice and supports_required_and_named:
|
||||
# "required" with standard JSON-based parsing
|
||||
parsed_calls = []
|
||||
with contextlib.suppress(ValidationError):
|
||||
@@ -604,7 +587,7 @@ class DelegatingParser(Parser):
|
||||
)
|
||||
)
|
||||
content = None
|
||||
elif is_auto_tool_choice or use_mistral_tool_parser:
|
||||
elif is_auto_tool_choice:
|
||||
# Automatic Tool Call Parsing (also used as fallback for
|
||||
# required/named when supports_required_and_named=False)
|
||||
tool_call_info = tool_parser.extract_tool_calls(
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
|
||||
class MistralParser(DelegatingParser):
|
||||
def __init__(self, tokenizer, tools=None, *args, **kwargs):
|
||||
super().__init__(tokenizer, tools, *args, **kwargs)
|
||||
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
|
||||
|
||||
if not isinstance(self._tool_parser, MistralToolParser):
|
||||
raise ValueError(
|
||||
"MistralParser requires --tool-call-parser mistral, "
|
||||
f"got {self._tool_parser.__class__.__name__}."
|
||||
)
|
||||
|
||||
def _maybe_force_auto_tool_parsing(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> None:
|
||||
# When the Mistral grammar factory injected structured outputs,
|
||||
# the model emits v11+ format ([TOOL_CALLS]name{args}) that the
|
||||
# named/required parsers can't handle. Disable them so all
|
||||
# tool_choice modes fall back to auto tool parsing via
|
||||
# extract_tool_calls.
|
||||
if getattr(request, "_grammar_from_tool_parser", False):
|
||||
assert self._tool_parser is not None
|
||||
self._tool_parser.supports_required_and_named = False
|
||||
|
||||
def parse(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
enable_auto_tools: bool = False,
|
||||
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
||||
self._maybe_force_auto_tool_parsing(request)
|
||||
reasoning, content, tool_calls = super().parse(
|
||||
model_output, request, enable_auto_tools
|
||||
)
|
||||
if tool_calls:
|
||||
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
|
||||
|
||||
# Named/required tool_choice builds FunctionCalls without
|
||||
# ID, backfill with Mistral-format IDs.
|
||||
for tc in tool_calls:
|
||||
if not tc.id:
|
||||
tc.id = MistralToolCall.generate_random_id()
|
||||
return reasoning, content, tool_calls
|
||||
|
||||
def parse_delta(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: list[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
self._maybe_force_auto_tool_parsing(request)
|
||||
return super().parse_delta(
|
||||
delta_text,
|
||||
delta_token_ids,
|
||||
request,
|
||||
prompt_token_ids,
|
||||
finished=finished,
|
||||
)
|
||||
@@ -106,6 +106,15 @@ class ParserManager:
|
||||
if reasoning_parser_cls is None and tool_parser_cls is None:
|
||||
return None
|
||||
|
||||
from vllm.utils.mistral import is_mistral_tool_parser
|
||||
|
||||
if is_mistral_tool_parser(tool_parser_cls):
|
||||
from vllm.parser.mistral import MistralParser
|
||||
|
||||
MistralParser.reasoning_parser_cls = reasoning_parser_cls
|
||||
MistralParser.tool_parser_cls = tool_parser_cls
|
||||
return MistralParser
|
||||
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
|
||||
r_cls = reasoning_parser_cls
|
||||
|
||||
Reference in New Issue
Block a user