[Refactor] Unify reasoning + tool-call parsing behind Parser.parse() (#44267)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -4,7 +4,6 @@
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
|
||||
@@ -32,11 +31,8 @@ class _DummyDelegatingParser(DelegatingParser):
|
||||
):
|
||||
return None
|
||||
|
||||
def extract_tool_calls(self, model_output: str, request):
|
||||
return None
|
||||
|
||||
|
||||
def test_parse_tool_calls_from_content_allows_named_tool_choice_with_none_content():
|
||||
def test_chat_completion_named_tool_choice_with_none_content():
|
||||
request = ChatCompletionRequest.model_validate(
|
||||
{
|
||||
"model": "test-model",
|
||||
@@ -53,17 +49,15 @@ def test_parse_tool_calls_from_content_allows_named_tool_choice_with_none_conten
|
||||
"tool_choice": {"type": "function", "function": {"name": "get_weather"}},
|
||||
}
|
||||
)
|
||||
parser = _DummyDelegatingParser(tokenizer=None)
|
||||
|
||||
tool_calls, content = OpenAIServing._parse_tool_calls_from_content(
|
||||
request=request,
|
||||
tokenizer=None,
|
||||
enable_auto_tools=True,
|
||||
tool_parser_cls=None,
|
||||
tool_calls, content = parser._extract_tool_calls(
|
||||
content=None,
|
||||
request=request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert tool_calls == []
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.parser.abstract_parser import _WrappedParser
|
||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
|
||||
|
||||
|
||||
class ThinkReasoningParser(BaseThinkingReasoningParser):
|
||||
@property
|
||||
def start_token(self) -> str:
|
||||
return "<think>"
|
||||
|
||||
@property
|
||||
def end_token(self) -> str:
|
||||
return "</think>"
|
||||
|
||||
|
||||
MODEL_OUTPUT = (
|
||||
"<think>let me think about this</think>"
|
||||
'<tool_call>\n{"name": "get_weather", '
|
||||
'"arguments": {"city": "Dallas"}}\n</tool_call>'
|
||||
)
|
||||
|
||||
PLAIN_TEXT = "The weather in Dallas is sunny and 75°F."
|
||||
|
||||
TOOL_CALL_ONLY = (
|
||||
'<tool_call>\n{"name": "get_weather", '
|
||||
'"arguments": {"city": "Dallas"}}\n</tool_call>'
|
||||
)
|
||||
|
||||
TOOL_ARGUMENTS = '{"city": "Dallas"}'
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer():
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
return get_tokenizer("Qwen/Qwen3-32B")
|
||||
|
||||
|
||||
def make_request(**overrides):
|
||||
base = {
|
||||
"model": "test-model",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}
|
||||
base.update(overrides)
|
||||
return ChatCompletionRequest.model_validate(base)
|
||||
|
||||
|
||||
TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def make_parser(tokenizer, reasoning=False, tool=False):
|
||||
_WrappedParser.reasoning_parser_cls = ThinkReasoningParser if reasoning else None
|
||||
_WrappedParser.tool_parser_cls = Hermes2ProToolParser if tool else None
|
||||
return _WrappedParser(tokenizer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"reasoning,tool",
|
||||
[(False, False), (False, True)],
|
||||
ids=["neither", "tool-only"],
|
||||
)
|
||||
def test_parse_plain_text_no_reasoning_parser(tokenizer, reasoning, tool):
|
||||
parser = make_parser(tokenizer, reasoning=reasoning, tool=tool)
|
||||
request = make_request()
|
||||
r, content, tool_calls = parser.parse(PLAIN_TEXT, request)
|
||||
|
||||
assert r is None
|
||||
assert content == PLAIN_TEXT
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"reasoning,tool",
|
||||
[(True, False), (True, True)],
|
||||
ids=["reasoning-only", "both"],
|
||||
)
|
||||
def test_parse_plain_text_with_reasoning_parser(tokenizer, reasoning, tool):
|
||||
parser = make_parser(tokenizer, reasoning=reasoning, tool=tool)
|
||||
request = make_request()
|
||||
r, content, tool_calls = parser.parse(PLAIN_TEXT, request)
|
||||
|
||||
assert r == PLAIN_TEXT
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
def test_parse_both_parsers(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=True)
|
||||
request = make_request(tools=TOOLS)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
MODEL_OUTPUT, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert reasoning is not None
|
||||
assert "let me think about this" in reasoning
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"}
|
||||
assert not content or content.strip() == ""
|
||||
|
||||
|
||||
def test_parse_reasoning_only(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=False)
|
||||
request = make_request()
|
||||
reasoning, content, tool_calls = parser.parse(MODEL_OUTPUT, request)
|
||||
|
||||
assert reasoning is not None
|
||||
assert "let me think about this" in reasoning
|
||||
assert content is not None
|
||||
assert "<tool_call>" in content
|
||||
assert "get_weather" in content
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
def test_parse_tool_only(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
request = make_request(tools=TOOLS)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
MODEL_OUTPUT, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"}
|
||||
|
||||
|
||||
def test_parse_named_tool_choice(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
request = make_request(
|
||||
tools=TOOLS,
|
||||
tool_choice={
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather"},
|
||||
},
|
||||
)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
TOOL_ARGUMENTS, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert tool_calls[0].arguments == TOOL_ARGUMENTS
|
||||
|
||||
|
||||
def test_parse_named_tool_choice_with_reasoning(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=True)
|
||||
model_output = f"<think>thinking</think>{TOOL_ARGUMENTS}"
|
||||
request = make_request(
|
||||
tools=TOOLS,
|
||||
tool_choice={
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather"},
|
||||
},
|
||||
)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
model_output, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert reasoning is not None
|
||||
assert "thinking" in reasoning
|
||||
assert content is None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert tool_calls[0].arguments == TOOL_ARGUMENTS
|
||||
|
||||
|
||||
def test_parse_required_tool_choice(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
functions_json = json.dumps(
|
||||
[
|
||||
{"name": "get_weather", "parameters": {"city": "Dallas"}},
|
||||
{"name": "get_time", "parameters": {"timezone": "UTC"}},
|
||||
]
|
||||
)
|
||||
request = make_request(tools=TOOLS, tool_choice="required")
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
functions_json, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 2
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"}
|
||||
assert tool_calls[1].name == "get_time"
|
||||
assert json.loads(tool_calls[1].arguments) == {"timezone": "UTC"}
|
||||
|
||||
|
||||
def test_parse_named_tool_choice_content_none(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
request = make_request(
|
||||
tools=TOOLS,
|
||||
tool_choice={
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather"},
|
||||
},
|
||||
)
|
||||
reasoning, content, tool_calls = parser.parse("", request, enable_auto_tools=True)
|
||||
assert reasoning is None
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
|
||||
|
||||
def test_parse_required_tool_choice_content_none(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
request = make_request(tools=TOOLS, tool_choice="required")
|
||||
reasoning, content, tool_calls = parser.parse("", request, enable_auto_tools=True)
|
||||
assert reasoning is None
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
def test_parse_auto_tools_no_parser(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=False)
|
||||
request = make_request()
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
TOOL_CALL_ONLY, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == TOOL_CALL_ONLY
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
def test_parse_auto_tools_no_calls_returns_none(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
request = make_request(tools=TOOLS)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
PLAIN_TEXT, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == PLAIN_TEXT
|
||||
assert tool_calls is None
|
||||
@@ -357,6 +357,14 @@ class OpenAIServingChat(OpenAIServing):
|
||||
assert len(generators) == 1
|
||||
(result_generator,) = generators
|
||||
|
||||
parser: Parser | None = None
|
||||
if self.parser_cls is not None:
|
||||
parser = self.parser_cls(
|
||||
tokenizer,
|
||||
request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
|
||||
if request.stream:
|
||||
return self.chat_completion_stream_generator(
|
||||
request,
|
||||
@@ -378,7 +386,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
conversation,
|
||||
tokenizer,
|
||||
request_metadata,
|
||||
reasoning_parser,
|
||||
parser,
|
||||
)
|
||||
|
||||
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
|
||||
@@ -934,7 +942,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
conversation: list[ConversationMessage],
|
||||
tokenizer: TokenizerLike,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
reasoning_parser: ReasoningParser | None = None,
|
||||
parser: Parser | None = None,
|
||||
) -> ErrorResponse | ChatCompletionResponse:
|
||||
created_time = int(time.time())
|
||||
final_res: RequestOutput | None = None
|
||||
@@ -1043,28 +1051,20 @@ class OpenAIServingChat(OpenAIServing):
|
||||
choices.append(choice_data)
|
||||
continue
|
||||
|
||||
if reasoning_parser:
|
||||
# If the reasoning parser is enabled,
|
||||
# tool calls are extracted exclusively from the content.
|
||||
reasoning, content = reasoning_parser.extract_reasoning(
|
||||
output.text, request=request
|
||||
if parser is not None:
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
output.text,
|
||||
request,
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
)
|
||||
if not request.include_reasoning:
|
||||
reasoning = None
|
||||
else:
|
||||
reasoning = None
|
||||
content = output.text
|
||||
tool_calls = []
|
||||
|
||||
auto_tools_called = False
|
||||
# if auto tools are not enabled, and a named tool choice using
|
||||
# outlines is not being used
|
||||
tool_calls, content = self._parse_tool_calls_from_content(
|
||||
request=request,
|
||||
tokenizer=tokenizer,
|
||||
content=content,
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
tool_parser_cls=self.tool_parser,
|
||||
)
|
||||
if is_mistral_tokenizer(tokenizer):
|
||||
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import contextlib
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Awaitable, Mapping
|
||||
@@ -9,8 +8,7 @@ from http import HTTPStatus
|
||||
from typing import Any, ClassVar, Generic, Protocol, TypeAlias, TypeVar
|
||||
|
||||
from fastapi import Request
|
||||
from openai.types.responses import ToolChoiceFunction
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError
|
||||
from pydantic import ConfigDict
|
||||
from starlette.datastructures import Headers
|
||||
|
||||
import vllm.envs as envs
|
||||
@@ -21,7 +19,6 @@ from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
)
|
||||
@@ -31,8 +28,6 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
FunctionCall,
|
||||
FunctionDefinition,
|
||||
GenerationError,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
@@ -61,14 +56,12 @@ from vllm.renderers.inputs.preprocess import (
|
||||
)
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers import ToolParser
|
||||
from vllm.tracing import (
|
||||
contains_trace_headers,
|
||||
extract_trace_headers,
|
||||
log_tracing_disabled_warning,
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.mistral import is_mistral_tool_parser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -451,124 +444,6 @@ class OpenAIServing(BeamSearchOnlineMixin):
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_tool_calls_from_content(
|
||||
request: ResponsesRequest | ChatCompletionRequest,
|
||||
tokenizer: TokenizerLike | None,
|
||||
enable_auto_tools: bool,
|
||||
tool_parser_cls: type[ToolParser] | None,
|
||||
content: str | None = None,
|
||||
) -> tuple[list[FunctionCall] | None, str | None]:
|
||||
# When the Mistral grammar factory injected structured outputs,
|
||||
# let the parser handle the output.
|
||||
use_mistral_tool_parser = (
|
||||
isinstance(request, ChatCompletionRequest)
|
||||
and is_mistral_tool_parser(tool_parser_cls)
|
||||
and request._grammar_from_tool_parser
|
||||
)
|
||||
|
||||
function_calls = list[FunctionCall]()
|
||||
if (
|
||||
not use_mistral_tool_parser
|
||||
and request.tool_choice
|
||||
and isinstance(request.tool_choice, ToolChoiceFunction)
|
||||
):
|
||||
# Forced Function Call (Responses API)
|
||||
if content is None:
|
||||
return [], None
|
||||
function_calls.append(
|
||||
FunctionCall(name=request.tool_choice.name, arguments=content)
|
||||
)
|
||||
content = None # Clear content since tool is called.
|
||||
elif (
|
||||
not use_mistral_tool_parser
|
||||
and request.tool_choice
|
||||
and isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
|
||||
and (tool_parser_cls is None or tool_parser_cls.supports_required_and_named)
|
||||
):
|
||||
# Named function with standard JSON-based parsing
|
||||
if content is None:
|
||||
return [], None
|
||||
function_calls.append(
|
||||
FunctionCall(name=request.tool_choice.function.name, arguments=content)
|
||||
)
|
||||
content = None # Clear content since tool is called.
|
||||
elif (
|
||||
not use_mistral_tool_parser
|
||||
and request.tool_choice == "required"
|
||||
and (tool_parser_cls is None or tool_parser_cls.supports_required_and_named)
|
||||
):
|
||||
# "required" with standard JSON-based parsing
|
||||
tool_calls = []
|
||||
with contextlib.suppress(ValidationError):
|
||||
content = content or ""
|
||||
tool_calls = TypeAdapter(list[FunctionDefinition]).validate_json(
|
||||
content
|
||||
)
|
||||
for tool_call in tool_calls:
|
||||
function_calls.append(
|
||||
FunctionCall(
|
||||
name=tool_call.name,
|
||||
arguments=json.dumps(tool_call.parameters, ensure_ascii=False),
|
||||
)
|
||||
)
|
||||
content = None # Clear content since tool is called.
|
||||
elif tool_parser_cls and (
|
||||
use_mistral_tool_parser
|
||||
or (
|
||||
enable_auto_tools
|
||||
and (
|
||||
request.tool_choice == "auto"
|
||||
or request.tool_choice is None
|
||||
or (
|
||||
not tool_parser_cls.supports_required_and_named
|
||||
and request.tools
|
||||
and (
|
||||
request.tool_choice == "required"
|
||||
or isinstance(
|
||||
request.tool_choice,
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
):
|
||||
# Automatic Tool Call Parsing (also used as fallback for
|
||||
# required/named when supports_required_and_named=False)
|
||||
if tokenizer is None:
|
||||
raise ValueError(
|
||||
"Tokenizer not available when `skip_tokenizer_init=True`"
|
||||
)
|
||||
|
||||
try:
|
||||
tool_parser = tool_parser_cls(tokenizer, request.tools)
|
||||
except RuntimeError as e:
|
||||
logger.exception("Error in tool parser creation.")
|
||||
raise e
|
||||
tool_call_info = tool_parser.extract_tool_calls(
|
||||
content if content is not None else "",
|
||||
request=request, # type: ignore
|
||||
)
|
||||
if tool_call_info is not None and tool_call_info.tools_called:
|
||||
# extract_tool_calls() returns a list of tool calls.
|
||||
function_calls.extend(
|
||||
FunctionCall(
|
||||
id=tool_call.id,
|
||||
name=tool_call.function.name,
|
||||
arguments=tool_call.function.arguments,
|
||||
)
|
||||
for tool_call in tool_call_info.tool_calls
|
||||
)
|
||||
content = tool_call_info.content
|
||||
if content and content.strip() == "":
|
||||
content = None
|
||||
else:
|
||||
# No tool calls.
|
||||
return None, content
|
||||
|
||||
return function_calls, content
|
||||
|
||||
@staticmethod
|
||||
def _get_decoded_token(
|
||||
logprob: Logprob,
|
||||
|
||||
@@ -44,6 +44,7 @@ from vllm.tool_parsers.streaming import (
|
||||
)
|
||||
from vllm.tool_parsers.utils import Tool
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.mistral import is_mistral_tool_parser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -313,6 +314,24 @@ class Parser:
|
||||
A DeltaMessage with tool_calls field, or None.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def parse(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
enable_auto_tools: bool = False,
|
||||
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
||||
"""Parse a complete model output, extracting reasoning and tool calls.
|
||||
|
||||
Args:
|
||||
model_output: The complete model-generated string.
|
||||
request: The request object used to generate the output.
|
||||
enable_auto_tools: Whether to enable automatic tool call parsing.
|
||||
|
||||
Returns:
|
||||
A tuple of (reasoning, content, tool_calls).
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def parse_delta(
|
||||
self,
|
||||
@@ -511,6 +530,99 @@ class DelegatingParser(Parser):
|
||||
# No tool calls
|
||||
return [], content
|
||||
|
||||
def _extract_tool_calls(
|
||||
self,
|
||||
content: str | None,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
enable_auto_tools: bool = False,
|
||||
) -> tuple[list[FunctionCall] | None, str | None]:
|
||||
tool_parser = self._tool_parser
|
||||
if tool_parser is None:
|
||||
return [], content
|
||||
|
||||
# When the Mistral grammar factory injected structured outputs,
|
||||
# let the parser handle the output.
|
||||
use_mistral_tool_parser = (
|
||||
is_mistral_tool_parser(type(tool_parser))
|
||||
and isinstance(request, ChatCompletionRequest)
|
||||
and request._grammar_from_tool_parser
|
||||
)
|
||||
|
||||
supports_required_and_named = tool_parser.supports_required_and_named
|
||||
is_named_tool_choice = request.tool_choice and isinstance(
|
||||
request.tool_choice,
|
||||
(ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
|
||||
)
|
||||
is_required_tool_choice = request.tool_choice == "required"
|
||||
is_auto_tool_choice = enable_auto_tools and (
|
||||
request.tool_choice == "auto"
|
||||
or request.tool_choice is None
|
||||
or (
|
||||
not supports_required_and_named
|
||||
and (is_named_tool_choice or is_required_tool_choice)
|
||||
)
|
||||
)
|
||||
|
||||
tool_calls = list[FunctionCall]()
|
||||
if (
|
||||
is_named_tool_choice
|
||||
and supports_required_and_named
|
||||
and not use_mistral_tool_parser
|
||||
):
|
||||
if content is None:
|
||||
return [], None
|
||||
tool_calls.append(
|
||||
FunctionCall(
|
||||
name=self._get_function_name(request),
|
||||
arguments=content,
|
||||
)
|
||||
)
|
||||
content = None
|
||||
elif (
|
||||
is_required_tool_choice
|
||||
and supports_required_and_named
|
||||
and not use_mistral_tool_parser
|
||||
):
|
||||
# "required" with standard JSON-based parsing
|
||||
parsed_calls = []
|
||||
with contextlib.suppress(ValidationError):
|
||||
content = content or ""
|
||||
parsed_calls = TypeAdapter(list[FunctionDefinition]).validate_json(
|
||||
content
|
||||
)
|
||||
for tc in parsed_calls:
|
||||
tool_calls.append(
|
||||
FunctionCall(
|
||||
name=tc.name,
|
||||
arguments=json.dumps(tc.parameters, ensure_ascii=False),
|
||||
)
|
||||
)
|
||||
content = None
|
||||
elif is_auto_tool_choice or use_mistral_tool_parser:
|
||||
# Automatic Tool Call Parsing (also used as fallback for
|
||||
# required/named when supports_required_and_named=False)
|
||||
tool_call_info = tool_parser.extract_tool_calls(
|
||||
content if content is not None else "",
|
||||
request=request, # type: ignore
|
||||
)
|
||||
if tool_call_info is not None and tool_call_info.tools_called:
|
||||
tool_calls.extend(
|
||||
FunctionCall(
|
||||
id=tc.id,
|
||||
name=tc.function.name,
|
||||
arguments=tc.function.arguments,
|
||||
)
|
||||
for tc in tool_call_info.tool_calls
|
||||
)
|
||||
content = tool_call_info.content
|
||||
if content and content.strip() == "":
|
||||
content = None
|
||||
else:
|
||||
# No tool calls.
|
||||
return None, content
|
||||
|
||||
return tool_calls, content
|
||||
|
||||
def adjust_request(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
@@ -672,6 +784,20 @@ class DelegatingParser(Parser):
|
||||
last_tc.function.arguments or ""
|
||||
) + self._tool_parser.get_remaining_unstreamed_args()
|
||||
|
||||
def parse(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
enable_auto_tools: bool = False,
|
||||
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
||||
reasoning, content = self.extract_reasoning(model_output, request)
|
||||
tool_calls, content = self._extract_tool_calls(
|
||||
content=content,
|
||||
request=request,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
)
|
||||
return reasoning, content, tool_calls
|
||||
|
||||
def parse_delta(
|
||||
self,
|
||||
delta_text: str,
|
||||
|
||||
Reference in New Issue
Block a user