[Refactor] Unify reasoning + tool-call parsing behind Parser.parse() (#44267)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-06-02 15:11:42 +08:00
committed by GitHub
parent 1edfd09ffd
commit 68dafcca75
5 changed files with 410 additions and 153 deletions
@@ -4,7 +4,6 @@
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.engine.serving import OpenAIServing
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.abstract_parser import DelegatingParser
@@ -32,11 +31,8 @@ class _DummyDelegatingParser(DelegatingParser):
):
return None
def extract_tool_calls(self, model_output: str, request):
return None
def test_parse_tool_calls_from_content_allows_named_tool_choice_with_none_content():
def test_chat_completion_named_tool_choice_with_none_content():
request = ChatCompletionRequest.model_validate(
{
"model": "test-model",
@@ -53,17 +49,15 @@ def test_parse_tool_calls_from_content_allows_named_tool_choice_with_none_conten
"tool_choice": {"type": "function", "function": {"name": "get_weather"}},
}
)
parser = _DummyDelegatingParser(tokenizer=None)
tool_calls, content = OpenAIServing._parse_tool_calls_from_content(
request=request,
tokenizer=None,
enable_auto_tools=True,
tool_parser_cls=None,
tool_calls, content = parser._extract_tool_calls(
content=None,
request=request,
enable_auto_tools=True,
)
assert content is None
assert tool_calls is not None
assert tool_calls == []
+262
View File
@@ -0,0 +1,262 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.parser.abstract_parser import _WrappedParser
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
class ThinkReasoningParser(BaseThinkingReasoningParser):
@property
def start_token(self) -> str:
return "<think>"
@property
def end_token(self) -> str:
return "</think>"
MODEL_OUTPUT = (
"<think>let me think about this</think>"
'<tool_call>\n{"name": "get_weather", '
'"arguments": {"city": "Dallas"}}\n</tool_call>'
)
PLAIN_TEXT = "The weather in Dallas is sunny and 75°F."
TOOL_CALL_ONLY = (
'<tool_call>\n{"name": "get_weather", '
'"arguments": {"city": "Dallas"}}\n</tool_call>'
)
TOOL_ARGUMENTS = '{"city": "Dallas"}'
@pytest.fixture(scope="module")
def tokenizer():
from vllm.tokenizers import get_tokenizer
return get_tokenizer("Qwen/Qwen3-32B")
def make_request(**overrides):
base = {
"model": "test-model",
"messages": [{"role": "user", "content": "hi"}],
}
base.update(overrides)
return ChatCompletionRequest.model_validate(base)
TOOLS = [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {"type": "object", "properties": {}},
},
}
]
def make_parser(tokenizer, reasoning=False, tool=False):
_WrappedParser.reasoning_parser_cls = ThinkReasoningParser if reasoning else None
_WrappedParser.tool_parser_cls = Hermes2ProToolParser if tool else None
return _WrappedParser(tokenizer)
@pytest.mark.parametrize(
"reasoning,tool",
[(False, False), (False, True)],
ids=["neither", "tool-only"],
)
def test_parse_plain_text_no_reasoning_parser(tokenizer, reasoning, tool):
parser = make_parser(tokenizer, reasoning=reasoning, tool=tool)
request = make_request()
r, content, tool_calls = parser.parse(PLAIN_TEXT, request)
assert r is None
assert content == PLAIN_TEXT
assert tool_calls is not None
assert len(tool_calls) == 0
@pytest.mark.parametrize(
"reasoning,tool",
[(True, False), (True, True)],
ids=["reasoning-only", "both"],
)
def test_parse_plain_text_with_reasoning_parser(tokenizer, reasoning, tool):
parser = make_parser(tokenizer, reasoning=reasoning, tool=tool)
request = make_request()
r, content, tool_calls = parser.parse(PLAIN_TEXT, request)
assert r == PLAIN_TEXT
assert content is None
assert tool_calls is not None
assert len(tool_calls) == 0
def test_parse_both_parsers(tokenizer):
parser = make_parser(tokenizer, reasoning=True, tool=True)
request = make_request(tools=TOOLS)
reasoning, content, tool_calls = parser.parse(
MODEL_OUTPUT, request, enable_auto_tools=True
)
assert reasoning is not None
assert "let me think about this" in reasoning
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "get_weather"
assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"}
assert not content or content.strip() == ""
def test_parse_reasoning_only(tokenizer):
parser = make_parser(tokenizer, reasoning=True, tool=False)
request = make_request()
reasoning, content, tool_calls = parser.parse(MODEL_OUTPUT, request)
assert reasoning is not None
assert "let me think about this" in reasoning
assert content is not None
assert "<tool_call>" in content
assert "get_weather" in content
assert tool_calls is not None
assert len(tool_calls) == 0
def test_parse_tool_only(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=True)
request = make_request(tools=TOOLS)
reasoning, content, tool_calls = parser.parse(
MODEL_OUTPUT, request, enable_auto_tools=True
)
assert reasoning is None
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "get_weather"
assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"}
def test_parse_named_tool_choice(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=True)
request = make_request(
tools=TOOLS,
tool_choice={
"type": "function",
"function": {"name": "get_weather"},
},
)
reasoning, content, tool_calls = parser.parse(
TOOL_ARGUMENTS, request, enable_auto_tools=True
)
assert reasoning is None
assert content is None
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "get_weather"
assert tool_calls[0].arguments == TOOL_ARGUMENTS
def test_parse_named_tool_choice_with_reasoning(tokenizer):
parser = make_parser(tokenizer, reasoning=True, tool=True)
model_output = f"<think>thinking</think>{TOOL_ARGUMENTS}"
request = make_request(
tools=TOOLS,
tool_choice={
"type": "function",
"function": {"name": "get_weather"},
},
)
reasoning, content, tool_calls = parser.parse(
model_output, request, enable_auto_tools=True
)
assert reasoning is not None
assert "thinking" in reasoning
assert content is None
assert len(tool_calls) == 1
assert tool_calls[0].name == "get_weather"
assert tool_calls[0].arguments == TOOL_ARGUMENTS
def test_parse_required_tool_choice(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=True)
functions_json = json.dumps(
[
{"name": "get_weather", "parameters": {"city": "Dallas"}},
{"name": "get_time", "parameters": {"timezone": "UTC"}},
]
)
request = make_request(tools=TOOLS, tool_choice="required")
reasoning, content, tool_calls = parser.parse(
functions_json, request, enable_auto_tools=True
)
assert reasoning is None
assert content is None
assert tool_calls is not None
assert len(tool_calls) == 2
assert tool_calls[0].name == "get_weather"
assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"}
assert tool_calls[1].name == "get_time"
assert json.loads(tool_calls[1].arguments) == {"timezone": "UTC"}
def test_parse_named_tool_choice_content_none(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=True)
request = make_request(
tools=TOOLS,
tool_choice={
"type": "function",
"function": {"name": "get_weather"},
},
)
reasoning, content, tool_calls = parser.parse("", request, enable_auto_tools=True)
assert reasoning is None
assert content is None
assert tool_calls is not None
def test_parse_required_tool_choice_content_none(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=True)
request = make_request(tools=TOOLS, tool_choice="required")
reasoning, content, tool_calls = parser.parse("", request, enable_auto_tools=True)
assert reasoning is None
assert content is None
assert tool_calls is not None
assert len(tool_calls) == 0
def test_parse_auto_tools_no_parser(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=False)
request = make_request()
reasoning, content, tool_calls = parser.parse(
TOOL_CALL_ONLY, request, enable_auto_tools=True
)
assert reasoning is None
assert content == TOOL_CALL_ONLY
assert tool_calls is not None
assert len(tool_calls) == 0
def test_parse_auto_tools_no_calls_returns_none(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=True)
request = make_request(tools=TOOLS)
reasoning, content, tool_calls = parser.parse(
PLAIN_TEXT, request, enable_auto_tools=True
)
assert reasoning is None
assert content == PLAIN_TEXT
assert tool_calls is None
@@ -357,6 +357,14 @@ class OpenAIServingChat(OpenAIServing):
assert len(generators) == 1
(result_generator,) = generators
parser: Parser | None = None
if self.parser_cls is not None:
parser = self.parser_cls(
tokenizer,
request.tools,
chat_template_kwargs=chat_template_kwargs,
)
if request.stream:
return self.chat_completion_stream_generator(
request,
@@ -378,7 +386,7 @@ class OpenAIServingChat(OpenAIServing):
conversation,
tokenizer,
request_metadata,
reasoning_parser,
parser,
)
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
@@ -934,7 +942,7 @@ class OpenAIServingChat(OpenAIServing):
conversation: list[ConversationMessage],
tokenizer: TokenizerLike,
request_metadata: RequestResponseMetadata,
reasoning_parser: ReasoningParser | None = None,
parser: Parser | None = None,
) -> ErrorResponse | ChatCompletionResponse:
created_time = int(time.time())
final_res: RequestOutput | None = None
@@ -1043,28 +1051,20 @@ class OpenAIServingChat(OpenAIServing):
choices.append(choice_data)
continue
if reasoning_parser:
# If the reasoning parser is enabled,
# tool calls are extracted exclusively from the content.
reasoning, content = reasoning_parser.extract_reasoning(
output.text, request=request
if parser is not None:
reasoning, content, tool_calls = parser.parse(
output.text,
request,
enable_auto_tools=self.enable_auto_tools,
)
if not request.include_reasoning:
reasoning = None
else:
reasoning = None
content = output.text
tool_calls = []
auto_tools_called = False
# if auto tools are not enabled, and a named tool choice using
# outlines is not being used
tool_calls, content = self._parse_tool_calls_from_content(
request=request,
tokenizer=tokenizer,
content=content,
enable_auto_tools=self.enable_auto_tools,
tool_parser_cls=self.tool_parser,
)
if is_mistral_tokenizer(tokenizer):
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
+1 -126
View File
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import json
import time
from collections.abc import Awaitable, Mapping
@@ -9,8 +8,7 @@ from http import HTTPStatus
from typing import Any, ClassVar, Generic, Protocol, TypeAlias, TypeVar
from fastapi import Request
from openai.types.responses import ToolChoiceFunction
from pydantic import ConfigDict, TypeAdapter, ValidationError
from pydantic import ConfigDict
from starlette.datastructures import Headers
import vllm.envs as envs
@@ -21,7 +19,6 @@ from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
ChatCompletionResponse,
)
@@ -31,8 +28,6 @@ from vllm.entrypoints.openai.completion.protocol import (
)
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
FunctionCall,
FunctionDefinition,
GenerationError,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
@@ -61,14 +56,12 @@ from vllm.renderers.inputs.preprocess import (
)
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers import ToolParser
from vllm.tracing import (
contains_trace_headers,
extract_trace_headers,
log_tracing_disabled_warning,
)
from vllm.utils import random_uuid
from vllm.utils.mistral import is_mistral_tool_parser
logger = init_logger(__name__)
@@ -451,124 +444,6 @@ class OpenAIServing(BeamSearchOnlineMixin):
exc_info=True,
)
@staticmethod
def _parse_tool_calls_from_content(
request: ResponsesRequest | ChatCompletionRequest,
tokenizer: TokenizerLike | None,
enable_auto_tools: bool,
tool_parser_cls: type[ToolParser] | None,
content: str | None = None,
) -> tuple[list[FunctionCall] | None, str | None]:
# When the Mistral grammar factory injected structured outputs,
# let the parser handle the output.
use_mistral_tool_parser = (
isinstance(request, ChatCompletionRequest)
and is_mistral_tool_parser(tool_parser_cls)
and request._grammar_from_tool_parser
)
function_calls = list[FunctionCall]()
if (
not use_mistral_tool_parser
and request.tool_choice
and isinstance(request.tool_choice, ToolChoiceFunction)
):
# Forced Function Call (Responses API)
if content is None:
return [], None
function_calls.append(
FunctionCall(name=request.tool_choice.name, arguments=content)
)
content = None # Clear content since tool is called.
elif (
not use_mistral_tool_parser
and request.tool_choice
and isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
and (tool_parser_cls is None or tool_parser_cls.supports_required_and_named)
):
# Named function with standard JSON-based parsing
if content is None:
return [], None
function_calls.append(
FunctionCall(name=request.tool_choice.function.name, arguments=content)
)
content = None # Clear content since tool is called.
elif (
not use_mistral_tool_parser
and request.tool_choice == "required"
and (tool_parser_cls is None or tool_parser_cls.supports_required_and_named)
):
# "required" with standard JSON-based parsing
tool_calls = []
with contextlib.suppress(ValidationError):
content = content or ""
tool_calls = TypeAdapter(list[FunctionDefinition]).validate_json(
content
)
for tool_call in tool_calls:
function_calls.append(
FunctionCall(
name=tool_call.name,
arguments=json.dumps(tool_call.parameters, ensure_ascii=False),
)
)
content = None # Clear content since tool is called.
elif tool_parser_cls and (
use_mistral_tool_parser
or (
enable_auto_tools
and (
request.tool_choice == "auto"
or request.tool_choice is None
or (
not tool_parser_cls.supports_required_and_named
and request.tools
and (
request.tool_choice == "required"
or isinstance(
request.tool_choice,
ChatCompletionNamedToolChoiceParam,
)
)
)
)
)
):
# Automatic Tool Call Parsing (also used as fallback for
# required/named when supports_required_and_named=False)
if tokenizer is None:
raise ValueError(
"Tokenizer not available when `skip_tokenizer_init=True`"
)
try:
tool_parser = tool_parser_cls(tokenizer, request.tools)
except RuntimeError as e:
logger.exception("Error in tool parser creation.")
raise e
tool_call_info = tool_parser.extract_tool_calls(
content if content is not None else "",
request=request, # type: ignore
)
if tool_call_info is not None and tool_call_info.tools_called:
# extract_tool_calls() returns a list of tool calls.
function_calls.extend(
FunctionCall(
id=tool_call.id,
name=tool_call.function.name,
arguments=tool_call.function.arguments,
)
for tool_call in tool_call_info.tool_calls
)
content = tool_call_info.content
if content and content.strip() == "":
content = None
else:
# No tool calls.
return None, content
return function_calls, content
@staticmethod
def _get_decoded_token(
logprob: Logprob,
+126
View File
@@ -44,6 +44,7 @@ from vllm.tool_parsers.streaming import (
)
from vllm.tool_parsers.utils import Tool
from vllm.utils import random_uuid
from vllm.utils.mistral import is_mistral_tool_parser
logger = init_logger(__name__)
@@ -313,6 +314,24 @@ class Parser:
A DeltaMessage with tool_calls field, or None.
"""
@abstractmethod
def parse(
self,
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
enable_auto_tools: bool = False,
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
"""Parse a complete model output, extracting reasoning and tool calls.
Args:
model_output: The complete model-generated string.
request: The request object used to generate the output.
enable_auto_tools: Whether to enable automatic tool call parsing.
Returns:
A tuple of (reasoning, content, tool_calls).
"""
@abstractmethod
def parse_delta(
self,
@@ -511,6 +530,99 @@ class DelegatingParser(Parser):
# No tool calls
return [], content
def _extract_tool_calls(
self,
content: str | None,
request: ChatCompletionRequest | ResponsesRequest,
enable_auto_tools: bool = False,
) -> tuple[list[FunctionCall] | None, str | None]:
tool_parser = self._tool_parser
if tool_parser is None:
return [], content
# When the Mistral grammar factory injected structured outputs,
# let the parser handle the output.
use_mistral_tool_parser = (
is_mistral_tool_parser(type(tool_parser))
and isinstance(request, ChatCompletionRequest)
and request._grammar_from_tool_parser
)
supports_required_and_named = tool_parser.supports_required_and_named
is_named_tool_choice = request.tool_choice and isinstance(
request.tool_choice,
(ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
)
is_required_tool_choice = request.tool_choice == "required"
is_auto_tool_choice = enable_auto_tools and (
request.tool_choice == "auto"
or request.tool_choice is None
or (
not supports_required_and_named
and (is_named_tool_choice or is_required_tool_choice)
)
)
tool_calls = list[FunctionCall]()
if (
is_named_tool_choice
and supports_required_and_named
and not use_mistral_tool_parser
):
if content is None:
return [], None
tool_calls.append(
FunctionCall(
name=self._get_function_name(request),
arguments=content,
)
)
content = None
elif (
is_required_tool_choice
and supports_required_and_named
and not use_mistral_tool_parser
):
# "required" with standard JSON-based parsing
parsed_calls = []
with contextlib.suppress(ValidationError):
content = content or ""
parsed_calls = TypeAdapter(list[FunctionDefinition]).validate_json(
content
)
for tc in parsed_calls:
tool_calls.append(
FunctionCall(
name=tc.name,
arguments=json.dumps(tc.parameters, ensure_ascii=False),
)
)
content = None
elif is_auto_tool_choice or use_mistral_tool_parser:
# Automatic Tool Call Parsing (also used as fallback for
# required/named when supports_required_and_named=False)
tool_call_info = tool_parser.extract_tool_calls(
content if content is not None else "",
request=request, # type: ignore
)
if tool_call_info is not None and tool_call_info.tools_called:
tool_calls.extend(
FunctionCall(
id=tc.id,
name=tc.function.name,
arguments=tc.function.arguments,
)
for tc in tool_call_info.tool_calls
)
content = tool_call_info.content
if content and content.strip() == "":
content = None
else:
# No tool calls.
return None, content
return tool_calls, content
def adjust_request(
self, request: ChatCompletionRequest | ResponsesRequest
) -> ChatCompletionRequest | ResponsesRequest:
@@ -672,6 +784,20 @@ class DelegatingParser(Parser):
last_tc.function.arguments or ""
) + self._tool_parser.get_remaining_unstreamed_args()
def parse(
self,
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
enable_auto_tools: bool = False,
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
reasoning, content = self.extract_reasoning(model_output, request)
tool_calls, content = self._extract_tool_calls(
content=content,
request=request,
enable_auto_tools=enable_auto_tools,
)
return reasoning, content, tool_calls
def parse_delta(
self,
delta_text: str,