iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Bugfix][Frontend] Enforce parallel_tool_calls=false in the required-tool grammar (#50502)

Browse Source 
Signed-off-by: ostring <ostring@qq.com>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Ostring24
2026-09-29 21:33:28 +00:00
committed by GitHub
co-authored by Flora Feng
parent 1ee7f78e80
commit 4e7af3b0de
8 changed files with 261 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/tool_parsers/test_functiongemma_tool_parser.py
+1
View File
@@ -27,6 +27,7 @@ def mock_request():
request = MagicMock(spec=ChatCompletionRequest)
request.tools = []
request.tool_choice = "auto"
request.parallel_tool_calls = True
return request
tests/tool_parsers/test_structural_tag_registry.py
+139 -13
View File
@@ -1317,27 +1317,153 @@ def test_mimo_strict_compact_xml(policy):
)
@pytest.mark.parametrize("policy", ["auto", "required"])
def test_mimo_request_limits_parallel_calls(sample_tools_strict, policy):
_DS_CALLS_BEGIN = "<\uff5ctool\u2581calls\u2581begin\uff5c>"
_DS_CALL_BEGIN = "<\uff5ctool\u2581call\u2581begin\uff5c>"
_DS_SEP = "<\uff5ctool\u2581sep\uff5c>"
_DS_CALL_END = "<\uff5ctool\u2581call\u2581end\uff5c>"
_DS_CALLS_END = "<\uff5ctool\u2581calls\u2581end\uff5c>"
# One entry per tool-call list shape: top-level triggered tags (mimo, qwen3),
# tags with a separator (hermes required), and a calls block wrapping the list
# (deepseek). Each is (tool_parser, list prefix, one call, list suffix).
_SINGLE_CALL_CASES = [
(
"mimo",
"",
"<tool_call><function=get_weather><parameter=city>Paris</parameter>"
"</function></tool_call>",
"",
),
(
"hermes",
"",
'<tool_call>\n{"name": "get_weather", "arguments": {"city": "Paris"}}'
"\n</tool_call>",
"",
),
(
"qwen3_xml",
"",
"<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n"
"</parameter>\n</function>\n</tool_call>",
"",
),
(
"deepseek_v31",
_DS_CALLS_BEGIN,
f'{_DS_CALL_BEGIN}get_weather{_DS_SEP}{{"city": "Paris"}}{_DS_CALL_END}',
_DS_CALLS_END,
),
]
_SINGLE_CALL_IDS = [case[0] for case in _SINGLE_CALL_CASES]
def _tool_calling_grammar(
tool_parser: str,
tools: list[ChatCompletionToolsParam],
request: ChatCompletionRequest | ResponsesRequest,
) -> Grammar | None:
from vllm.parser.parser_manager import ParserManager
parser_cls = ParserManager.get_parser(
tool_parser_name=tool_parser, enable_auto_tools=True
)
adjusted = parser_cls(MagicMock(), tools=tools).adjust_request(request)
structured_outputs = adjusted.structured_outputs
if structured_outputs is None or structured_outputs.structural_tag is None:
return None
return Grammar.from_structural_tag(structured_outputs.structural_tag)
@pytest.mark.parametrize("policy", ["auto", "required"])
@pytest.mark.parametrize(
"tool_parser,prefix,call,suffix", _SINGLE_CALL_CASES, ids=_SINGLE_CALL_IDS
)
def test_parallel_tool_calls_false_limits_grammar_to_one_call(
sample_tools, tool_parser, prefix, call, suffix, policy
):
# Tools are not strict, so auto gets a grammar only because of the limit.
request = ChatCompletionRequest(
model="mimo",
model="m",
messages=[],
tools=[t.model_dump() for t in sample_tools_strict],
tools=[t.model_dump() for t in sample_tools],
tool_choice=policy,
parallel_tool_calls=False,
)
parser_cls = ParserManager.get_parser(
tool_parser_name="mimo", enable_auto_tools=True
grammar = _tool_calling_grammar(tool_parser, sample_tools, request)
assert grammar is not None
assert _is_grammar_accept_string(grammar, prefix + call + suffix)
assert not _is_grammar_accept_string(grammar, prefix + call * 2 + suffix)
assert not _is_grammar_accept_string(grammar, prefix + call + suffix + "text")
@pytest.mark.parametrize(
"tool_parser,prefix,call,suffix", _SINGLE_CALL_CASES, ids=_SINGLE_CALL_IDS
)
def test_parallel_tool_calls_default_keeps_grammar_unlimited(
sample_tools, tool_parser, prefix, call, suffix
):
def request(policy: str) -> ChatCompletionRequest:
return ChatCompletionRequest(
model="m",
messages=[],
tools=[t.model_dump() for t in sample_tools],
tool_choice=policy,
)
assert _tool_calling_grammar(tool_parser, sample_tools, request("auto")) is None
grammar = _tool_calling_grammar(tool_parser, sample_tools, request("required"))
assert grammar is not None
assert _is_grammar_accept_string(grammar, prefix + call * 2 + suffix)
def test_parallel_tool_calls_false_does_not_enable_calls_with_response_format(
sample_tools,
):
# With the flag unset, auto + response_format + non-strict tools is
# format-only. Setting the flag to false must not make tool calls possible.
request = ChatCompletionRequest(
model="m",
messages=[],
tools=[t.model_dump() for t in sample_tools],
tool_choice="auto",
parallel_tool_calls=False,
response_format={
"type": "json_schema",
"json_schema": {"name": "answer", "schema": {"type": "object"}},
},
)
parser = parser_cls(MagicMock(), tools=sample_tools_strict)
adjusted = parser.adjust_request(request)
grammar = Grammar.from_structural_tag(adjusted.structured_outputs.structural_tag)
call = (
"<tool_call><function=get_weather><parameter=city>北京</parameter>"
"</function></tool_call>"
assert _tool_calling_grammar("hermes", sample_tools, request) is None
def test_responses_parallel_tool_calls_false_limits_grammar_to_one_call():
# The Responses API has no post-hoc filter, so the grammar is the only limit.
request = ResponsesRequest.model_validate(
{
"input": "hi",
"tools": [
{
"type": "function",
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
}
],
"tool_choice": "required",
"parallel_tool_calls": False,
}
)
_, _, call, _ = _SINGLE_CALL_CASES[1]
grammar = _tool_calling_grammar("hermes", request.tools, request)
assert grammar is not None
assert _is_grammar_accept_string(grammar, call)
assert not _is_grammar_accept_string(grammar, call * 2)
assert not _is_grammar_accept_string(grammar, call + "extra text")
tests/tool_use/test_tool_choice_required.py
+65 -2
View File
@@ -9,6 +9,8 @@ from openai.types.responses import FunctionTool, WebSearchTool
from pydantic import TypeAdapter
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedFunction,
ChatCompletionNamedToolChoiceParam,
ChatCompletionToolsParam,
)
from vllm.tool_parsers.streaming import extract_required_tool_call_streaming
@@ -68,11 +70,18 @@ EXAMPLE_TOOLS = [
def _compile_and_check(
tools: list[ChatCompletionToolsParam], sample_output, should_match: bool
tools: list[ChatCompletionToolsParam],
sample_output,
should_match: bool,
parallel_tool_calls: bool | None = None,
):
# self = MagicMock(tool_choice="required", tools=tools)
# schema = ChatCompletionRequest._get_json_schema_from_tool(self)
schema = get_json_schema_from_tools(tools=tools, tool_choice="required")
schema = get_json_schema_from_tools(
tools=tools,
tool_choice="required",
parallel_tool_calls=parallel_tool_calls,
)
assert isinstance(schema, dict)
# use build_regex_from_schema used in JSONLogitsProcessor to create Guide
@@ -393,3 +402,57 @@ class TestNonFunctionToolsSkipped:
any_of = schema["items"]["anyOf"]
assert len(any_of) == 1
assert any_of[0]["properties"]["name"]["enum"] == ["get_weather"]
class TestParallelToolCallsConstraint:
"""`parallel_tool_calls=false` must be enforced by the decoding grammar.
Without a `maxItems` bound the model is free to emit an unbounded run of
tool calls that are only discarded afterwards, wasting the token budget and
risking truncation of the one call the client actually receives."""
TOOLS = TypeAdapter(list[ChatCompletionToolsParam]).validate_python(EXAMPLE_TOOLS)
ONE_CALL = [{"name": "get_current_weather", "parameters": {"city": "Vienna"}}]
TWO_CALLS = [
{"name": "get_current_weather", "parameters": {"city": "Vienna"}},
{"name": "get_current_weather", "parameters": {"city": "Berlin"}},
]
def test_disabled_rejects_a_second_tool_call(self):
_compile_and_check(self.TOOLS, self.ONE_CALL, True, parallel_tool_calls=False)
_compile_and_check(self.TOOLS, self.TWO_CALLS, False, parallel_tool_calls=False)
@pytest.mark.parametrize("parallel_tool_calls", [True, None])
def test_enabled_or_unset_still_allows_multiple(self, parallel_tool_calls):
_compile_and_check(
self.TOOLS,
self.TWO_CALLS,
True,
parallel_tool_calls=parallel_tool_calls,
)
@pytest.mark.parametrize(
"parallel_tool_calls,expected",
[(False, 1), (True, None), (None, None)],
)
def test_max_items_bound(self, parallel_tool_calls, expected):
schema = get_json_schema_from_tools(
tools=self.TOOLS,
tool_choice="required",
parallel_tool_calls=parallel_tool_calls,
)
assert isinstance(schema, dict)
assert schema["minItems"] == 1
assert schema.get("maxItems") == expected
def test_forced_named_tool_is_unaffected(self):
# Named tool choice yields a bare parameters object, never an array.
schema = get_json_schema_from_tools(
tools=self.TOOLS,
tool_choice=ChatCompletionNamedToolChoiceParam(
function=ChatCompletionNamedFunction(name="get_current_weather")
),
parallel_tool_calls=False,
)
assert isinstance(schema, dict)
assert "maxItems" not in schema
vllm/parser/abstract_parser.py
+15 -3
View File
@@ -49,7 +49,10 @@ from vllm.tool_parsers.streaming import (
extract_named_tool_call_streaming,
extract_required_tool_call_streaming,
)
from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness
from vllm.tool_parsers.structural_tag_registry import (
limit_to_single_tool_call,
resolve_tool_strictness,
)
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
logger = init_logger(__name__)
@@ -496,13 +499,20 @@ class DelegatingParser(Parser):
structured_outputs = request.extract_structured_outputs()
is_auto = request.tool_choice == "auto"
single_call = request.parallel_tool_calls is False
strict_level = self.tool_strict_level
if single_call and not (is_auto and structured_outputs is not None):
# Limiting the call count needs the call envelope in the grammar.
# Auto with structured outputs keeps its format-only grammar, so the
# flag never makes calls possible that are impossible without it.
strict_level = max(strict_level, ToolStrictLevel.FUNCTION)
resolved_tools = None
if tool_parser.structural_tag_model is not None:
resolved_tools = resolve_tool_strictness(
request.tools,
request.tool_choice,
self.tool_strict_level,
strict_level,
)
output_format = None
@@ -529,7 +539,7 @@ class DelegatingParser(Parser):
tools_structural_tag = tool_parser.get_structural_tag(
tag_request,
reasoning=False,
strict_level=self.tool_strict_level,
strict_level=strict_level,
)
else:
tools_structural_tag = None
@@ -544,6 +554,8 @@ class DelegatingParser(Parser):
)
return request
if single_call:
limit_to_single_tool_call(tools_structural_tag)
structural_tag = tools_structural_tag
if output_format is not None:
structural_tag = StructuralTag(
vllm/tool_parsers/abstract_tool_parser.py
+3 -1
View File
@@ -141,7 +141,9 @@ class ToolParser:
return request
json_schema_from_tool = get_json_schema_from_tools(
tool_choice=request.tool_choice, tools=request.tools
tool_choice=request.tool_choice,
tools=request.tools,
parallel_tool_calls=request.parallel_tool_calls,
)
# Set structured output params for tool calling
if json_schema_from_tool is not None:
vllm/tool_parsers/mimo_tool_parser.py
-17
View File
@@ -1,25 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from xgrammar.structural_tag import TriggeredTagsFormat
from vllm.parser.engine.registered_adapters import MiMoParserToolAdapter
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
class MiMoToolParser(MiMoParserToolAdapter): # type: ignore[valid-type, misc]
structural_tag_model = "mimo"
def get_structural_tag(
self, request, *, reasoning=False, strict_level=ToolStrictLevel.AUTO
):
tag = super().get_structural_tag(
request, reasoning=reasoning, strict_level=strict_level
)
if (
tag is not None
and request.parallel_tool_calls is False
and isinstance(tag.format, TriggeredTagsFormat)
):
tag.format.stop_after_first = True
return tag
vllm/tool_parsers/structural_tag_registry.py
+30
View File
@@ -9,6 +9,7 @@ from openai.types.responses.response import ToolChoice as ResponsesToolChoice
from openai.types.responses.tool import Tool as ResponsesTool
from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed
from openai.types.responses.tool_choice_function import ToolChoiceFunction
from pydantic import BaseModel
from xgrammar import StructuralTag, normalize_tool_choice
from xgrammar import get_model_structural_tag as get_xgrammar_model_structural_tag
from xgrammar.openai_tool_call_schema import (
@@ -27,6 +28,7 @@ from xgrammar.structural_tag import (
StarFormat,
TagFormat,
TagsWithSeparatorFormat,
TokenTriggeredTagsFormat,
TriggeredTagsFormat,
)
@@ -197,6 +199,34 @@ def get_model_structural_tag(
)
_TOOL_CALL_LIST_FORMATS = (
TriggeredTagsFormat,
TokenTriggeredTagsFormat,
TagsWithSeparatorFormat,
)
def limit_to_single_tool_call(tag: StructuralTag) -> StructuralTag:
"""Stop every tool-call list in ``tag`` after its first call, in place.
Enforces ``parallel_tool_calls=false`` in the grammar. In every registered
tag, ``_TOOL_CALL_LIST_FORMATS`` repeat tool calls, never arguments.
"""
def visit(node: Any) -> None:
if isinstance(node, BaseModel):
if isinstance(node, _TOOL_CALL_LIST_FORMATS):
node.stop_after_first = True
for name in type(node).model_fields:
visit(getattr(node, name))
elif isinstance(node, (list, tuple)):
for item in node:
visit(item)
visit(tag.format)
return tag
def _dump_tool_for_xgrammar(
tool: ChatCompletionToolsParam | ResponsesTool,
) -> dict[str, Any]:
vllm/tool_parsers/utils.py
+8 -2
View File
@@ -347,6 +347,7 @@ def _get_tool_schema_defs(
def _get_json_schema_from_tools(
tools: list[Tool],
parallel_tool_calls: bool | None = None,
) -> dict:
fn_tool_schemas: list[dict[str, Any]] = []
fn_tools: list[Tool] = []
@@ -361,7 +362,7 @@ def _get_json_schema_from_tools(
elif _is_function_tool(tool):
fn_tool_schemas.append(_get_tool_schema_from_tool(tool))
fn_tools.append(tool)
json_schema = {
json_schema: dict[str, Any] = {
"type": "array",
"minItems": 1,
"items": {
@@ -369,6 +370,10 @@ def _get_json_schema_from_tools(
"anyOf": fn_tool_schemas,
},
}
# Only an explicit `false` opts out of parallel tool calls; `None` means the
# field was never set and the default (true) applies.
if parallel_tool_calls is False:
json_schema["maxItems"] = 1
json_schema_defs = _get_tool_schema_defs(fn_tools)
if json_schema_defs:
json_schema["$defs"] = json_schema_defs
@@ -378,6 +383,7 @@ def _get_json_schema_from_tools(
def get_json_schema_from_tools(
tool_choice: str | ToolChoiceFunction | ChatCompletionNamedToolChoiceParam,
tools: list[Tool] | None,
parallel_tool_calls: bool | None = None,
) -> str | dict | None:
# tool_choice: "none"
if tool_choice in ("none", None) or tools is None:
@@ -413,7 +419,7 @@ def get_json_schema_from_tools(
return chat_tool_map[tool_name].function.parameters
# tool_choice: "required"
if tool_choice == "required":
return _get_json_schema_from_tools(tools)
return _get_json_schema_from_tools(tools, parallel_tool_calls)
# tool_choice: "auto"
return None