From 4e7af3b0de4cae4f70056da590f2f3d676223978 Mon Sep 17 00:00:00 2001 From: Ostring24 <163995157+Ostring24@users.noreply.github.com> Date: Wed, 30 Sep 2026 05:33:28 +0800 Subject: [PATCH] [Bugfix][Frontend] Enforce parallel_tool_calls=false in the required-tool grammar (#50502) Signed-off-by: ostring Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> --- .../test_functiongemma_tool_parser.py | 1 + .../test_structural_tag_registry.py | 152 ++++++++++++++++-- tests/tool_use/test_tool_choice_required.py | 67 +++++++- vllm/parser/abstract_parser.py | 18 ++- vllm/tool_parsers/abstract_tool_parser.py | 4 +- vllm/tool_parsers/mimo_tool_parser.py | 17 -- vllm/tool_parsers/structural_tag_registry.py | 30 ++++ vllm/tool_parsers/utils.py | 10 +- 8 files changed, 261 insertions(+), 38 deletions(-) diff --git a/tests/tool_parsers/test_functiongemma_tool_parser.py b/tests/tool_parsers/test_functiongemma_tool_parser.py index d32aba3085..10792c1660 100644 --- a/tests/tool_parsers/test_functiongemma_tool_parser.py +++ b/tests/tool_parsers/test_functiongemma_tool_parser.py @@ -27,6 +27,7 @@ def mock_request(): request = MagicMock(spec=ChatCompletionRequest) request.tools = [] request.tool_choice = "auto" + request.parallel_tool_calls = True return request diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 3f22da17cd..2c4de60a9d 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -1317,27 +1317,153 @@ def test_mimo_strict_compact_xml(policy): ) -@pytest.mark.parametrize("policy", ["auto", "required"]) -def test_mimo_request_limits_parallel_calls(sample_tools_strict, policy): +_DS_CALLS_BEGIN = "<\uff5ctool\u2581calls\u2581begin\uff5c>" +_DS_CALL_BEGIN = "<\uff5ctool\u2581call\u2581begin\uff5c>" +_DS_SEP = "<\uff5ctool\u2581sep\uff5c>" +_DS_CALL_END = "<\uff5ctool\u2581call\u2581end\uff5c>" +_DS_CALLS_END = "<\uff5ctool\u2581calls\u2581end\uff5c>" + +# One entry per tool-call list shape: top-level triggered tags (mimo, qwen3), +# tags with a separator (hermes required), and a calls block wrapping the list +# (deepseek). Each is (tool_parser, list prefix, one call, list suffix). +_SINGLE_CALL_CASES = [ + ( + "mimo", + "", + "Paris" + "", + "", + ), + ( + "hermes", + "", + '\n{"name": "get_weather", "arguments": {"city": "Paris"}}' + "\n", + "", + ), + ( + "qwen3_xml", + "", + "\n\n\nParis\n" + "\n\n", + "", + ), + ( + "deepseek_v31", + _DS_CALLS_BEGIN, + f'{_DS_CALL_BEGIN}get_weather{_DS_SEP}{{"city": "Paris"}}{_DS_CALL_END}', + _DS_CALLS_END, + ), +] + +_SINGLE_CALL_IDS = [case[0] for case in _SINGLE_CALL_CASES] + + +def _tool_calling_grammar( + tool_parser: str, + tools: list[ChatCompletionToolsParam], + request: ChatCompletionRequest | ResponsesRequest, +) -> Grammar | None: from vllm.parser.parser_manager import ParserManager + parser_cls = ParserManager.get_parser( + tool_parser_name=tool_parser, enable_auto_tools=True + ) + adjusted = parser_cls(MagicMock(), tools=tools).adjust_request(request) + structured_outputs = adjusted.structured_outputs + if structured_outputs is None or structured_outputs.structural_tag is None: + return None + return Grammar.from_structural_tag(structured_outputs.structural_tag) + + +@pytest.mark.parametrize("policy", ["auto", "required"]) +@pytest.mark.parametrize( + "tool_parser,prefix,call,suffix", _SINGLE_CALL_CASES, ids=_SINGLE_CALL_IDS +) +def test_parallel_tool_calls_false_limits_grammar_to_one_call( + sample_tools, tool_parser, prefix, call, suffix, policy +): + # Tools are not strict, so auto gets a grammar only because of the limit. request = ChatCompletionRequest( - model="mimo", + model="m", messages=[], - tools=[t.model_dump() for t in sample_tools_strict], + tools=[t.model_dump() for t in sample_tools], tool_choice=policy, parallel_tool_calls=False, ) - parser_cls = ParserManager.get_parser( - tool_parser_name="mimo", enable_auto_tools=True + + grammar = _tool_calling_grammar(tool_parser, sample_tools, request) + + assert grammar is not None + assert _is_grammar_accept_string(grammar, prefix + call + suffix) + assert not _is_grammar_accept_string(grammar, prefix + call * 2 + suffix) + assert not _is_grammar_accept_string(grammar, prefix + call + suffix + "text") + + +@pytest.mark.parametrize( + "tool_parser,prefix,call,suffix", _SINGLE_CALL_CASES, ids=_SINGLE_CALL_IDS +) +def test_parallel_tool_calls_default_keeps_grammar_unlimited( + sample_tools, tool_parser, prefix, call, suffix +): + def request(policy: str) -> ChatCompletionRequest: + return ChatCompletionRequest( + model="m", + messages=[], + tools=[t.model_dump() for t in sample_tools], + tool_choice=policy, + ) + + assert _tool_calling_grammar(tool_parser, sample_tools, request("auto")) is None + grammar = _tool_calling_grammar(tool_parser, sample_tools, request("required")) + assert grammar is not None + assert _is_grammar_accept_string(grammar, prefix + call * 2 + suffix) + + +def test_parallel_tool_calls_false_does_not_enable_calls_with_response_format( + sample_tools, +): + # With the flag unset, auto + response_format + non-strict tools is + # format-only. Setting the flag to false must not make tool calls possible. + request = ChatCompletionRequest( + model="m", + messages=[], + tools=[t.model_dump() for t in sample_tools], + tool_choice="auto", + parallel_tool_calls=False, + response_format={ + "type": "json_schema", + "json_schema": {"name": "answer", "schema": {"type": "object"}}, + }, ) - parser = parser_cls(MagicMock(), tools=sample_tools_strict) - adjusted = parser.adjust_request(request) - grammar = Grammar.from_structural_tag(adjusted.structured_outputs.structural_tag) - call = ( - "北京" - "" + + assert _tool_calling_grammar("hermes", sample_tools, request) is None + + +def test_responses_parallel_tool_calls_false_limits_grammar_to_one_call(): + # The Responses API has no post-hoc filter, so the grammar is the only limit. + request = ResponsesRequest.model_validate( + { + "input": "hi", + "tools": [ + { + "type": "function", + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + ], + "tool_choice": "required", + "parallel_tool_calls": False, + } ) + _, _, call, _ = _SINGLE_CALL_CASES[1] + + grammar = _tool_calling_grammar("hermes", request.tools, request) + + assert grammar is not None assert _is_grammar_accept_string(grammar, call) assert not _is_grammar_accept_string(grammar, call * 2) - assert not _is_grammar_accept_string(grammar, call + "extra text") diff --git a/tests/tool_use/test_tool_choice_required.py b/tests/tool_use/test_tool_choice_required.py index f2b98eb17b..aaa499da16 100644 --- a/tests/tool_use/test_tool_choice_required.py +++ b/tests/tool_use/test_tool_choice_required.py @@ -9,6 +9,8 @@ from openai.types.responses import FunctionTool, WebSearchTool from pydantic import TypeAdapter from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionNamedFunction, + ChatCompletionNamedToolChoiceParam, ChatCompletionToolsParam, ) from vllm.tool_parsers.streaming import extract_required_tool_call_streaming @@ -68,11 +70,18 @@ EXAMPLE_TOOLS = [ def _compile_and_check( - tools: list[ChatCompletionToolsParam], sample_output, should_match: bool + tools: list[ChatCompletionToolsParam], + sample_output, + should_match: bool, + parallel_tool_calls: bool | None = None, ): # self = MagicMock(tool_choice="required", tools=tools) # schema = ChatCompletionRequest._get_json_schema_from_tool(self) - schema = get_json_schema_from_tools(tools=tools, tool_choice="required") + schema = get_json_schema_from_tools( + tools=tools, + tool_choice="required", + parallel_tool_calls=parallel_tool_calls, + ) assert isinstance(schema, dict) # use build_regex_from_schema used in JSONLogitsProcessor to create Guide @@ -393,3 +402,57 @@ class TestNonFunctionToolsSkipped: any_of = schema["items"]["anyOf"] assert len(any_of) == 1 assert any_of[0]["properties"]["name"]["enum"] == ["get_weather"] + + +class TestParallelToolCallsConstraint: + """`parallel_tool_calls=false` must be enforced by the decoding grammar. + + Without a `maxItems` bound the model is free to emit an unbounded run of + tool calls that are only discarded afterwards, wasting the token budget and + risking truncation of the one call the client actually receives.""" + + TOOLS = TypeAdapter(list[ChatCompletionToolsParam]).validate_python(EXAMPLE_TOOLS) + ONE_CALL = [{"name": "get_current_weather", "parameters": {"city": "Vienna"}}] + TWO_CALLS = [ + {"name": "get_current_weather", "parameters": {"city": "Vienna"}}, + {"name": "get_current_weather", "parameters": {"city": "Berlin"}}, + ] + + def test_disabled_rejects_a_second_tool_call(self): + _compile_and_check(self.TOOLS, self.ONE_CALL, True, parallel_tool_calls=False) + _compile_and_check(self.TOOLS, self.TWO_CALLS, False, parallel_tool_calls=False) + + @pytest.mark.parametrize("parallel_tool_calls", [True, None]) + def test_enabled_or_unset_still_allows_multiple(self, parallel_tool_calls): + _compile_and_check( + self.TOOLS, + self.TWO_CALLS, + True, + parallel_tool_calls=parallel_tool_calls, + ) + + @pytest.mark.parametrize( + "parallel_tool_calls,expected", + [(False, 1), (True, None), (None, None)], + ) + def test_max_items_bound(self, parallel_tool_calls, expected): + schema = get_json_schema_from_tools( + tools=self.TOOLS, + tool_choice="required", + parallel_tool_calls=parallel_tool_calls, + ) + assert isinstance(schema, dict) + assert schema["minItems"] == 1 + assert schema.get("maxItems") == expected + + def test_forced_named_tool_is_unaffected(self): + # Named tool choice yields a bare parameters object, never an array. + schema = get_json_schema_from_tools( + tools=self.TOOLS, + tool_choice=ChatCompletionNamedToolChoiceParam( + function=ChatCompletionNamedFunction(name="get_current_weather") + ), + parallel_tool_calls=False, + ) + assert isinstance(schema, dict) + assert "maxItems" not in schema diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index b15d86fd07..2a348697c7 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -49,7 +49,10 @@ from vllm.tool_parsers.streaming import ( extract_named_tool_call_streaming, extract_required_tool_call_streaming, ) -from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness +from vllm.tool_parsers.structural_tag_registry import ( + limit_to_single_tool_call, + resolve_tool_strictness, +) from vllm.tool_parsers.tool_strict_level import ToolStrictLevel logger = init_logger(__name__) @@ -496,13 +499,20 @@ class DelegatingParser(Parser): structured_outputs = request.extract_structured_outputs() is_auto = request.tool_choice == "auto" + single_call = request.parallel_tool_calls is False + strict_level = self.tool_strict_level + if single_call and not (is_auto and structured_outputs is not None): + # Limiting the call count needs the call envelope in the grammar. + # Auto with structured outputs keeps its format-only grammar, so the + # flag never makes calls possible that are impossible without it. + strict_level = max(strict_level, ToolStrictLevel.FUNCTION) resolved_tools = None if tool_parser.structural_tag_model is not None: resolved_tools = resolve_tool_strictness( request.tools, request.tool_choice, - self.tool_strict_level, + strict_level, ) output_format = None @@ -529,7 +539,7 @@ class DelegatingParser(Parser): tools_structural_tag = tool_parser.get_structural_tag( tag_request, reasoning=False, - strict_level=self.tool_strict_level, + strict_level=strict_level, ) else: tools_structural_tag = None @@ -544,6 +554,8 @@ class DelegatingParser(Parser): ) return request + if single_call: + limit_to_single_tool_call(tools_structural_tag) structural_tag = tools_structural_tag if output_format is not None: structural_tag = StructuralTag( diff --git a/vllm/tool_parsers/abstract_tool_parser.py b/vllm/tool_parsers/abstract_tool_parser.py index bcbb5757c1..bc5abea3a2 100644 --- a/vllm/tool_parsers/abstract_tool_parser.py +++ b/vllm/tool_parsers/abstract_tool_parser.py @@ -141,7 +141,9 @@ class ToolParser: return request json_schema_from_tool = get_json_schema_from_tools( - tool_choice=request.tool_choice, tools=request.tools + tool_choice=request.tool_choice, + tools=request.tools, + parallel_tool_calls=request.parallel_tool_calls, ) # Set structured output params for tool calling if json_schema_from_tool is not None: diff --git a/vllm/tool_parsers/mimo_tool_parser.py b/vllm/tool_parsers/mimo_tool_parser.py index cca77eb56b..3b44c5d989 100644 --- a/vllm/tool_parsers/mimo_tool_parser.py +++ b/vllm/tool_parsers/mimo_tool_parser.py @@ -1,25 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from xgrammar.structural_tag import TriggeredTagsFormat - from vllm.parser.engine.registered_adapters import MiMoParserToolAdapter -from vllm.tool_parsers.tool_strict_level import ToolStrictLevel class MiMoToolParser(MiMoParserToolAdapter): # type: ignore[valid-type, misc] structural_tag_model = "mimo" - - def get_structural_tag( - self, request, *, reasoning=False, strict_level=ToolStrictLevel.AUTO - ): - tag = super().get_structural_tag( - request, reasoning=reasoning, strict_level=strict_level - ) - if ( - tag is not None - and request.parallel_tool_calls is False - and isinstance(tag.format, TriggeredTagsFormat) - ): - tag.format.stop_after_first = True - return tag diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index bc8fab8fe6..ef57e85fab 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -9,6 +9,7 @@ from openai.types.responses.response import ToolChoice as ResponsesToolChoice from openai.types.responses.tool import Tool as ResponsesTool from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed from openai.types.responses.tool_choice_function import ToolChoiceFunction +from pydantic import BaseModel from xgrammar import StructuralTag, normalize_tool_choice from xgrammar import get_model_structural_tag as get_xgrammar_model_structural_tag from xgrammar.openai_tool_call_schema import ( @@ -27,6 +28,7 @@ from xgrammar.structural_tag import ( StarFormat, TagFormat, TagsWithSeparatorFormat, + TokenTriggeredTagsFormat, TriggeredTagsFormat, ) @@ -197,6 +199,34 @@ def get_model_structural_tag( ) +_TOOL_CALL_LIST_FORMATS = ( + TriggeredTagsFormat, + TokenTriggeredTagsFormat, + TagsWithSeparatorFormat, +) + + +def limit_to_single_tool_call(tag: StructuralTag) -> StructuralTag: + """Stop every tool-call list in ``tag`` after its first call, in place. + + Enforces ``parallel_tool_calls=false`` in the grammar. In every registered + tag, ``_TOOL_CALL_LIST_FORMATS`` repeat tool calls, never arguments. + """ + + def visit(node: Any) -> None: + if isinstance(node, BaseModel): + if isinstance(node, _TOOL_CALL_LIST_FORMATS): + node.stop_after_first = True + for name in type(node).model_fields: + visit(getattr(node, name)) + elif isinstance(node, (list, tuple)): + for item in node: + visit(item) + + visit(tag.format) + return tag + + def _dump_tool_for_xgrammar( tool: ChatCompletionToolsParam | ResponsesTool, ) -> dict[str, Any]: diff --git a/vllm/tool_parsers/utils.py b/vllm/tool_parsers/utils.py index d2c8c6f412..081e58da04 100644 --- a/vllm/tool_parsers/utils.py +++ b/vllm/tool_parsers/utils.py @@ -347,6 +347,7 @@ def _get_tool_schema_defs( def _get_json_schema_from_tools( tools: list[Tool], + parallel_tool_calls: bool | None = None, ) -> dict: fn_tool_schemas: list[dict[str, Any]] = [] fn_tools: list[Tool] = [] @@ -361,7 +362,7 @@ def _get_json_schema_from_tools( elif _is_function_tool(tool): fn_tool_schemas.append(_get_tool_schema_from_tool(tool)) fn_tools.append(tool) - json_schema = { + json_schema: dict[str, Any] = { "type": "array", "minItems": 1, "items": { @@ -369,6 +370,10 @@ def _get_json_schema_from_tools( "anyOf": fn_tool_schemas, }, } + # Only an explicit `false` opts out of parallel tool calls; `None` means the + # field was never set and the default (true) applies. + if parallel_tool_calls is False: + json_schema["maxItems"] = 1 json_schema_defs = _get_tool_schema_defs(fn_tools) if json_schema_defs: json_schema["$defs"] = json_schema_defs @@ -378,6 +383,7 @@ def _get_json_schema_from_tools( def get_json_schema_from_tools( tool_choice: str | ToolChoiceFunction | ChatCompletionNamedToolChoiceParam, tools: list[Tool] | None, + parallel_tool_calls: bool | None = None, ) -> str | dict | None: # tool_choice: "none" if tool_choice in ("none", None) or tools is None: @@ -413,7 +419,7 @@ def get_json_schema_from_tools( return chat_tool_map[tool_name].function.parameters # tool_choice: "required" if tool_choice == "required": - return _get_json_schema_from_tools(tools) + return _get_json_schema_from_tools(tools, parallel_tool_calls) # tool_choice: "auto" return None