From 2b9b55c7f1bd344d07ca5545cd31735e580b2d1a Mon Sep 17 00:00:00 2001 From: Artem Perevedentsev Date: Sun, 27 Sep 2026 20:17:34 +0300 Subject: [PATCH] [Bugfix] Support repsonse_format + tool_choice=auto (#56086) Signed-off-by: Artem Perevedentsev Co-authored-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Co-authored-by: pablopupo <145598901+pablopupo@users.noreply.github.com> Co-authored-by: hubunt <150658615+hubunt@users.noreply.github.com> Co-authored-by: Claude Sonnet 5 --- tests/parser/test_abstract_parser.py | 305 +++++++++++++++++++ tests/parser/test_harmony.py | 16 + vllm/parser/abstract_parser.py | 154 +++++++++- vllm/parser/cohere_command.py | 9 + vllm/parser/harmony.py | 46 +-- vllm/tool_parsers/structural_tag_registry.py | 4 +- 6 files changed, 472 insertions(+), 62 deletions(-) create mode 100644 tests/parser/test_abstract_parser.py diff --git a/tests/parser/test_abstract_parser.py b/tests/parser/test_abstract_parser.py new file mode 100644 index 0000000000..56d20417d1 --- /dev/null +++ b/tests/parser/test_abstract_parser.py @@ -0,0 +1,305 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import json +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from xgrammar import Grammar +from xgrammar.testing import _is_grammar_accept_string + +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ChatCompletionToolsParam, +) +from vllm.parser.abstract_parser import DelegatingParser +from vllm.sampling_params import StructuredOutputsParams +from vllm.tool_parsers.abstract_tool_parser import ToolParser +from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser +from vllm.tool_parsers.structural_tag_registry import ToolChoice + + +class TestToolChoice_Plus_ResponseFormat: + """Note(arpera): + Test cases for tool_choice={auto,required} + response_format + To keep it short: + DelegatingParser.adjust_request behavior in some corner cases is checked there + + Initial bug report: + https://github.com/vllm-project/vllm/issues/39929 + And PR that fixed this: + https://github.com/vllm-project/vllm/pull/56086 + """ + + # ================================ + # Helper methods + # ================================ + + @staticmethod + def _tools(strict: bool) -> list[ChatCompletionToolsParam]: + """Single get_weather tool, optionally marked as strict""" + function: dict[str, Any] = { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + if strict: + function["strict"] = True + return [ChatCompletionToolsParam(type="function", function=function)] + + @staticmethod + def _qwen_tool_call() -> str: + """Tool call for Qwen model that do support structural tag""" + return ( + "\n\n\nParis\n" + "\n\n" + ) + + @staticmethod + def _json_schema_response_format() -> dict: + return { + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": { + "type": "object", + "properties": {"text": {"type": "string"}}, + "required": ["text"], + }, + }, + } + + @staticmethod + def _setup_request( + tools: list[ChatCompletionToolsParam], + tool_choice: ToolChoice, + response_format: dict, + ) -> ChatCompletionRequest: + request = ChatCompletionRequest( + messages=[], # for our test cases it's always empty + model="m", # Just a placeholder, don't pay much attention + tools=tools, + tool_choice=tool_choice, + response_format=response_format, + ) + return request + + @staticmethod + def _setup_abstract_parser( + tools: list[ChatCompletionToolsParam], + ) -> DelegatingParser: + """Construct parser that does NOT support structural tag""" + + class TestParser(DelegatingParser): + tool_parser_cls = ToolParser + + return TestParser(MagicMock(), tools=tools) + + @staticmethod + def _setup_qwen_parser( + tools: list[ChatCompletionToolsParam], + ) -> DelegatingParser: + """Construct parser that supports structural tag""" + + class TestParser(DelegatingParser): + tool_parser_cls = Qwen3EngineToolParser + + return TestParser(MagicMock(), tools=tools) + + # ================================ + # Test cases + # tool_choice=auto + response_format + # ================================ + + @pytest.mark.parametrize( + # In this test we check that for response_format + # resulting grammar accepts @compliant_output and rejects @non_compliant_output + # You can add more examples here if you see some corner cases not covered + ("response_format", "compliant_output", "non_compliant_output"), + [ + (_json_schema_response_format(), '{"text": "hi"}', '{"foo": 1}'), + ({"type": "json_object"}, '{"any": 1}', "[1, 2]"), + ], + # We test here two different response_format types: + ids=["json_schema", "json_object"], + ) + def test_auto_with_strict_tools( + self, + response_format: dict, + compliant_output: str, + non_compliant_output: str, + ): + tools = self._tools(strict=True) + request = self._setup_request( + tools=tools, + tool_choice="auto", + response_format=response_format, + ) + parser = self._setup_qwen_parser(tools) + out = parser.adjust_request(request) + + # Now check that request does not have response_format anymore + # but instead has structured_outputs set as structural tag + # And this structural tag is OR operation + assert out.tool_choice == "auto" + assert out.response_format is None + assert out.structured_outputs is not None + tag = json.loads(out.structured_outputs.structural_tag) + assert tag["format"]["type"] == "or" + grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag) + + assert _is_grammar_accept_string(grammar, compliant_output) + assert not _is_grammar_accept_string(grammar, non_compliant_output) + + # Also tool call must be accepted by grammar + assert _is_grammar_accept_string(grammar, self._qwen_tool_call()) + + # IMPORTANT(arpera): Regression test + # If we in adjust_request implementation by mistake + # construct structural tag using tool_choice=auto + # then such a structural tag would allow plain text as well + # We need to be sure that plain text is NOT accepted in our case + assert not _is_grammar_accept_string(grammar, "Hello") + + def test_auto_without_strict_tools(self): + tools = self._tools(strict=False) + request = self._setup_request( + tools=tools, + tool_choice="auto", + response_format=self._json_schema_response_format(), + ) + parser = self._setup_qwen_parser(tools) + + # There must be a warning that tool calls are disabled + # Consume that warning + with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn: + out = parser.adjust_request(request) + + assert out.response_format is not None + assert out.structured_outputs is None + mock_warn.assert_called_once() + + def test_when_model_does_not_have_structural_tag(self): + """Note(arpera): + When model does NOT have structural tag support + we apply constraint only for response_format + """ + tools = self._tools(strict=True) + request = self._setup_request( + tools=tools, + tool_choice="auto", + response_format=self._json_schema_response_format(), + ) + # SIC! use parser whose model does NOT support structural tag + parser = self._setup_abstract_parser(tools) + + with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn: + out = parser.adjust_request(request) + + assert out.response_format is not None + assert out.structured_outputs is None + mock_warn.assert_called_once() + + def test_auto_with_xgrammar_unsupported_schema(self): + """Conrer case based on Vadim's feedback in PR #56086 + https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459 + """ + tools = self._tools(strict=True) + request = self._setup_request( + tools=tools, + tool_choice="auto", + response_format={ + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": {"type": "integer", "multipleOf": 2}, + }, + }, + ) + parser = self._setup_qwen_parser(tools) + + with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn: + out = parser.adjust_request(request) + + assert out.response_format is not None + assert out.structured_outputs is None + mock_warn.assert_called_once() + + def test_auto_with_xgrammar_unsupported_nested_schema(self): + """Conrer case based on Vadim's feedback in PR #56086 + https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459 + """ + tools = self._tools(strict=True) + request = self._setup_request( + tools=tools, + tool_choice="auto", + response_format={ + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": { + "type": "object", + "properties": {"value": {"type": "number", "multipleOf": 0.5}}, + "required": ["value"], + }, + }, + }, + ) + parser = self._setup_qwen_parser(tools) + + with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn: + out = parser.adjust_request(request) + + assert out.response_format is not None + assert out.structured_outputs is None + mock_warn.assert_called_once() + + def test_auto_with_lark_grammar(self): + """Conrer case based on Vadim's feedback in PR #56086 + https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459 + Lark grammars are converted to EBNF before being merged into the tag. + """ + tools = self._tools(strict=True) + request = ChatCompletionRequest( + messages=[], + model="m", + tools=tools, + tool_choice="auto", + structured_outputs=StructuredOutputsParams(grammar='start: "ok"'), + ) + parser = self._setup_qwen_parser(tools) + out = parser.adjust_request(request) + + assert out.structured_outputs is not None + grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag) + assert _is_grammar_accept_string(grammar, "ok") + assert not _is_grammar_accept_string(grammar, "bad") + assert _is_grammar_accept_string(grammar, self._qwen_tool_call()) + + # ================================ + # Test cases + # tool_choice=required + response_format + # ================================ + + def test_required(self): + tools = self._tools(strict=True) + request = self._setup_request( + tools=tools, + tool_choice="required", + response_format=self._json_schema_response_format(), + ) + parser = self._setup_qwen_parser(tools) + + with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn: + out = parser.adjust_request(request) + + assert out.response_format is None + assert out.structured_outputs is not None + grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag) + assert _is_grammar_accept_string(grammar, self._qwen_tool_call()) + assert not _is_grammar_accept_string(grammar, '{"text": "hi"}') + mock_warn.assert_called_once() diff --git a/tests/parser/test_harmony.py b/tests/parser/test_harmony.py index 3d6fff04bf..d8db7f4057 100644 --- a/tests/parser/test_harmony.py +++ b/tests/parser/test_harmony.py @@ -1196,6 +1196,21 @@ class TestAdjustRequest: "TOOL_CALL_2_FUNCTION_FIRST", ], ), + ( + { + "tool_choice": "auto", + "strict_tools": True, + "response_format_type": "json_schema", + }, + [ + "COMMENTARY", + "TOOL_CALL_1_CHANNEL_FIRST", + "TOOL_CALL_1_FUNCTION_FIRST", + "TOOL_CALL_2_CHANNEL_FIRST", + "TOOL_CALL_2_FUNCTION_FIRST", + "FINAL_JSON_SCHEMA", + ], + ), ( {"response_format_type": "json_schema"}, ["FINAL_JSON_SCHEMA"], @@ -1281,6 +1296,7 @@ class TestAdjustRequest: "tool_auto_strict", "tool_required", "tool_named", + "pr56086_tool_auto_strict_response_format_json_schema", "response_format_json_schema", "response_format_json_object", "structured_outputs_json", diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 0e84e710ed..20f22bd0e3 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -5,10 +5,21 @@ import contextlib import json from abc import abstractmethod from collections.abc import Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from openai.types.responses import ToolChoiceFunction from pydantic import TypeAdapter, ValidationError +from xgrammar import Grammar, StructuralTag +from xgrammar.structural_tag import ( + ConstStringFormat, + Format, + GrammarFormat, + JSONSchemaFormat, + OrFormat, + RegexFormat, + TagFormat, + TriggeredTagsFormat, +) from vllm.entrypoints.chat_utils import ( get_tool_call_id_type, @@ -25,17 +36,19 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ) from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger from vllm.parser.metrics import record_tool_parser_invocation from vllm.parser.utils import count_history_tool_calls from vllm.reasoning.abs_reasoning_parsers import ReasoningParser -from vllm.sampling_params import StructuredOutputsParams +from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser from vllm.tool_parsers.streaming import ( extract_named_tool_call_streaming, extract_required_tool_call_streaming, ) +from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness from vllm.tool_parsers.tool_strict_level import ToolStrictLevel logger = init_logger(__name__) @@ -236,6 +249,67 @@ class Parser: return 0 +def structured_outputs_to_format(params: StructuredOutputsParams) -> Format | None: + """Map StructuredOutputsParams in a XGrammar Format.""" + if params.json_object: + return JSONSchemaFormat(json_schema={"type": "object"}) + if params.json is not None: + schema = params.json + if isinstance(schema, str): + schema = json.loads(schema) + return JSONSchemaFormat(json_schema=schema) + if params.regex is not None: + return RegexFormat(pattern=params.regex) + if params.choice is not None: + return OrFormat( + elements=[ConstStringFormat(value=choice) for choice in params.choice] + ) + if params.grammar is not None: + from vllm.v1.structured_output.utils import grammar_is_likely_lark + + grammar = params.grammar + # IMPORTANT(arpera): + # GrammarFormat only accepts EBNF. + if grammar_is_likely_lark(grammar): + try: + grammar = str(Grammar.from_lark(grammar)) + except Exception as e: + raise VLLMValidationError("Invalid grammar specification.") from e + return GrammarFormat(grammar=grammar) + if params.structural_tag is not None: + s_tag = json.loads(params.structural_tag) + if "structures" in s_tag: + # LegacyStructuralTagResponseFormat + return TriggeredTagsFormat( + triggers=s_tag["triggers"], + tags=[ + TagFormat( + begin=structure["begin"], + content=JSONSchemaFormat(json_schema=structure["schema"]), + end=structure["end"], + ) + for structure in s_tag["structures"] + ], + ) + # StructuralTagResponseFormat + return StructuralTag.model_validate(s_tag).format + return None + + +def _xgrammar_supports(structured_outputs: StructuredOutputsParams) -> bool: + """Whether the constraint passes xgrammar validation on its own.""" + from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar + + try: + # validate_xgrammar_grammar rewrites `choice` into `grammar` in place. + validate_xgrammar_grammar( + SamplingParams(structured_outputs=replace(structured_outputs)) + ) + except (TypeError, ValueError, VLLMValidationError): + return False + return True + + class DelegatingParser(Parser): """A Parser implementation that delegates to separate ReasoningParser and ToolParser instances. @@ -390,18 +464,14 @@ class DelegatingParser(Parser): request = self._reasoning_parser.adjust_request(request) if self._tool_parser is not None: request = self._apply_structural_tag(request) - if self._tool_parser is not None: request = self._tool_parser.adjust_request(request) return request def _apply_structural_tag( self, request: ChatCompletionRequest | ResponsesRequest ) -> ChatCompletionRequest | ResponsesRequest: - if ( - self._tool_parser is None - or self._tool_parser.structural_tag_model is None - or not request.tools - ): + tool_parser = self._tool_parser + if tool_parser is None or not request.tools: return request need_tool_calling = ( @@ -415,17 +485,69 @@ class DelegatingParser(Parser): if not need_tool_calling: return request - structure_tag = self._tool_parser.get_structural_tag( - request, - reasoning=False, - strict_level=self.tool_strict_level, - ) - if structure_tag is None: + structured_outputs = request.extract_structured_outputs() + is_auto = request.tool_choice == "auto" + + resolved_tools = None + if tool_parser.structural_tag_model is not None: + resolved_tools = resolve_tool_strictness( + request.tools, + request.tool_choice, + self.tool_strict_level, + ) + + output_format = None + if resolved_tools and is_auto and structured_outputs: + if not _xgrammar_supports(structured_outputs): + logger.warning_once( + "Tool calls are not constrained for tool_choice=auto with " + "structured outputs because xgrammar does not support the " + "structured output constraint; the structured output " + "constraint applies.", + scope="local", + ) + return request + output_format = structured_outputs_to_format(structured_outputs) + + if resolved_tools is not None: + tag_request = request + if output_format is not None: + # IMPORTANT(arpera): + # The "auto" tag allows plain text in response. + # Use tag "required" here to ensure structured-output branch + # remains meaningful. + tag_request = request.model_copy(update={"tool_choice": "required"}) + tools_structural_tag = tool_parser.get_structural_tag( + tag_request, + reasoning=False, + strict_level=self.tool_strict_level, + ) + else: + tools_structural_tag = None + + if tools_structural_tag is None: + if is_auto and structured_outputs is not None: + logger.warning_once( + "Tool calls are not constrained for tool_choice=auto with " + "structured outputs because structural tags are unavailable; " + "the structured output constraint applies.", + scope="local", + ) return request - structural_tag = json.dumps(structure_tag.model_dump()) + structural_tag = tools_structural_tag + if output_format is not None: + structural_tag = StructuralTag( + format=OrFormat(elements=[tools_structural_tag.format, output_format]) + ) + elif structured_outputs is not None and not is_auto: + logger.warning_once( + "structured outputs are ignored because tool_choice forces tool call.", + scope="local", + ) + request.structured_outputs = StructuredOutputsParams( - structural_tag=structural_tag, + structural_tag=json.dumps(structural_tag.model_dump()), ) if isinstance(request, ResponsesRequest): request.text = None diff --git a/vllm/parser/cohere_command.py b/vllm/parser/cohere_command.py index e5ed7aed33..c37cc569ac 100644 --- a/vllm/parser/cohere_command.py +++ b/vllm/parser/cohere_command.py @@ -563,6 +563,15 @@ class CohereCommandParser(DelegatingParser): request.response_format = None return request + def _apply_structural_tag( + self, request: ChatCompletionRequest | ResponsesRequest + ) -> ChatCompletionRequest | ResponsesRequest: + # Note(arpera): + # No need to go through _apply_structural_tag in abstract_parser.py + # since we have already go through _apply_structural_tags here + # So, override this method to return request as is + return request + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: count = 0 depth = 0 diff --git a/vllm/parser/harmony.py b/vllm/parser/harmony.py index 5748bd6653..26f311c7e9 100644 --- a/vllm/parser/harmony.py +++ b/vllm/parser/harmony.py @@ -16,14 +16,11 @@ from xgrammar.structural_tag import ( AnyTextFormat, ConstStringFormat, Format, - GrammarFormat, JSONSchemaFormat, OptionalFormat, OrFormat, - RegexFormat, SequenceFormat, TagFormat, - TriggeredTagsFormat, ) from vllm.entrypoints.chat_utils import make_tool_call_id @@ -44,9 +41,8 @@ from vllm.entrypoints.openai.parser.harmony_utils import ( ) from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.logger import init_logger -from vllm.parser.abstract_parser import DelegatingParser +from vllm.parser.abstract_parser import DelegatingParser, structured_outputs_to_format from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser -from vllm.sampling_params import StructuredOutputsParams from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser from vllm.tool_parsers.structural_tag_registry import ( SimplifiedToolChoice, @@ -428,7 +424,6 @@ _FUNCTION_CALL_BEGINS = [ " to=functions.{name}{channel}{constrain}<|message|>", "{channel} to=functions.{name}{constrain}<|message|>", ] -_JSON_CONTENT = JSONSchemaFormat(json_schema={"type": "object"}) _ANY_CONTENT = AnyTextFormat() @@ -531,43 +526,6 @@ def get_harmony_structural_tag( ) -def _params_to_final_content(params: StructuredOutputsParams) -> Format | None: - """Map StructuredOutputsParams in a XGrammar Format.""" - if params.json_object: - return _JSON_CONTENT - if params.json is not None: - schema = params.json - if isinstance(schema, str): - schema = json.loads(schema) - return JSONSchemaFormat(json_schema=schema) - if params.regex is not None: - return RegexFormat(pattern=params.regex) - if params.choice is not None: - return OrFormat( - elements=[ConstStringFormat(value=choice) for choice in params.choice] - ) - if params.grammar is not None: - return GrammarFormat(grammar=params.grammar) - if params.structural_tag is not None: - s_tag = json.loads(params.structural_tag) - if "structures" in s_tag: - # LegacyStructuralTagResponseFormat - return TriggeredTagsFormat( - triggers=s_tag["triggers"], - tags=[ - TagFormat( - begin=structure["begin"], - content=JSONSchemaFormat(json_schema=structure["schema"]), - end=structure["end"], - ) - for structure in s_tag["structures"] - ], - ) - # StructuralTagResponseFormat - return StructuralTag.model_validate(s_tag).format - return None - - def _adjust_output_format( request: ChatCompletionRequest | ResponsesRequest, ) -> ChatCompletionRequest | ResponsesRequest: @@ -576,7 +534,7 @@ def _adjust_output_format( if params is None: return request - final_content = _params_to_final_content(params) + final_content = structured_outputs_to_format(params) if final_content is None: return request diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index ba86395b7a..f3e5c9ec98 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -121,7 +121,7 @@ def _with_tool_strict( return tool -def _resolve_tool_strictness( +def resolve_tool_strictness( tools: Sequence[ChatCompletionToolsParam | ResponsesTool], tool_choice: ToolChoice, strict_level: ToolStrictLevel, @@ -158,7 +158,7 @@ def get_model_structural_tag( if not tools or tool_choice == "none": return None - tools = _resolve_tool_strictness(tools, tool_choice, strict_level) + tools = resolve_tool_strictness(tools, tool_choice, strict_level) if tools is None: return None