[Bugfix] Support repsonse_format + tool_choice=auto (#56086)

Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
Co-authored-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com>
Co-authored-by: pablopupo <145598901+pablopupo@users.noreply.github.com>
Co-authored-by: hubunt <150658615+hubunt@users.noreply.github.com>
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Artem Perevedentsev
2026-09-27 21:17:34 +04:00
committed by GitHub
co-authored by Yufeng He pablopupo hubunt Claude Sonnet 5
parent 0376f81530
commit 2b9b55c7f1
6 changed files with 472 additions and 62 deletions
+305
View File
@@ -0,0 +1,305 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
from xgrammar import Grammar
from xgrammar.testing import _is_grammar_accept_string
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
)
from vllm.parser.abstract_parser import DelegatingParser
from vllm.sampling_params import StructuredOutputsParams
from vllm.tool_parsers.abstract_tool_parser import ToolParser
from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser
from vllm.tool_parsers.structural_tag_registry import ToolChoice
class TestToolChoice_Plus_ResponseFormat:
"""Note(arpera):
Test cases for tool_choice={auto,required} + response_format
To keep it short:
DelegatingParser.adjust_request behavior in some corner cases is checked there
Initial bug report:
https://github.com/vllm-project/vllm/issues/39929
And PR that fixed this:
https://github.com/vllm-project/vllm/pull/56086
"""
# ================================
# Helper methods
# ================================
@staticmethod
def _tools(strict: bool) -> list[ChatCompletionToolsParam]:
"""Single get_weather tool, optionally marked as strict"""
function: dict[str, Any] = {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
}
if strict:
function["strict"] = True
return [ChatCompletionToolsParam(type="function", function=function)]
@staticmethod
def _qwen_tool_call() -> str:
"""Tool call for Qwen model that do support structural tag"""
return (
"<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n"
"</parameter>\n</function>\n</tool_call>"
)
@staticmethod
def _json_schema_response_format() -> dict:
return {
"type": "json_schema",
"json_schema": {
"name": "answer",
"schema": {
"type": "object",
"properties": {"text": {"type": "string"}},
"required": ["text"],
},
},
}
@staticmethod
def _setup_request(
tools: list[ChatCompletionToolsParam],
tool_choice: ToolChoice,
response_format: dict,
) -> ChatCompletionRequest:
request = ChatCompletionRequest(
messages=[], # for our test cases it's always empty
model="m", # Just a placeholder, don't pay much attention
tools=tools,
tool_choice=tool_choice,
response_format=response_format,
)
return request
@staticmethod
def _setup_abstract_parser(
tools: list[ChatCompletionToolsParam],
) -> DelegatingParser:
"""Construct parser that does NOT support structural tag"""
class TestParser(DelegatingParser):
tool_parser_cls = ToolParser
return TestParser(MagicMock(), tools=tools)
@staticmethod
def _setup_qwen_parser(
tools: list[ChatCompletionToolsParam],
) -> DelegatingParser:
"""Construct parser that supports structural tag"""
class TestParser(DelegatingParser):
tool_parser_cls = Qwen3EngineToolParser
return TestParser(MagicMock(), tools=tools)
# ================================
# Test cases
# tool_choice=auto + response_format
# ================================
@pytest.mark.parametrize(
# In this test we check that for response_format
# resulting grammar accepts @compliant_output and rejects @non_compliant_output
# You can add more examples here if you see some corner cases not covered
("response_format", "compliant_output", "non_compliant_output"),
[
(_json_schema_response_format(), '{"text": "hi"}', '{"foo": 1}'),
({"type": "json_object"}, '{"any": 1}', "[1, 2]"),
],
# We test here two different response_format types:
ids=["json_schema", "json_object"],
)
def test_auto_with_strict_tools(
self,
response_format: dict,
compliant_output: str,
non_compliant_output: str,
):
tools = self._tools(strict=True)
request = self._setup_request(
tools=tools,
tool_choice="auto",
response_format=response_format,
)
parser = self._setup_qwen_parser(tools)
out = parser.adjust_request(request)
# Now check that request does not have response_format anymore
# but instead has structured_outputs set as structural tag
# And this structural tag is OR operation
assert out.tool_choice == "auto"
assert out.response_format is None
assert out.structured_outputs is not None
tag = json.loads(out.structured_outputs.structural_tag)
assert tag["format"]["type"] == "or"
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
assert _is_grammar_accept_string(grammar, compliant_output)
assert not _is_grammar_accept_string(grammar, non_compliant_output)
# Also tool call must be accepted by grammar
assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
# IMPORTANT(arpera): Regression test
# If we in adjust_request implementation by mistake
# construct structural tag using tool_choice=auto
# then such a structural tag would allow plain text as well
# We need to be sure that plain text is NOT accepted in our case
assert not _is_grammar_accept_string(grammar, "Hello")
def test_auto_without_strict_tools(self):
tools = self._tools(strict=False)
request = self._setup_request(
tools=tools,
tool_choice="auto",
response_format=self._json_schema_response_format(),
)
parser = self._setup_qwen_parser(tools)
# There must be a warning that tool calls are disabled
# Consume that warning
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
out = parser.adjust_request(request)
assert out.response_format is not None
assert out.structured_outputs is None
mock_warn.assert_called_once()
def test_when_model_does_not_have_structural_tag(self):
"""Note(arpera):
When model does NOT have structural tag support
we apply constraint only for response_format
"""
tools = self._tools(strict=True)
request = self._setup_request(
tools=tools,
tool_choice="auto",
response_format=self._json_schema_response_format(),
)
# SIC! use parser whose model does NOT support structural tag
parser = self._setup_abstract_parser(tools)
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
out = parser.adjust_request(request)
assert out.response_format is not None
assert out.structured_outputs is None
mock_warn.assert_called_once()
def test_auto_with_xgrammar_unsupported_schema(self):
"""Conrer case based on Vadim's feedback in PR #56086
https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459
"""
tools = self._tools(strict=True)
request = self._setup_request(
tools=tools,
tool_choice="auto",
response_format={
"type": "json_schema",
"json_schema": {
"name": "answer",
"schema": {"type": "integer", "multipleOf": 2},
},
},
)
parser = self._setup_qwen_parser(tools)
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
out = parser.adjust_request(request)
assert out.response_format is not None
assert out.structured_outputs is None
mock_warn.assert_called_once()
def test_auto_with_xgrammar_unsupported_nested_schema(self):
"""Conrer case based on Vadim's feedback in PR #56086
https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459
"""
tools = self._tools(strict=True)
request = self._setup_request(
tools=tools,
tool_choice="auto",
response_format={
"type": "json_schema",
"json_schema": {
"name": "answer",
"schema": {
"type": "object",
"properties": {"value": {"type": "number", "multipleOf": 0.5}},
"required": ["value"],
},
},
},
)
parser = self._setup_qwen_parser(tools)
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
out = parser.adjust_request(request)
assert out.response_format is not None
assert out.structured_outputs is None
mock_warn.assert_called_once()
def test_auto_with_lark_grammar(self):
"""Conrer case based on Vadim's feedback in PR #56086
https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459
Lark grammars are converted to EBNF before being merged into the tag.
"""
tools = self._tools(strict=True)
request = ChatCompletionRequest(
messages=[],
model="m",
tools=tools,
tool_choice="auto",
structured_outputs=StructuredOutputsParams(grammar='start: "ok"'),
)
parser = self._setup_qwen_parser(tools)
out = parser.adjust_request(request)
assert out.structured_outputs is not None
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
assert _is_grammar_accept_string(grammar, "ok")
assert not _is_grammar_accept_string(grammar, "bad")
assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
# ================================
# Test cases
# tool_choice=required + response_format
# ================================
def test_required(self):
tools = self._tools(strict=True)
request = self._setup_request(
tools=tools,
tool_choice="required",
response_format=self._json_schema_response_format(),
)
parser = self._setup_qwen_parser(tools)
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
out = parser.adjust_request(request)
assert out.response_format is None
assert out.structured_outputs is not None
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
assert not _is_grammar_accept_string(grammar, '{"text": "hi"}')
mock_warn.assert_called_once()
+16
View File
@@ -1196,6 +1196,21 @@ class TestAdjustRequest:
"TOOL_CALL_2_FUNCTION_FIRST",
],
),
(
{
"tool_choice": "auto",
"strict_tools": True,
"response_format_type": "json_schema",
},
[
"COMMENTARY",
"TOOL_CALL_1_CHANNEL_FIRST",
"TOOL_CALL_1_FUNCTION_FIRST",
"TOOL_CALL_2_CHANNEL_FIRST",
"TOOL_CALL_2_FUNCTION_FIRST",
"FINAL_JSON_SCHEMA",
],
),
(
{"response_format_type": "json_schema"},
["FINAL_JSON_SCHEMA"],
@@ -1281,6 +1296,7 @@ class TestAdjustRequest:
"tool_auto_strict",
"tool_required",
"tool_named",
"pr56086_tool_auto_strict_response_format_json_schema",
"response_format_json_schema",
"response_format_json_object",
"structured_outputs_json",
+138 -16
View File
@@ -5,10 +5,21 @@ import contextlib
import json
from abc import abstractmethod
from collections.abc import Sequence
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from openai.types.responses import ToolChoiceFunction
from pydantic import TypeAdapter, ValidationError
from xgrammar import Grammar, StructuralTag
from xgrammar.structural_tag import (
ConstStringFormat,
Format,
GrammarFormat,
JSONSchemaFormat,
OrFormat,
RegexFormat,
TagFormat,
TriggeredTagsFormat,
)
from vllm.entrypoints.chat_utils import (
get_tool_call_id_type,
@@ -25,17 +36,19 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.parser.metrics import record_tool_parser_invocation
from vllm.parser.utils import count_history_tool_calls
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
from vllm.sampling_params import StructuredOutputsParams
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
from vllm.tool_parsers.streaming import (
extract_named_tool_call_streaming,
extract_required_tool_call_streaming,
)
from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
logger = init_logger(__name__)
@@ -236,6 +249,67 @@ class Parser:
return 0
def structured_outputs_to_format(params: StructuredOutputsParams) -> Format | None:
"""Map StructuredOutputsParams in a XGrammar Format."""
if params.json_object:
return JSONSchemaFormat(json_schema={"type": "object"})
if params.json is not None:
schema = params.json
if isinstance(schema, str):
schema = json.loads(schema)
return JSONSchemaFormat(json_schema=schema)
if params.regex is not None:
return RegexFormat(pattern=params.regex)
if params.choice is not None:
return OrFormat(
elements=[ConstStringFormat(value=choice) for choice in params.choice]
)
if params.grammar is not None:
from vllm.v1.structured_output.utils import grammar_is_likely_lark
grammar = params.grammar
# IMPORTANT(arpera):
# GrammarFormat only accepts EBNF.
if grammar_is_likely_lark(grammar):
try:
grammar = str(Grammar.from_lark(grammar))
except Exception as e:
raise VLLMValidationError("Invalid grammar specification.") from e
return GrammarFormat(grammar=grammar)
if params.structural_tag is not None:
s_tag = json.loads(params.structural_tag)
if "structures" in s_tag:
# LegacyStructuralTagResponseFormat
return TriggeredTagsFormat(
triggers=s_tag["triggers"],
tags=[
TagFormat(
begin=structure["begin"],
content=JSONSchemaFormat(json_schema=structure["schema"]),
end=structure["end"],
)
for structure in s_tag["structures"]
],
)
# StructuralTagResponseFormat
return StructuralTag.model_validate(s_tag).format
return None
def _xgrammar_supports(structured_outputs: StructuredOutputsParams) -> bool:
"""Whether the constraint passes xgrammar validation on its own."""
from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
try:
# validate_xgrammar_grammar rewrites `choice` into `grammar` in place.
validate_xgrammar_grammar(
SamplingParams(structured_outputs=replace(structured_outputs))
)
except (TypeError, ValueError, VLLMValidationError):
return False
return True
class DelegatingParser(Parser):
"""A Parser implementation that delegates to separate ReasoningParser and
ToolParser instances.
@@ -390,18 +464,14 @@ class DelegatingParser(Parser):
request = self._reasoning_parser.adjust_request(request)
if self._tool_parser is not None:
request = self._apply_structural_tag(request)
if self._tool_parser is not None:
request = self._tool_parser.adjust_request(request)
return request
def _apply_structural_tag(
self, request: ChatCompletionRequest | ResponsesRequest
) -> ChatCompletionRequest | ResponsesRequest:
if (
self._tool_parser is None
or self._tool_parser.structural_tag_model is None
or not request.tools
):
tool_parser = self._tool_parser
if tool_parser is None or not request.tools:
return request
need_tool_calling = (
@@ -415,17 +485,69 @@ class DelegatingParser(Parser):
if not need_tool_calling:
return request
structure_tag = self._tool_parser.get_structural_tag(
request,
reasoning=False,
strict_level=self.tool_strict_level,
)
if structure_tag is None:
structured_outputs = request.extract_structured_outputs()
is_auto = request.tool_choice == "auto"
resolved_tools = None
if tool_parser.structural_tag_model is not None:
resolved_tools = resolve_tool_strictness(
request.tools,
request.tool_choice,
self.tool_strict_level,
)
output_format = None
if resolved_tools and is_auto and structured_outputs:
if not _xgrammar_supports(structured_outputs):
logger.warning_once(
"Tool calls are not constrained for tool_choice=auto with "
"structured outputs because xgrammar does not support the "
"structured output constraint; the structured output "
"constraint applies.",
scope="local",
)
return request
output_format = structured_outputs_to_format(structured_outputs)
if resolved_tools is not None:
tag_request = request
if output_format is not None:
# IMPORTANT(arpera):
# The "auto" tag allows plain text in response.
# Use tag "required" here to ensure structured-output branch
# remains meaningful.
tag_request = request.model_copy(update={"tool_choice": "required"})
tools_structural_tag = tool_parser.get_structural_tag(
tag_request,
reasoning=False,
strict_level=self.tool_strict_level,
)
else:
tools_structural_tag = None
if tools_structural_tag is None:
if is_auto and structured_outputs is not None:
logger.warning_once(
"Tool calls are not constrained for tool_choice=auto with "
"structured outputs because structural tags are unavailable; "
"the structured output constraint applies.",
scope="local",
)
return request
structural_tag = json.dumps(structure_tag.model_dump())
structural_tag = tools_structural_tag
if output_format is not None:
structural_tag = StructuralTag(
format=OrFormat(elements=[tools_structural_tag.format, output_format])
)
elif structured_outputs is not None and not is_auto:
logger.warning_once(
"structured outputs are ignored because tool_choice forces tool call.",
scope="local",
)
request.structured_outputs = StructuredOutputsParams(
structural_tag=structural_tag,
structural_tag=json.dumps(structural_tag.model_dump()),
)
if isinstance(request, ResponsesRequest):
request.text = None
+9
View File
@@ -563,6 +563,15 @@ class CohereCommandParser(DelegatingParser):
request.response_format = None
return request
def _apply_structural_tag(
self, request: ChatCompletionRequest | ResponsesRequest
) -> ChatCompletionRequest | ResponsesRequest:
# Note(arpera):
# No need to go through _apply_structural_tag in abstract_parser.py
# since we have already go through _apply_structural_tags here
# So, override this method to return request as is
return request
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
count = 0
depth = 0
+2 -44
View File
@@ -16,14 +16,11 @@ from xgrammar.structural_tag import (
AnyTextFormat,
ConstStringFormat,
Format,
GrammarFormat,
JSONSchemaFormat,
OptionalFormat,
OrFormat,
RegexFormat,
SequenceFormat,
TagFormat,
TriggeredTagsFormat,
)
from vllm.entrypoints.chat_utils import make_tool_call_id
@@ -44,9 +41,8 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.logger import init_logger
from vllm.parser.abstract_parser import DelegatingParser
from vllm.parser.abstract_parser import DelegatingParser, structured_outputs_to_format
from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
from vllm.sampling_params import StructuredOutputsParams
from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser
from vllm.tool_parsers.structural_tag_registry import (
SimplifiedToolChoice,
@@ -428,7 +424,6 @@ _FUNCTION_CALL_BEGINS = [
" to=functions.{name}{channel}{constrain}<|message|>",
"{channel} to=functions.{name}{constrain}<|message|>",
]
_JSON_CONTENT = JSONSchemaFormat(json_schema={"type": "object"})
_ANY_CONTENT = AnyTextFormat()
@@ -531,43 +526,6 @@ def get_harmony_structural_tag(
)
def _params_to_final_content(params: StructuredOutputsParams) -> Format | None:
"""Map StructuredOutputsParams in a XGrammar Format."""
if params.json_object:
return _JSON_CONTENT
if params.json is not None:
schema = params.json
if isinstance(schema, str):
schema = json.loads(schema)
return JSONSchemaFormat(json_schema=schema)
if params.regex is not None:
return RegexFormat(pattern=params.regex)
if params.choice is not None:
return OrFormat(
elements=[ConstStringFormat(value=choice) for choice in params.choice]
)
if params.grammar is not None:
return GrammarFormat(grammar=params.grammar)
if params.structural_tag is not None:
s_tag = json.loads(params.structural_tag)
if "structures" in s_tag:
# LegacyStructuralTagResponseFormat
return TriggeredTagsFormat(
triggers=s_tag["triggers"],
tags=[
TagFormat(
begin=structure["begin"],
content=JSONSchemaFormat(json_schema=structure["schema"]),
end=structure["end"],
)
for structure in s_tag["structures"]
],
)
# StructuralTagResponseFormat
return StructuralTag.model_validate(s_tag).format
return None
def _adjust_output_format(
request: ChatCompletionRequest | ResponsesRequest,
) -> ChatCompletionRequest | ResponsesRequest:
@@ -576,7 +534,7 @@ def _adjust_output_format(
if params is None:
return request
final_content = _params_to_final_content(params)
final_content = structured_outputs_to_format(params)
if final_content is None:
return request
+2 -2
View File
@@ -121,7 +121,7 @@ def _with_tool_strict(
return tool
def _resolve_tool_strictness(
def resolve_tool_strictness(
tools: Sequence[ChatCompletionToolsParam | ResponsesTool],
tool_choice: ToolChoice,
strict_level: ToolStrictLevel,
@@ -158,7 +158,7 @@ def get_model_structural_tag(
if not tools or tool_choice == "none":
return None
tools = _resolve_tool_strictness(tools, tool_choice, strict_level)
tools = resolve_tool_strictness(tools, tool_choice, strict_level)
if tools is None:
return None