[Bugfix] Support repsonse_format + tool_choice=auto (#56086)

Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
Co-authored-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com>
Co-authored-by: pablopupo <145598901+pablopupo@users.noreply.github.com>
Co-authored-by: hubunt <150658615+hubunt@users.noreply.github.com>
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Artem Perevedentsev
2026-09-27 21:17:34 +04:00
committed by GitHub
co-authored by Yufeng He pablopupo hubunt Claude Sonnet 5
parent 0376f81530
commit 2b9b55c7f1
6 changed files with 472 additions and 62 deletions
+138 -16
View File
@@ -5,10 +5,21 @@ import contextlib
import json
from abc import abstractmethod
from collections.abc import Sequence
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from openai.types.responses import ToolChoiceFunction
from pydantic import TypeAdapter, ValidationError
from xgrammar import Grammar, StructuralTag
from xgrammar.structural_tag import (
ConstStringFormat,
Format,
GrammarFormat,
JSONSchemaFormat,
OrFormat,
RegexFormat,
TagFormat,
TriggeredTagsFormat,
)
from vllm.entrypoints.chat_utils import (
get_tool_call_id_type,
@@ -25,17 +36,19 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.parser.metrics import record_tool_parser_invocation
from vllm.parser.utils import count_history_tool_calls
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
from vllm.sampling_params import StructuredOutputsParams
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
from vllm.tool_parsers.streaming import (
extract_named_tool_call_streaming,
extract_required_tool_call_streaming,
)
from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
logger = init_logger(__name__)
@@ -236,6 +249,67 @@ class Parser:
return 0
def structured_outputs_to_format(params: StructuredOutputsParams) -> Format | None:
"""Map StructuredOutputsParams in a XGrammar Format."""
if params.json_object:
return JSONSchemaFormat(json_schema={"type": "object"})
if params.json is not None:
schema = params.json
if isinstance(schema, str):
schema = json.loads(schema)
return JSONSchemaFormat(json_schema=schema)
if params.regex is not None:
return RegexFormat(pattern=params.regex)
if params.choice is not None:
return OrFormat(
elements=[ConstStringFormat(value=choice) for choice in params.choice]
)
if params.grammar is not None:
from vllm.v1.structured_output.utils import grammar_is_likely_lark
grammar = params.grammar
# IMPORTANT(arpera):
# GrammarFormat only accepts EBNF.
if grammar_is_likely_lark(grammar):
try:
grammar = str(Grammar.from_lark(grammar))
except Exception as e:
raise VLLMValidationError("Invalid grammar specification.") from e
return GrammarFormat(grammar=grammar)
if params.structural_tag is not None:
s_tag = json.loads(params.structural_tag)
if "structures" in s_tag:
# LegacyStructuralTagResponseFormat
return TriggeredTagsFormat(
triggers=s_tag["triggers"],
tags=[
TagFormat(
begin=structure["begin"],
content=JSONSchemaFormat(json_schema=structure["schema"]),
end=structure["end"],
)
for structure in s_tag["structures"]
],
)
# StructuralTagResponseFormat
return StructuralTag.model_validate(s_tag).format
return None
def _xgrammar_supports(structured_outputs: StructuredOutputsParams) -> bool:
"""Whether the constraint passes xgrammar validation on its own."""
from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
try:
# validate_xgrammar_grammar rewrites `choice` into `grammar` in place.
validate_xgrammar_grammar(
SamplingParams(structured_outputs=replace(structured_outputs))
)
except (TypeError, ValueError, VLLMValidationError):
return False
return True
class DelegatingParser(Parser):
"""A Parser implementation that delegates to separate ReasoningParser and
ToolParser instances.
@@ -390,18 +464,14 @@ class DelegatingParser(Parser):
request = self._reasoning_parser.adjust_request(request)
if self._tool_parser is not None:
request = self._apply_structural_tag(request)
if self._tool_parser is not None:
request = self._tool_parser.adjust_request(request)
return request
def _apply_structural_tag(
self, request: ChatCompletionRequest | ResponsesRequest
) -> ChatCompletionRequest | ResponsesRequest:
if (
self._tool_parser is None
or self._tool_parser.structural_tag_model is None
or not request.tools
):
tool_parser = self._tool_parser
if tool_parser is None or not request.tools:
return request
need_tool_calling = (
@@ -415,17 +485,69 @@ class DelegatingParser(Parser):
if not need_tool_calling:
return request
structure_tag = self._tool_parser.get_structural_tag(
request,
reasoning=False,
strict_level=self.tool_strict_level,
)
if structure_tag is None:
structured_outputs = request.extract_structured_outputs()
is_auto = request.tool_choice == "auto"
resolved_tools = None
if tool_parser.structural_tag_model is not None:
resolved_tools = resolve_tool_strictness(
request.tools,
request.tool_choice,
self.tool_strict_level,
)
output_format = None
if resolved_tools and is_auto and structured_outputs:
if not _xgrammar_supports(structured_outputs):
logger.warning_once(
"Tool calls are not constrained for tool_choice=auto with "
"structured outputs because xgrammar does not support the "
"structured output constraint; the structured output "
"constraint applies.",
scope="local",
)
return request
output_format = structured_outputs_to_format(structured_outputs)
if resolved_tools is not None:
tag_request = request
if output_format is not None:
# IMPORTANT(arpera):
# The "auto" tag allows plain text in response.
# Use tag "required" here to ensure structured-output branch
# remains meaningful.
tag_request = request.model_copy(update={"tool_choice": "required"})
tools_structural_tag = tool_parser.get_structural_tag(
tag_request,
reasoning=False,
strict_level=self.tool_strict_level,
)
else:
tools_structural_tag = None
if tools_structural_tag is None:
if is_auto and structured_outputs is not None:
logger.warning_once(
"Tool calls are not constrained for tool_choice=auto with "
"structured outputs because structural tags are unavailable; "
"the structured output constraint applies.",
scope="local",
)
return request
structural_tag = json.dumps(structure_tag.model_dump())
structural_tag = tools_structural_tag
if output_format is not None:
structural_tag = StructuralTag(
format=OrFormat(elements=[tools_structural_tag.format, output_format])
)
elif structured_outputs is not None and not is_auto:
logger.warning_once(
"structured outputs are ignored because tool_choice forces tool call.",
scope="local",
)
request.structured_outputs = StructuredOutputsParams(
structural_tag=structural_tag,
structural_tag=json.dumps(structural_tag.model_dump()),
)
if isinstance(request, ResponsesRequest):
request.text = None