[Bugfix] Support repsonse_format + tool_choice=auto (#56086)
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com> Co-authored-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Co-authored-by: pablopupo <145598901+pablopupo@users.noreply.github.com> Co-authored-by: hubunt <150658615+hubunt@users.noreply.github.com> Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Yufeng He
pablopupo
hubunt
Claude Sonnet 5
parent
0376f81530
commit
2b9b55c7f1
@@ -0,0 +1,305 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from xgrammar import Grammar
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tool_parsers.abstract_tool_parser import ToolParser
|
||||
from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser
|
||||
from vllm.tool_parsers.structural_tag_registry import ToolChoice
|
||||
|
||||
|
||||
class TestToolChoice_Plus_ResponseFormat:
|
||||
"""Note(arpera):
|
||||
Test cases for tool_choice={auto,required} + response_format
|
||||
To keep it short:
|
||||
DelegatingParser.adjust_request behavior in some corner cases is checked there
|
||||
|
||||
Initial bug report:
|
||||
https://github.com/vllm-project/vllm/issues/39929
|
||||
And PR that fixed this:
|
||||
https://github.com/vllm-project/vllm/pull/56086
|
||||
"""
|
||||
|
||||
# ================================
|
||||
# Helper methods
|
||||
# ================================
|
||||
|
||||
@staticmethod
|
||||
def _tools(strict: bool) -> list[ChatCompletionToolsParam]:
|
||||
"""Single get_weather tool, optionally marked as strict"""
|
||||
function: dict[str, Any] = {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
}
|
||||
if strict:
|
||||
function["strict"] = True
|
||||
return [ChatCompletionToolsParam(type="function", function=function)]
|
||||
|
||||
@staticmethod
|
||||
def _qwen_tool_call() -> str:
|
||||
"""Tool call for Qwen model that do support structural tag"""
|
||||
return (
|
||||
"<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n"
|
||||
"</parameter>\n</function>\n</tool_call>"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _json_schema_response_format() -> dict:
|
||||
return {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "answer",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"text": {"type": "string"}},
|
||||
"required": ["text"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _setup_request(
|
||||
tools: list[ChatCompletionToolsParam],
|
||||
tool_choice: ToolChoice,
|
||||
response_format: dict,
|
||||
) -> ChatCompletionRequest:
|
||||
request = ChatCompletionRequest(
|
||||
messages=[], # for our test cases it's always empty
|
||||
model="m", # Just a placeholder, don't pay much attention
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
response_format=response_format,
|
||||
)
|
||||
return request
|
||||
|
||||
@staticmethod
|
||||
def _setup_abstract_parser(
|
||||
tools: list[ChatCompletionToolsParam],
|
||||
) -> DelegatingParser:
|
||||
"""Construct parser that does NOT support structural tag"""
|
||||
|
||||
class TestParser(DelegatingParser):
|
||||
tool_parser_cls = ToolParser
|
||||
|
||||
return TestParser(MagicMock(), tools=tools)
|
||||
|
||||
@staticmethod
|
||||
def _setup_qwen_parser(
|
||||
tools: list[ChatCompletionToolsParam],
|
||||
) -> DelegatingParser:
|
||||
"""Construct parser that supports structural tag"""
|
||||
|
||||
class TestParser(DelegatingParser):
|
||||
tool_parser_cls = Qwen3EngineToolParser
|
||||
|
||||
return TestParser(MagicMock(), tools=tools)
|
||||
|
||||
# ================================
|
||||
# Test cases
|
||||
# tool_choice=auto + response_format
|
||||
# ================================
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
# In this test we check that for response_format
|
||||
# resulting grammar accepts @compliant_output and rejects @non_compliant_output
|
||||
# You can add more examples here if you see some corner cases not covered
|
||||
("response_format", "compliant_output", "non_compliant_output"),
|
||||
[
|
||||
(_json_schema_response_format(), '{"text": "hi"}', '{"foo": 1}'),
|
||||
({"type": "json_object"}, '{"any": 1}', "[1, 2]"),
|
||||
],
|
||||
# We test here two different response_format types:
|
||||
ids=["json_schema", "json_object"],
|
||||
)
|
||||
def test_auto_with_strict_tools(
|
||||
self,
|
||||
response_format: dict,
|
||||
compliant_output: str,
|
||||
non_compliant_output: str,
|
||||
):
|
||||
tools = self._tools(strict=True)
|
||||
request = self._setup_request(
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
response_format=response_format,
|
||||
)
|
||||
parser = self._setup_qwen_parser(tools)
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
# Now check that request does not have response_format anymore
|
||||
# but instead has structured_outputs set as structural tag
|
||||
# And this structural tag is OR operation
|
||||
assert out.tool_choice == "auto"
|
||||
assert out.response_format is None
|
||||
assert out.structured_outputs is not None
|
||||
tag = json.loads(out.structured_outputs.structural_tag)
|
||||
assert tag["format"]["type"] == "or"
|
||||
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
|
||||
|
||||
assert _is_grammar_accept_string(grammar, compliant_output)
|
||||
assert not _is_grammar_accept_string(grammar, non_compliant_output)
|
||||
|
||||
# Also tool call must be accepted by grammar
|
||||
assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
|
||||
|
||||
# IMPORTANT(arpera): Regression test
|
||||
# If we in adjust_request implementation by mistake
|
||||
# construct structural tag using tool_choice=auto
|
||||
# then such a structural tag would allow plain text as well
|
||||
# We need to be sure that plain text is NOT accepted in our case
|
||||
assert not _is_grammar_accept_string(grammar, "Hello")
|
||||
|
||||
def test_auto_without_strict_tools(self):
|
||||
tools = self._tools(strict=False)
|
||||
request = self._setup_request(
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
response_format=self._json_schema_response_format(),
|
||||
)
|
||||
parser = self._setup_qwen_parser(tools)
|
||||
|
||||
# There must be a warning that tool calls are disabled
|
||||
# Consume that warning
|
||||
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.response_format is not None
|
||||
assert out.structured_outputs is None
|
||||
mock_warn.assert_called_once()
|
||||
|
||||
def test_when_model_does_not_have_structural_tag(self):
|
||||
"""Note(arpera):
|
||||
When model does NOT have structural tag support
|
||||
we apply constraint only for response_format
|
||||
"""
|
||||
tools = self._tools(strict=True)
|
||||
request = self._setup_request(
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
response_format=self._json_schema_response_format(),
|
||||
)
|
||||
# SIC! use parser whose model does NOT support structural tag
|
||||
parser = self._setup_abstract_parser(tools)
|
||||
|
||||
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.response_format is not None
|
||||
assert out.structured_outputs is None
|
||||
mock_warn.assert_called_once()
|
||||
|
||||
def test_auto_with_xgrammar_unsupported_schema(self):
|
||||
"""Conrer case based on Vadim's feedback in PR #56086
|
||||
https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459
|
||||
"""
|
||||
tools = self._tools(strict=True)
|
||||
request = self._setup_request(
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
response_format={
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "answer",
|
||||
"schema": {"type": "integer", "multipleOf": 2},
|
||||
},
|
||||
},
|
||||
)
|
||||
parser = self._setup_qwen_parser(tools)
|
||||
|
||||
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.response_format is not None
|
||||
assert out.structured_outputs is None
|
||||
mock_warn.assert_called_once()
|
||||
|
||||
def test_auto_with_xgrammar_unsupported_nested_schema(self):
|
||||
"""Conrer case based on Vadim's feedback in PR #56086
|
||||
https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459
|
||||
"""
|
||||
tools = self._tools(strict=True)
|
||||
request = self._setup_request(
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
response_format={
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "answer",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"value": {"type": "number", "multipleOf": 0.5}},
|
||||
"required": ["value"],
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
parser = self._setup_qwen_parser(tools)
|
||||
|
||||
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.response_format is not None
|
||||
assert out.structured_outputs is None
|
||||
mock_warn.assert_called_once()
|
||||
|
||||
def test_auto_with_lark_grammar(self):
|
||||
"""Conrer case based on Vadim's feedback in PR #56086
|
||||
https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459
|
||||
Lark grammars are converted to EBNF before being merged into the tag.
|
||||
"""
|
||||
tools = self._tools(strict=True)
|
||||
request = ChatCompletionRequest(
|
||||
messages=[],
|
||||
model="m",
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
structured_outputs=StructuredOutputsParams(grammar='start: "ok"'),
|
||||
)
|
||||
parser = self._setup_qwen_parser(tools)
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.structured_outputs is not None
|
||||
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
|
||||
assert _is_grammar_accept_string(grammar, "ok")
|
||||
assert not _is_grammar_accept_string(grammar, "bad")
|
||||
assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
|
||||
|
||||
# ================================
|
||||
# Test cases
|
||||
# tool_choice=required + response_format
|
||||
# ================================
|
||||
|
||||
def test_required(self):
|
||||
tools = self._tools(strict=True)
|
||||
request = self._setup_request(
|
||||
tools=tools,
|
||||
tool_choice="required",
|
||||
response_format=self._json_schema_response_format(),
|
||||
)
|
||||
parser = self._setup_qwen_parser(tools)
|
||||
|
||||
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.response_format is None
|
||||
assert out.structured_outputs is not None
|
||||
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
|
||||
assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
|
||||
assert not _is_grammar_accept_string(grammar, '{"text": "hi"}')
|
||||
mock_warn.assert_called_once()
|
||||
@@ -1196,6 +1196,21 @@ class TestAdjustRequest:
|
||||
"TOOL_CALL_2_FUNCTION_FIRST",
|
||||
],
|
||||
),
|
||||
(
|
||||
{
|
||||
"tool_choice": "auto",
|
||||
"strict_tools": True,
|
||||
"response_format_type": "json_schema",
|
||||
},
|
||||
[
|
||||
"COMMENTARY",
|
||||
"TOOL_CALL_1_CHANNEL_FIRST",
|
||||
"TOOL_CALL_1_FUNCTION_FIRST",
|
||||
"TOOL_CALL_2_CHANNEL_FIRST",
|
||||
"TOOL_CALL_2_FUNCTION_FIRST",
|
||||
"FINAL_JSON_SCHEMA",
|
||||
],
|
||||
),
|
||||
(
|
||||
{"response_format_type": "json_schema"},
|
||||
["FINAL_JSON_SCHEMA"],
|
||||
@@ -1281,6 +1296,7 @@ class TestAdjustRequest:
|
||||
"tool_auto_strict",
|
||||
"tool_required",
|
||||
"tool_named",
|
||||
"pr56086_tool_auto_strict_response_format_json_schema",
|
||||
"response_format_json_schema",
|
||||
"response_format_json_object",
|
||||
"structured_outputs_json",
|
||||
|
||||
+138
-16
@@ -5,10 +5,21 @@ import contextlib
|
||||
import json
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
|
||||
from openai.types.responses import ToolChoiceFunction
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from xgrammar import Grammar, StructuralTag
|
||||
from xgrammar.structural_tag import (
|
||||
ConstStringFormat,
|
||||
Format,
|
||||
GrammarFormat,
|
||||
JSONSchemaFormat,
|
||||
OrFormat,
|
||||
RegexFormat,
|
||||
TagFormat,
|
||||
TriggeredTagsFormat,
|
||||
)
|
||||
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
get_tool_call_id_type,
|
||||
@@ -25,17 +36,19 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.metrics import record_tool_parser_invocation
|
||||
from vllm.parser.utils import count_history_tool_calls
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
|
||||
from vllm.tool_parsers.streaming import (
|
||||
extract_named_tool_call_streaming,
|
||||
extract_required_tool_call_streaming,
|
||||
)
|
||||
from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness
|
||||
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -236,6 +249,67 @@ class Parser:
|
||||
return 0
|
||||
|
||||
|
||||
def structured_outputs_to_format(params: StructuredOutputsParams) -> Format | None:
|
||||
"""Map StructuredOutputsParams in a XGrammar Format."""
|
||||
if params.json_object:
|
||||
return JSONSchemaFormat(json_schema={"type": "object"})
|
||||
if params.json is not None:
|
||||
schema = params.json
|
||||
if isinstance(schema, str):
|
||||
schema = json.loads(schema)
|
||||
return JSONSchemaFormat(json_schema=schema)
|
||||
if params.regex is not None:
|
||||
return RegexFormat(pattern=params.regex)
|
||||
if params.choice is not None:
|
||||
return OrFormat(
|
||||
elements=[ConstStringFormat(value=choice) for choice in params.choice]
|
||||
)
|
||||
if params.grammar is not None:
|
||||
from vllm.v1.structured_output.utils import grammar_is_likely_lark
|
||||
|
||||
grammar = params.grammar
|
||||
# IMPORTANT(arpera):
|
||||
# GrammarFormat only accepts EBNF.
|
||||
if grammar_is_likely_lark(grammar):
|
||||
try:
|
||||
grammar = str(Grammar.from_lark(grammar))
|
||||
except Exception as e:
|
||||
raise VLLMValidationError("Invalid grammar specification.") from e
|
||||
return GrammarFormat(grammar=grammar)
|
||||
if params.structural_tag is not None:
|
||||
s_tag = json.loads(params.structural_tag)
|
||||
if "structures" in s_tag:
|
||||
# LegacyStructuralTagResponseFormat
|
||||
return TriggeredTagsFormat(
|
||||
triggers=s_tag["triggers"],
|
||||
tags=[
|
||||
TagFormat(
|
||||
begin=structure["begin"],
|
||||
content=JSONSchemaFormat(json_schema=structure["schema"]),
|
||||
end=structure["end"],
|
||||
)
|
||||
for structure in s_tag["structures"]
|
||||
],
|
||||
)
|
||||
# StructuralTagResponseFormat
|
||||
return StructuralTag.model_validate(s_tag).format
|
||||
return None
|
||||
|
||||
|
||||
def _xgrammar_supports(structured_outputs: StructuredOutputsParams) -> bool:
|
||||
"""Whether the constraint passes xgrammar validation on its own."""
|
||||
from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
|
||||
|
||||
try:
|
||||
# validate_xgrammar_grammar rewrites `choice` into `grammar` in place.
|
||||
validate_xgrammar_grammar(
|
||||
SamplingParams(structured_outputs=replace(structured_outputs))
|
||||
)
|
||||
except (TypeError, ValueError, VLLMValidationError):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class DelegatingParser(Parser):
|
||||
"""A Parser implementation that delegates to separate ReasoningParser and
|
||||
ToolParser instances.
|
||||
@@ -390,18 +464,14 @@ class DelegatingParser(Parser):
|
||||
request = self._reasoning_parser.adjust_request(request)
|
||||
if self._tool_parser is not None:
|
||||
request = self._apply_structural_tag(request)
|
||||
if self._tool_parser is not None:
|
||||
request = self._tool_parser.adjust_request(request)
|
||||
return request
|
||||
|
||||
def _apply_structural_tag(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
if (
|
||||
self._tool_parser is None
|
||||
or self._tool_parser.structural_tag_model is None
|
||||
or not request.tools
|
||||
):
|
||||
tool_parser = self._tool_parser
|
||||
if tool_parser is None or not request.tools:
|
||||
return request
|
||||
|
||||
need_tool_calling = (
|
||||
@@ -415,17 +485,69 @@ class DelegatingParser(Parser):
|
||||
if not need_tool_calling:
|
||||
return request
|
||||
|
||||
structure_tag = self._tool_parser.get_structural_tag(
|
||||
request,
|
||||
reasoning=False,
|
||||
strict_level=self.tool_strict_level,
|
||||
)
|
||||
if structure_tag is None:
|
||||
structured_outputs = request.extract_structured_outputs()
|
||||
is_auto = request.tool_choice == "auto"
|
||||
|
||||
resolved_tools = None
|
||||
if tool_parser.structural_tag_model is not None:
|
||||
resolved_tools = resolve_tool_strictness(
|
||||
request.tools,
|
||||
request.tool_choice,
|
||||
self.tool_strict_level,
|
||||
)
|
||||
|
||||
output_format = None
|
||||
if resolved_tools and is_auto and structured_outputs:
|
||||
if not _xgrammar_supports(structured_outputs):
|
||||
logger.warning_once(
|
||||
"Tool calls are not constrained for tool_choice=auto with "
|
||||
"structured outputs because xgrammar does not support the "
|
||||
"structured output constraint; the structured output "
|
||||
"constraint applies.",
|
||||
scope="local",
|
||||
)
|
||||
return request
|
||||
output_format = structured_outputs_to_format(structured_outputs)
|
||||
|
||||
if resolved_tools is not None:
|
||||
tag_request = request
|
||||
if output_format is not None:
|
||||
# IMPORTANT(arpera):
|
||||
# The "auto" tag allows plain text in response.
|
||||
# Use tag "required" here to ensure structured-output branch
|
||||
# remains meaningful.
|
||||
tag_request = request.model_copy(update={"tool_choice": "required"})
|
||||
tools_structural_tag = tool_parser.get_structural_tag(
|
||||
tag_request,
|
||||
reasoning=False,
|
||||
strict_level=self.tool_strict_level,
|
||||
)
|
||||
else:
|
||||
tools_structural_tag = None
|
||||
|
||||
if tools_structural_tag is None:
|
||||
if is_auto and structured_outputs is not None:
|
||||
logger.warning_once(
|
||||
"Tool calls are not constrained for tool_choice=auto with "
|
||||
"structured outputs because structural tags are unavailable; "
|
||||
"the structured output constraint applies.",
|
||||
scope="local",
|
||||
)
|
||||
return request
|
||||
|
||||
structural_tag = json.dumps(structure_tag.model_dump())
|
||||
structural_tag = tools_structural_tag
|
||||
if output_format is not None:
|
||||
structural_tag = StructuralTag(
|
||||
format=OrFormat(elements=[tools_structural_tag.format, output_format])
|
||||
)
|
||||
elif structured_outputs is not None and not is_auto:
|
||||
logger.warning_once(
|
||||
"structured outputs are ignored because tool_choice forces tool call.",
|
||||
scope="local",
|
||||
)
|
||||
|
||||
request.structured_outputs = StructuredOutputsParams(
|
||||
structural_tag=structural_tag,
|
||||
structural_tag=json.dumps(structural_tag.model_dump()),
|
||||
)
|
||||
if isinstance(request, ResponsesRequest):
|
||||
request.text = None
|
||||
|
||||
@@ -563,6 +563,15 @@ class CohereCommandParser(DelegatingParser):
|
||||
request.response_format = None
|
||||
return request
|
||||
|
||||
def _apply_structural_tag(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
# Note(arpera):
|
||||
# No need to go through _apply_structural_tag in abstract_parser.py
|
||||
# since we have already go through _apply_structural_tags here
|
||||
# So, override this method to return request as is
|
||||
return request
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
count = 0
|
||||
depth = 0
|
||||
|
||||
+2
-44
@@ -16,14 +16,11 @@ from xgrammar.structural_tag import (
|
||||
AnyTextFormat,
|
||||
ConstStringFormat,
|
||||
Format,
|
||||
GrammarFormat,
|
||||
JSONSchemaFormat,
|
||||
OptionalFormat,
|
||||
OrFormat,
|
||||
RegexFormat,
|
||||
SequenceFormat,
|
||||
TagFormat,
|
||||
TriggeredTagsFormat,
|
||||
)
|
||||
|
||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||
@@ -44,9 +41,8 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.parser.abstract_parser import DelegatingParser, structured_outputs_to_format
|
||||
from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser
|
||||
from vllm.tool_parsers.structural_tag_registry import (
|
||||
SimplifiedToolChoice,
|
||||
@@ -428,7 +424,6 @@ _FUNCTION_CALL_BEGINS = [
|
||||
" to=functions.{name}{channel}{constrain}<|message|>",
|
||||
"{channel} to=functions.{name}{constrain}<|message|>",
|
||||
]
|
||||
_JSON_CONTENT = JSONSchemaFormat(json_schema={"type": "object"})
|
||||
_ANY_CONTENT = AnyTextFormat()
|
||||
|
||||
|
||||
@@ -531,43 +526,6 @@ def get_harmony_structural_tag(
|
||||
)
|
||||
|
||||
|
||||
def _params_to_final_content(params: StructuredOutputsParams) -> Format | None:
|
||||
"""Map StructuredOutputsParams in a XGrammar Format."""
|
||||
if params.json_object:
|
||||
return _JSON_CONTENT
|
||||
if params.json is not None:
|
||||
schema = params.json
|
||||
if isinstance(schema, str):
|
||||
schema = json.loads(schema)
|
||||
return JSONSchemaFormat(json_schema=schema)
|
||||
if params.regex is not None:
|
||||
return RegexFormat(pattern=params.regex)
|
||||
if params.choice is not None:
|
||||
return OrFormat(
|
||||
elements=[ConstStringFormat(value=choice) for choice in params.choice]
|
||||
)
|
||||
if params.grammar is not None:
|
||||
return GrammarFormat(grammar=params.grammar)
|
||||
if params.structural_tag is not None:
|
||||
s_tag = json.loads(params.structural_tag)
|
||||
if "structures" in s_tag:
|
||||
# LegacyStructuralTagResponseFormat
|
||||
return TriggeredTagsFormat(
|
||||
triggers=s_tag["triggers"],
|
||||
tags=[
|
||||
TagFormat(
|
||||
begin=structure["begin"],
|
||||
content=JSONSchemaFormat(json_schema=structure["schema"]),
|
||||
end=structure["end"],
|
||||
)
|
||||
for structure in s_tag["structures"]
|
||||
],
|
||||
)
|
||||
# StructuralTagResponseFormat
|
||||
return StructuralTag.model_validate(s_tag).format
|
||||
return None
|
||||
|
||||
|
||||
def _adjust_output_format(
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
@@ -576,7 +534,7 @@ def _adjust_output_format(
|
||||
if params is None:
|
||||
return request
|
||||
|
||||
final_content = _params_to_final_content(params)
|
||||
final_content = structured_outputs_to_format(params)
|
||||
if final_content is None:
|
||||
return request
|
||||
|
||||
|
||||
@@ -121,7 +121,7 @@ def _with_tool_strict(
|
||||
return tool
|
||||
|
||||
|
||||
def _resolve_tool_strictness(
|
||||
def resolve_tool_strictness(
|
||||
tools: Sequence[ChatCompletionToolsParam | ResponsesTool],
|
||||
tool_choice: ToolChoice,
|
||||
strict_level: ToolStrictLevel,
|
||||
@@ -158,7 +158,7 @@ def get_model_structural_tag(
|
||||
if not tools or tool_choice == "none":
|
||||
return None
|
||||
|
||||
tools = _resolve_tool_strictness(tools, tool_choice, strict_level)
|
||||
tools = resolve_tool_strictness(tools, tool_choice, strict_level)
|
||||
if tools is None:
|
||||
return None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user