iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Bugfix][Structured Output] Flag multi-branch allOf as unsupported for xgrammar (#59061)

Browse Source 
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
Co-authored-by: Shashank Varma <324153016+shashankvarma499@users.noreply.github.com>
This commit is contained in:
Artem Perevedentsev
2026-10-06 17:01:04 +04:00
committed by GitHub
co-authored by Shashank Varma
parent 1a86733c7b
commit 049507aa76
2 changed files with 184 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/v1/structured_output/test_utils.py → tests/v1/structured_output/test_backend_xgrammar.py
+159 -20
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from xgrammar import Grammar
from xgrammar.testing import _is_grammar_accept_string
@@ -19,6 +20,11 @@ def grammar_accepts(schema: dict, text: str) -> bool:
return _is_grammar_accept_string(Grammar.from_json_schema(schema), text)
# ================================================
# Unsupported schemas
# ================================================
@pytest.fixture
def unsupported_string_schemas():
return [
@@ -35,16 +41,12 @@ def unsupported_string_schemas():
@pytest.fixture
def unsupported_integer_schemas():
return [
{"type": "integer", "multipleOf": 120},
]
@pytest.fixture
def unsupported_number_schemas():
def unsupported_multipleOf_schemas():
return [
{"type": "number", "multipleOf": 120},
{"Even": {"type": "number", "multipleOf": 2}},
{"type": "integer", "multipleOf": 120},
{"Even": {"type": "integer", "multipleOf": 2}},
]
@@ -140,6 +142,81 @@ def unsupported_pattern_properties_combinations():
]
@pytest.fixture
def unsupported_multibranch_allof():
"""Regression tests for issue #56556
[Bug]: JSON schema with multiple allOf branches is silently ignored
by the xgrammar structured-output backend #56556
Xgrammar currently does NOT support multi-branch allOf.
Bug reported to xgrammar team for tracking progress on resolving:
https://github.com/mlc-ai/xgrammar/issues/937
"""
return [
{
"allOf": [
{"type": "string"},
{"enum": ["yes", "no"]},
]
},
# Multi-branch test case with > 2 branches:
{
"allOf": [
{"type": "string"},
{"minLength": 2},
{"maxLength": 10},
]
},
{
"allOf": [
{"type": "integer"},
{"minimum": 0},
]
},
# Non root multi-branch allOf:
{
"type": "object",
"properties": {
"is_this_sparta": {
"allOf": [
{"type": "string"},
{"enum": ["yes", "no"]},
],
},
},
},
]
@pytest.fixture
def unsupported_vllm_issue_56556_schema():
"""Bug repro from vLLM issue #56556"""
return [
{
"$defs": {
"Base": {
"type": "object",
"properties": {"x": {"type": "integer", "minimum": 10}},
"required": ["x"],
}
},
"allOf": [
{"$ref": "#/$defs/Base"},
{
"type": "object",
"properties": {"y": {"type": "string"}},
"required": ["y"],
},
],
},
]
# ================================================
# Supported schemas
# ================================================
@pytest.fixture
def supported_frankenstein_schema():
# IMPORTANT(arpera):
@@ -177,11 +254,6 @@ def supported_frankenstein_schema():
}
@pytest.fixture
def property_names_schema():
return {"type": "object", "propertyNames": {"pattern": "^[a-z_]+$"}}
@pytest.fixture
def pattern_properties_schema():
return {
@@ -190,19 +262,80 @@ def pattern_properties_schema():
}
@pytest.fixture
def property_names_schema():
return {"type": "object", "propertyNames": {"pattern": "^[a-z_]+$"}}
@pytest.fixture
def supported_allof_anyof_and_oneof():
"""Xgrammar DO support some variants of anyOf, allOf, oneOf
This test case was made during working on covering
unsupported multi-branch allOf to make sure the bug
is NOT present in other combinators.
See for more context in this file test `unsupported_multibranch_allof`
"""
return [
# single-branch allOf:
{
"allOf": [
{"type": "string"},
],
},
# Corner cases: empty allOf, this is a valid schema
{"allOf": []},
# multi-branch allOf:
# is NOT supported yet, see vLLM issue #56556
# single-branch anyOf:
{
"anyOf": [],
},
# multi-branch anyOf:
{
"anyOf": [
{"type": "string"},
{"type": "integer"},
],
},
# single-branch oneOf:
{
"oneOf": [
{"type": "integer", "minimum": 0, "maximum": 100},
],
},
# multi-branch oneOf:
{
"oneOf": [
{"type": "integer", "minimum": 0, "maximum": 100},
{"type": "string", "enum": ["auto", "none"]},
],
},
# Corner case:
# "allOf" is a property name, not allOf combinator keyword
{"type": "object", "properties": {"allOf": {"type": "string"}}},
]
# ================================================
# Test has_xgrammar_unsupported_json_features functionality
# ================================================
class TestHasXGrammarUnsupportedJsonFeatures:
@pytest.mark.parametrize(
"schema_type",
[
"unsupported_string_schemas",
"unsupported_integer_schemas",
"unsupported_number_schemas",
"unsupported_multipleOf_schemas",
"unsupported_array_schemas",
"unsupported_property_names_combinations",
"unsupported_pattern_properties_combinations",
# vLLM issue #56556
"unsupported_multibranch_allof",
"unsupported_vllm_issue_56556_schema",
],
)
def test_unsupported_json_features_by_type(self, schema_type, request):
def test_unsupported_json_features(self, schema_type, request):
schemas = request.getfixturevalue(schema_type)
for schema in schemas:
assert has_xgrammar_unsupported_json_features(schema), (
@@ -215,13 +348,19 @@ class TestHasXGrammarUnsupportedJsonFeatures:
"supported_frankenstein_schema",
"pattern_properties_schema",
"property_names_schema",
# Additional test cases implemented during work on
# vLLM issue #56556
"supported_allof_anyof_and_oneof",
],
)
def test_supported_json_features(self, schema_type, request):
schema = request.getfixturevalue(schema_type)
assert not has_xgrammar_unsupported_json_features(schema), (
f"Schema should be supported: {schema}"
)
schemas = request.getfixturevalue(schema_type)
if not isinstance(schemas, list):
schemas = [schemas]
for schema in schemas:
assert not has_xgrammar_unsupported_json_features(schema), (
f"Schema should be supported: {schema}"
)
class TestPR48416Regressions:
@pytest.mark.parametrize(
vllm/v1/structured_output/backend_xgrammar.py
+25 -5
View File
@@ -335,6 +335,24 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool:
):
return True
# Note(arpera):
# Xgrammar lacks support of multi-branch allOf
# For instance, this schema:
# {
# "allOf": [
# { "type": "string" },
# { "enum": ["yes", "no"] }
# ]
# }
# would accept any kind of json, such as
# "maybe", "", 42, {}, [], {"a": 1}, etc.
# which is NOT what is expected.
# Reported this issue to xgrammar team to track progress on resolving:
# https://github.com/mlc-ai/xgrammar/issues/937
allof = obj.get("allOf")
if isinstance(allof, list) and len(allof) >= 2:
return True
# Recursively check all nested objects and arrays
for value in obj.values():
if isinstance(value, dict):
@@ -401,13 +419,15 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
else:
schema = so_params.json
if has_xgrammar_unsupported_json_features(schema):
raise VLLMValidationError(
"The provided JSON schema contains features not supported by xgrammar."
)
try:
if has_xgrammar_unsupported_json_features(schema):
raise VLLMValidationError(
"The provided JSON schema contains features not supported "
"by xgrammar."
)
xgr.Grammar.from_json_schema(schema)
except VLLMValidationError:
raise
except Exception as err:
raise VLLMValidationError(
f"Failed to transform json schema into a grammar: {err}"