From 049507aa76200090f8bc6ca4e48ab0e09f2f0d29 Mon Sep 17 00:00:00 2001 From: Artem Perevedentsev Date: Tue, 6 Oct 2026 16:01:04 +0300 Subject: [PATCH] [Bugfix][Structured Output] Flag multi-branch allOf as unsupported for xgrammar (#59061) Signed-off-by: Artem Perevedentsev Co-authored-by: Shashank Varma <324153016+shashankvarma499@users.noreply.github.com> --- ...test_utils.py => test_backend_xgrammar.py} | 179 ++++++++++++++++-- vllm/v1/structured_output/backend_xgrammar.py | 30 ++- 2 files changed, 184 insertions(+), 25 deletions(-) rename tests/v1/structured_output/{test_utils.py => test_backend_xgrammar.py} (72%) diff --git a/tests/v1/structured_output/test_utils.py b/tests/v1/structured_output/test_backend_xgrammar.py similarity index 72% rename from tests/v1/structured_output/test_utils.py rename to tests/v1/structured_output/test_backend_xgrammar.py index d3187c428c..339f0de17f 100644 --- a/tests/v1/structured_output/test_utils.py +++ b/tests/v1/structured_output/test_backend_xgrammar.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project + import pytest from xgrammar import Grammar from xgrammar.testing import _is_grammar_accept_string @@ -19,6 +20,11 @@ def grammar_accepts(schema: dict, text: str) -> bool: return _is_grammar_accept_string(Grammar.from_json_schema(schema), text) +# ================================================ +# Unsupported schemas +# ================================================ + + @pytest.fixture def unsupported_string_schemas(): return [ @@ -35,16 +41,12 @@ def unsupported_string_schemas(): @pytest.fixture -def unsupported_integer_schemas(): - return [ - {"type": "integer", "multipleOf": 120}, - ] - - -@pytest.fixture -def unsupported_number_schemas(): +def unsupported_multipleOf_schemas(): return [ {"type": "number", "multipleOf": 120}, + {"Even": {"type": "number", "multipleOf": 2}}, + {"type": "integer", "multipleOf": 120}, + {"Even": {"type": "integer", "multipleOf": 2}}, ] @@ -140,6 +142,81 @@ def unsupported_pattern_properties_combinations(): ] +@pytest.fixture +def unsupported_multibranch_allof(): + """Regression tests for issue #56556 + [Bug]: JSON schema with multiple allOf branches is silently ignored + by the xgrammar structured-output backend #56556 + + Xgrammar currently does NOT support multi-branch allOf. + Bug reported to xgrammar team for tracking progress on resolving: + https://github.com/mlc-ai/xgrammar/issues/937 + """ + return [ + { + "allOf": [ + {"type": "string"}, + {"enum": ["yes", "no"]}, + ] + }, + # Multi-branch test case with > 2 branches: + { + "allOf": [ + {"type": "string"}, + {"minLength": 2}, + {"maxLength": 10}, + ] + }, + { + "allOf": [ + {"type": "integer"}, + {"minimum": 0}, + ] + }, + # Non root multi-branch allOf: + { + "type": "object", + "properties": { + "is_this_sparta": { + "allOf": [ + {"type": "string"}, + {"enum": ["yes", "no"]}, + ], + }, + }, + }, + ] + + +@pytest.fixture +def unsupported_vllm_issue_56556_schema(): + """Bug repro from vLLM issue #56556""" + return [ + { + "$defs": { + "Base": { + "type": "object", + "properties": {"x": {"type": "integer", "minimum": 10}}, + "required": ["x"], + } + }, + "allOf": [ + {"$ref": "#/$defs/Base"}, + { + "type": "object", + "properties": {"y": {"type": "string"}}, + "required": ["y"], + }, + ], + }, + ] + + +# ================================================ +# Supported schemas +# ================================================ + + @pytest.fixture def supported_frankenstein_schema(): # IMPORTANT(arpera): @@ -177,11 +254,6 @@ def supported_frankenstein_schema(): } -@pytest.fixture -def property_names_schema(): - return {"type": "object", "propertyNames": {"pattern": "^[a-z_]+$"}} - - @pytest.fixture def pattern_properties_schema(): return { @@ -190,19 +262,80 @@ def pattern_properties_schema(): } +@pytest.fixture +def property_names_schema(): + return {"type": "object", "propertyNames": {"pattern": "^[a-z_]+$"}} + + +@pytest.fixture +def supported_allof_anyof_and_oneof(): + """Xgrammar DO support some variants of anyOf, allOf, oneOf + This test case was made during working on covering + unsupported multi-branch allOf to make sure the bug + is NOT present in other combinators. + See for more context in this file test `unsupported_multibranch_allof` + """ + return [ + # single-branch allOf: + { + "allOf": [ + {"type": "string"}, + ], + }, + # Corner cases: empty allOf, this is a valid schema + {"allOf": []}, + # multi-branch allOf: + # is NOT supported yet, see vLLM issue #56556 + # single-branch anyOf: + { + "anyOf": [], + }, + # multi-branch anyOf: + { + "anyOf": [ + {"type": "string"}, + {"type": "integer"}, + ], + }, + # single-branch oneOf: + { + "oneOf": [ + {"type": "integer", "minimum": 0, "maximum": 100}, + ], + }, + # multi-branch oneOf: + { + "oneOf": [ + {"type": "integer", "minimum": 0, "maximum": 100}, + {"type": "string", "enum": ["auto", "none"]}, + ], + }, + # Corner case: + # "allOf" is a property name, not allOf combinator keyword + {"type": "object", "properties": {"allOf": {"type": "string"}}}, + ] + + +# ================================================ +# Test has_xgrammar_unsupported_json_features functionality +# ================================================ + + class TestHasXGrammarUnsupportedJsonFeatures: @pytest.mark.parametrize( "schema_type", [ "unsupported_string_schemas", - "unsupported_integer_schemas", - "unsupported_number_schemas", + "unsupported_multipleOf_schemas", "unsupported_array_schemas", "unsupported_property_names_combinations", "unsupported_pattern_properties_combinations", + # vLLM issue #56556 + "unsupported_multibranch_allof", + "unsupported_vllm_issue_56556_schema", ], ) - def test_unsupported_json_features_by_type(self, schema_type, request): + def test_unsupported_json_features(self, schema_type, request): schemas = request.getfixturevalue(schema_type) for schema in schemas: assert has_xgrammar_unsupported_json_features(schema), ( @@ -215,13 +348,19 @@ class TestHasXGrammarUnsupportedJsonFeatures: "supported_frankenstein_schema", "pattern_properties_schema", "property_names_schema", + # Additional test cases implemented during work on + # vLLM issue #56556 + "supported_allof_anyof_and_oneof", ], ) def test_supported_json_features(self, schema_type, request): - schema = request.getfixturevalue(schema_type) - assert not has_xgrammar_unsupported_json_features(schema), ( - f"Schema should be supported: {schema}" - ) + schemas = request.getfixturevalue(schema_type) + if not isinstance(schemas, list): + schemas = [schemas] + for schema in schemas: + assert not has_xgrammar_unsupported_json_features(schema), ( + f"Schema should be supported: {schema}" + ) class TestPR48416Regressions: @pytest.mark.parametrize( diff --git a/vllm/v1/structured_output/backend_xgrammar.py b/vllm/v1/structured_output/backend_xgrammar.py index 4697b00487..733ed7db09 100644 --- a/vllm/v1/structured_output/backend_xgrammar.py +++ b/vllm/v1/structured_output/backend_xgrammar.py @@ -335,6 +335,24 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool: ): return True + # Note(arpera): + # Xgrammar lacks support of multi-branch allOf + # For instance, this schema: + # { + # "allOf": [ + # { "type": "string" }, + # { "enum": ["yes", "no"] } + # ] + # } + # would accept any kind of json, such as + # "maybe", "", 42, {}, [], {"a": 1}, etc. + # which is NOT what is expected. + # Reported this issue to xgrammar team to track progress on resolving: + # https://github.com/mlc-ai/xgrammar/issues/937 + allof = obj.get("allOf") + if isinstance(allof, list) and len(allof) >= 2: + return True + # Recursively check all nested objects and arrays for value in obj.values(): if isinstance(value, dict): @@ -401,13 +419,15 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: else: schema = so_params.json - if has_xgrammar_unsupported_json_features(schema): - raise VLLMValidationError( - "The provided JSON schema contains features not supported by xgrammar." - ) - try: + if has_xgrammar_unsupported_json_features(schema): + raise VLLMValidationError( + "The provided JSON schema contains features not supported " + "by xgrammar." + ) xgr.Grammar.from_json_schema(schema) + except VLLMValidationError: + raise except Exception as err: raise VLLMValidationError( f"Failed to transform json schema into a grammar: {err}"