[Bugfix][Structured Output] Flag multi-branch allOf as unsupported for xgrammar (#59061)
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com> Co-authored-by: Shashank Varma <324153016+shashankvarma499@users.noreply.github.com>
This commit is contained in:
co-authored by
Shashank Varma
parent
1a86733c7b
commit
049507aa76
@@ -0,0 +1,464 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
from xgrammar import Grammar
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.v1.structured_output.backend_xgrammar import (
|
||||
has_xgrammar_unsupported_json_features,
|
||||
validate_xgrammar_grammar,
|
||||
)
|
||||
from vllm.v1.structured_output.utils import choice_as_grammar
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def grammar_accepts(schema: dict, text: str) -> bool:
|
||||
return _is_grammar_accept_string(Grammar.from_json_schema(schema), text)
|
||||
|
||||
|
||||
# ================================================
|
||||
# Unsupported schemas
|
||||
# ================================================
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_string_schemas():
|
||||
return [
|
||||
{"type": "string", "format": "non_existing_format"},
|
||||
# TODO(arpera):
|
||||
# pattern/format is compiled but length bounds are silently dropped,
|
||||
# so the combination must be rejected instead of producing quietly
|
||||
# wrong output
|
||||
# https://github.com/mlc-ai/xgrammar/issues/749
|
||||
{"type": "string", "pattern": "^a+$", "maxLength": 2},
|
||||
{"type": "string", "pattern": "^a+$", "minLength": 3},
|
||||
{"type": "string", "format": "email", "maxLength": 10},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_multipleOf_schemas():
|
||||
return [
|
||||
{"type": "number", "multipleOf": 120},
|
||||
{"Even": {"type": "number", "multipleOf": 2}},
|
||||
{"type": "integer", "multipleOf": 120},
|
||||
{"Even": {"type": "integer", "multipleOf": 2}},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_array_schemas():
|
||||
return [
|
||||
{"type": "array", "uniqueItems": True},
|
||||
{"type": "array", "contains": {"type": "string"}},
|
||||
{"type": "array", "minContains": 1},
|
||||
{"type": "array", "maxContains": 5},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_property_names_combinations():
|
||||
return [
|
||||
# TODO(arpera):
|
||||
# this case is not covered by xgrammar, so we should report this bug to xgrammar
|
||||
# xgrammar drops propertyNames whenever patternProperties is present,
|
||||
# so "grade_12" matches the pattern and escapes the name constraint
|
||||
{
|
||||
"type": "object",
|
||||
"patternProperties": {"^grade_[0-9]+$": {"type": "integer"}},
|
||||
"propertyNames": {"pattern": "^grade_[0-9]$"}, # does NOT match "grade_12"
|
||||
},
|
||||
# propertyNames makes xgrammar discard the sibling additionalProperties
|
||||
# value schema: https://github.com/mlc-ai/xgrammar/issues/826
|
||||
{
|
||||
"type": "object",
|
||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||
"additionalProperties": {"type": "integer"},
|
||||
},
|
||||
# propertyNames is a string schema that conventionally omits "type", so
|
||||
# it escapes the string check while xgrammar drops its length bound all
|
||||
# the same: https://github.com/mlc-ai/xgrammar/issues/749
|
||||
{
|
||||
"type": "object",
|
||||
"propertyNames": {"pattern": "^a+$", "maxLength": 2},
|
||||
},
|
||||
# xgrammar emits named `properties` separately from `propertyNames` and
|
||||
# only applies `propertyNames` in its additional-properties branch, so a
|
||||
# key declared in `properties` escapes the name constraint regardless of
|
||||
# what `propertyNames` contains.
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"Bad": {"type": "integer"}},
|
||||
"required": ["Bad"],
|
||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"Bad": {"type": "integer"}},
|
||||
"propertyNames": {"enum": ["good"]},
|
||||
},
|
||||
# xgrammar's propertyNames-only branch falls back to an unconstrained
|
||||
# value type, so it drops unevaluatedProperties whether it restricts
|
||||
# values (a schema) or forbids them (false).
|
||||
{
|
||||
"type": "object",
|
||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||
"unevaluatedProperties": {"type": "integer"},
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||
"unevaluatedProperties": False,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_pattern_properties_combinations():
|
||||
return [
|
||||
# JSON Schema requires a property matched by both `properties` and
|
||||
# `patternProperties` to satisfy both (conjunction), but xgrammar
|
||||
# compiles them as alternative branches, so satisfying either one is
|
||||
# enough.
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"x": {"type": "string"}},
|
||||
"patternProperties": {"^x$": {"type": "integer"}},
|
||||
},
|
||||
# The same alternative-branches problem applies to overlapping
|
||||
# patternProperties patterns: a key matching both patterns only has to
|
||||
# satisfy one of them.
|
||||
{
|
||||
"type": "object",
|
||||
"patternProperties": {
|
||||
"^a[a-z]*$": {"type": "string"},
|
||||
"^[a-z]*z$": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_multibranch_allof():
|
||||
"""Regression tests for issue #56556
|
||||
[Bug]: JSON schema with multiple allOf branches is silently ignored
|
||||
by the xgrammar structured-output backend #56556
|
||||
|
||||
Xgrammar currently does NOT support multi-branch allOf.
|
||||
Bug reported to xgrammar team for tracking progress on resolving:
|
||||
https://github.com/mlc-ai/xgrammar/issues/937
|
||||
"""
|
||||
return [
|
||||
{
|
||||
"allOf": [
|
||||
{"type": "string"},
|
||||
{"enum": ["yes", "no"]},
|
||||
]
|
||||
},
|
||||
# Multi-branch test case with > 2 branches:
|
||||
{
|
||||
"allOf": [
|
||||
{"type": "string"},
|
||||
{"minLength": 2},
|
||||
{"maxLength": 10},
|
||||
]
|
||||
},
|
||||
{
|
||||
"allOf": [
|
||||
{"type": "integer"},
|
||||
{"minimum": 0},
|
||||
]
|
||||
},
|
||||
# Non root multi-branch allOf:
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"is_this_sparta": {
|
||||
"allOf": [
|
||||
{"type": "string"},
|
||||
{"enum": ["yes", "no"]},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_vllm_issue_56556_schema():
|
||||
"""Bug repro from vLLM issue #56556"""
|
||||
return [
|
||||
{
|
||||
"$defs": {
|
||||
"Base": {
|
||||
"type": "object",
|
||||
"properties": {"x": {"type": "integer", "minimum": 10}},
|
||||
"required": ["x"],
|
||||
}
|
||||
},
|
||||
"allOf": [
|
||||
{"$ref": "#/$defs/Base"},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"y": {"type": "string"}},
|
||||
"required": ["y"],
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
# ================================================
|
||||
# Supported schemas
|
||||
# ================================================
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def supported_frankenstein_schema():
|
||||
# IMPORTANT(arpera):
|
||||
# Do NOT add more keywords here! This schema is overcrowded enough that a new
|
||||
# keyword can end up having no effect on the compiled grammar while the
|
||||
# test still passes. Give a new keyword its own fixture and named test.
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"age": {"type": "integer"},
|
||||
"email": {"type": "string", "format": "email"},
|
||||
"status": {"type": "string"},
|
||||
"scores": {"type": "array", "items": {"type": "number"}},
|
||||
"car_type": {"type": "string", "enum": ["sedan", "suv", "truck"]},
|
||||
"car_brand": {"type": "string", "pattern": "^[a-zA-Z]+$"},
|
||||
"short_description": {"type": "string", "maxLength": 50},
|
||||
"mileage": {"type": "number", "minimum": 0, "maximum": 1000000},
|
||||
"model_year": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 1900,
|
||||
"exclusiveMaximum": 2100,
|
||||
},
|
||||
"long_description": {"type": "string", "minLength": 50, "maxLength": 2000},
|
||||
"address": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"street": {"type": "string"},
|
||||
"city": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"minProperties": 1,
|
||||
"maxProperties": 100,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pattern_properties_schema():
|
||||
return {
|
||||
"type": "object",
|
||||
"patternProperties": {"^grade_[0-9]+$": {"type": "integer"}},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def property_names_schema():
|
||||
return {"type": "object", "propertyNames": {"pattern": "^[a-z_]+$"}}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def supported_allof_anyof_and_oneof():
|
||||
"""Xgrammar DO support some variants of anyOf, allOf, oneOf
|
||||
This test case was made during working on covering
|
||||
unsupported multi-branch allOf to make sure the bug
|
||||
is NOT present in other combinators.
|
||||
See for more context in this file test `unsupported_multibranch_allof`
|
||||
"""
|
||||
return [
|
||||
# single-branch allOf:
|
||||
{
|
||||
"allOf": [
|
||||
{"type": "string"},
|
||||
],
|
||||
},
|
||||
# Corner cases: empty allOf, this is a valid schema
|
||||
{"allOf": []},
|
||||
# multi-branch allOf:
|
||||
# is NOT supported yet, see vLLM issue #56556
|
||||
# single-branch anyOf:
|
||||
{
|
||||
"anyOf": [],
|
||||
},
|
||||
# multi-branch anyOf:
|
||||
{
|
||||
"anyOf": [
|
||||
{"type": "string"},
|
||||
{"type": "integer"},
|
||||
],
|
||||
},
|
||||
# single-branch oneOf:
|
||||
{
|
||||
"oneOf": [
|
||||
{"type": "integer", "minimum": 0, "maximum": 100},
|
||||
],
|
||||
},
|
||||
# multi-branch oneOf:
|
||||
{
|
||||
"oneOf": [
|
||||
{"type": "integer", "minimum": 0, "maximum": 100},
|
||||
{"type": "string", "enum": ["auto", "none"]},
|
||||
],
|
||||
},
|
||||
# Corner case:
|
||||
# "allOf" is a property name, not allOf combinator keyword
|
||||
{"type": "object", "properties": {"allOf": {"type": "string"}}},
|
||||
]
|
||||
|
||||
|
||||
# ================================================
|
||||
# Test has_xgrammar_unsupported_json_features functionality
|
||||
# ================================================
|
||||
|
||||
|
||||
class TestHasXGrammarUnsupportedJsonFeatures:
|
||||
@pytest.mark.parametrize(
|
||||
"schema_type",
|
||||
[
|
||||
"unsupported_string_schemas",
|
||||
"unsupported_multipleOf_schemas",
|
||||
"unsupported_array_schemas",
|
||||
"unsupported_property_names_combinations",
|
||||
"unsupported_pattern_properties_combinations",
|
||||
# vLLM issue #56556
|
||||
"unsupported_multibranch_allof",
|
||||
"unsupported_vllm_issue_56556_schema",
|
||||
],
|
||||
)
|
||||
def test_unsupported_json_features(self, schema_type, request):
|
||||
schemas = request.getfixturevalue(schema_type)
|
||||
for schema in schemas:
|
||||
assert has_xgrammar_unsupported_json_features(schema), (
|
||||
f"Schema should be unsupported: {schema}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"schema_type",
|
||||
[
|
||||
"supported_frankenstein_schema",
|
||||
"pattern_properties_schema",
|
||||
"property_names_schema",
|
||||
# Additional test cases implemented during work on
|
||||
# vLLM issue #56556
|
||||
"supported_allof_anyof_and_oneof",
|
||||
],
|
||||
)
|
||||
def test_supported_json_features(self, schema_type, request):
|
||||
schemas = request.getfixturevalue(schema_type)
|
||||
if not isinstance(schemas, list):
|
||||
schemas = [schemas]
|
||||
for schema in schemas:
|
||||
assert not has_xgrammar_unsupported_json_features(schema), (
|
||||
f"Schema should be supported: {schema}"
|
||||
)
|
||||
|
||||
class TestPR48416Regressions:
|
||||
@pytest.mark.parametrize(
|
||||
"schema",
|
||||
[
|
||||
{"type": ["number"], "multipleOf": 3},
|
||||
{
|
||||
"type": ["string", "null"],
|
||||
"pattern": "^a+$",
|
||||
"maxLength": 2,
|
||||
},
|
||||
{
|
||||
"type": ["array", "null"],
|
||||
"items": {"type": "integer"},
|
||||
"uniqueItems": True,
|
||||
},
|
||||
{
|
||||
"type": ["object", "null"],
|
||||
"properties": {"Bad": {"type": "integer"}},
|
||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_list_type_does_not_bypass_unsupported_feature_check(self, schema):
|
||||
assert has_xgrammar_unsupported_json_features(schema)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"schema",
|
||||
[
|
||||
{"type": ["string", "null"]},
|
||||
{"type": ["integer", "string"]},
|
||||
{"type": ["array", "null"], "items": {"type": "string"}},
|
||||
{
|
||||
"type": ["object", "null"],
|
||||
"propertyNames": {"pattern": "^[a-z_]+$"},
|
||||
},
|
||||
{
|
||||
"type": ["object", "null"],
|
||||
"patternProperties": {"^S": {"type": "string"}},
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_supported_list_type_json_features(self, schema):
|
||||
assert not has_xgrammar_unsupported_json_features(schema)
|
||||
|
||||
|
||||
class TestIsGrammarAcceptString:
|
||||
class TestPR42904Support:
|
||||
def test_property_names_constrains_keys(self, property_names_schema):
|
||||
assert grammar_accepts(property_names_schema, '{"score": 5}')
|
||||
assert grammar_accepts(property_names_schema, '{"score": "seven"}')
|
||||
assert not grammar_accepts(property_names_schema, '{"Score": 5}')
|
||||
assert not grammar_accepts(property_names_schema, '{"score_1": 5}')
|
||||
|
||||
def test_pattern_properties_constrains_keys_and_values(
|
||||
self, pattern_properties_schema
|
||||
):
|
||||
assert grammar_accepts(pattern_properties_schema, '{"grade_1": 5}')
|
||||
assert not grammar_accepts(pattern_properties_schema, '{"other": 5}')
|
||||
assert not grammar_accepts(pattern_properties_schema, '{"grade_1": "five"}')
|
||||
|
||||
class TestPR48115Regressions:
|
||||
@pytest.mark.parametrize("codepoint", [*range(0x20), 0x7F])
|
||||
def test_choice_as_grammar_preserves_control_characters(self, codepoint):
|
||||
choice = f"a{chr(codepoint)}f"
|
||||
grammar = Grammar.from_ebnf(choice_as_grammar([choice]))
|
||||
|
||||
assert _is_grammar_accept_string(grammar, choice)
|
||||
assert not _is_grammar_accept_string(grammar, "a")
|
||||
assert not _is_grammar_accept_string(grammar, "af")
|
||||
assert not _is_grammar_accept_string(grammar, f"a\\u{codepoint:04x}f")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("choice", "other"),
|
||||
[
|
||||
(r"a\nf", "a\nf"),
|
||||
('a quote " and a backslash \\', 'a quote " and a backslash '),
|
||||
("café", "cafe"),
|
||||
("日本語", "日本"),
|
||||
("😀", "😁"),
|
||||
],
|
||||
)
|
||||
def test_choice_as_grammar_preserves_literal_choices(self, choice, other):
|
||||
grammar = Grammar.from_ebnf(choice_as_grammar(["yes", choice]))
|
||||
|
||||
assert _is_grammar_accept_string(grammar, "yes")
|
||||
assert _is_grammar_accept_string(grammar, choice)
|
||||
assert not _is_grammar_accept_string(grammar, other)
|
||||
assert not _is_grammar_accept_string(grammar, choice + "extra")
|
||||
|
||||
def test_validate_xgrammar_preserves_multiline_choice(self):
|
||||
structured_outputs = StructuredOutputsParams(choice=["yes", "no\nplease"])
|
||||
validate_xgrammar_grammar(
|
||||
SamplingParams(structured_outputs=structured_outputs)
|
||||
)
|
||||
|
||||
assert structured_outputs.choice is None
|
||||
grammar = Grammar.from_ebnf(structured_outputs.grammar)
|
||||
assert _is_grammar_accept_string(grammar, "yes")
|
||||
assert _is_grammar_accept_string(grammar, "no\nplease")
|
||||
assert not _is_grammar_accept_string(grammar, r"no\nplease")
|
||||
Reference in New Issue
Block a user