[Structured Output] Allow propertyNames with additionalProperties on xgrammar (#57594)
Signed-off-by: errmakov <ide404@gmail.com> Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
co-authored by
Claude Opus 5
Artem Perevedentsev
parent
efaca9547d
commit
f43e4c39e3
@@ -31,21 +31,6 @@ def grammar_accepts(schema: dict, text: str) -> bool:
|
|||||||
# ================================================
|
# ================================================
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def unsupported_string_schemas():
|
|
||||||
return [
|
|
||||||
{"type": "string", "format": "non_existing_format"},
|
|
||||||
# TODO(arpera):
|
|
||||||
# pattern/format is compiled but length bounds are silently dropped,
|
|
||||||
# so the combination must be rejected instead of producing quietly
|
|
||||||
# wrong output
|
|
||||||
# https://github.com/mlc-ai/xgrammar/issues/749
|
|
||||||
{"type": "string", "pattern": "^a+$", "maxLength": 2},
|
|
||||||
{"type": "string", "pattern": "^a+$", "minLength": 3},
|
|
||||||
{"type": "string", "format": "email", "maxLength": 10},
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def unsupported_multipleOf_schemas():
|
def unsupported_multipleOf_schemas():
|
||||||
return [
|
return [
|
||||||
@@ -59,6 +44,8 @@ def unsupported_multipleOf_schemas():
|
|||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def unsupported_array_schemas():
|
def unsupported_array_schemas():
|
||||||
return [
|
return [
|
||||||
|
# array + some constraints is unsupported by xgrammar
|
||||||
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/968
|
||||||
{"type": "array", "uniqueItems": True},
|
{"type": "array", "uniqueItems": True},
|
||||||
{"type": "array", "contains": {"type": "string"}},
|
{"type": "array", "contains": {"type": "string"}},
|
||||||
{"type": "array", "minContains": 1},
|
{"type": "array", "minContains": 1},
|
||||||
@@ -67,74 +54,113 @@ def unsupported_array_schemas():
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def unsupported_property_names_combinations():
|
def unsupported_string_schemas():
|
||||||
return [
|
return [
|
||||||
# TODO(arpera):
|
# ========================================================
|
||||||
# this case is not covered by xgrammar, so we should report this bug to xgrammar
|
# string + format is unsupported by xgrammar
|
||||||
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/967
|
||||||
|
{"type": "string", "format": "non_existing_format"},
|
||||||
|
# ========================================================
|
||||||
|
#
|
||||||
|
# ========================================================
|
||||||
|
# string + format/pattern + length constraint is unsupported by xgrammar
|
||||||
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/966
|
||||||
|
#
|
||||||
|
# pattern/format is compiled but length bounds are silently dropped,
|
||||||
|
# so the combination must be rejected instead of producing quietly
|
||||||
|
# wrong output
|
||||||
|
{"type": "string", "pattern": "^a+$", "maxLength": 2},
|
||||||
|
{"type": "string", "pattern": "^a+$", "minLength": 3},
|
||||||
|
{"type": "string", "format": "email", "maxLength": 10},
|
||||||
|
# ========================================================
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def unsupported_propertyNames_combinations():
|
||||||
|
return [
|
||||||
|
# ========================================================
|
||||||
|
# propertyNames + patternProperties is unsupported by xgrammar
|
||||||
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/959
|
||||||
|
#
|
||||||
# xgrammar drops propertyNames whenever patternProperties is present,
|
# xgrammar drops propertyNames whenever patternProperties is present,
|
||||||
# so "grade_12" matches the pattern and escapes the name constraint
|
# so "grade_12" matches the pattern and escapes the name constraint
|
||||||
{
|
{
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"patternProperties": {"^grade_[0-9]+$": {"type": "integer"}},
|
|
||||||
"propertyNames": {"pattern": "^grade_[0-9]$"}, # does NOT match "grade_12"
|
"propertyNames": {"pattern": "^grade_[0-9]$"}, # does NOT match "grade_12"
|
||||||
|
"patternProperties": {"^grade_[0-9]+$": {"type": "integer"}},
|
||||||
},
|
},
|
||||||
# propertyNames makes xgrammar discard the sibling additionalProperties
|
# ========================================================
|
||||||
# value schema: https://github.com/mlc-ai/xgrammar/issues/826
|
#
|
||||||
{
|
# ========================================================
|
||||||
"type": "object",
|
# propertyNames + maxLength is unsupported by xgrammar
|
||||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/960
|
||||||
"additionalProperties": {"type": "integer"},
|
|
||||||
},
|
|
||||||
# propertyNames is a string schema that conventionally omits "type", so
|
|
||||||
# it escapes the string check while xgrammar drops its length bound all
|
|
||||||
# the same: https://github.com/mlc-ai/xgrammar/issues/749
|
|
||||||
{
|
{
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"propertyNames": {"pattern": "^a+$", "maxLength": 2},
|
"propertyNames": {"pattern": "^a+$", "maxLength": 2},
|
||||||
},
|
},
|
||||||
# xgrammar emits named `properties` separately from `propertyNames` and
|
# ========================================================
|
||||||
# only applies `propertyNames` in its additional-properties branch, so a
|
#
|
||||||
# key declared in `properties` escapes the name constraint regardless of
|
# ========================================================
|
||||||
# what `propertyNames` contains.
|
# propertyNames + properties is unsupported by xgrammar
|
||||||
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/958
|
||||||
{
|
{
|
||||||
"type": "object",
|
"type": "object",
|
||||||
|
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||||
"properties": {"Bad": {"type": "integer"}},
|
"properties": {"Bad": {"type": "integer"}},
|
||||||
"required": ["Bad"],
|
"required": ["Bad"],
|
||||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {"Bad": {"type": "integer"}},
|
|
||||||
"propertyNames": {"enum": ["good"]},
|
"propertyNames": {"enum": ["good"]},
|
||||||
|
"properties": {"Bad": {"type": "integer"}},
|
||||||
},
|
},
|
||||||
|
# ========================================================
|
||||||
|
#
|
||||||
|
# ========================================================
|
||||||
|
# propertyNames + unevaluatedProperties is unsupported by xgrammar
|
||||||
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/961
|
||||||
|
#
|
||||||
# xgrammar's propertyNames-only branch falls back to an unconstrained
|
# xgrammar's propertyNames-only branch falls back to an unconstrained
|
||||||
# value type, so it drops unevaluatedProperties whether it restricts
|
# value type, so it drops unevaluatedProperties whether it restricts
|
||||||
# values (a schema) or forbids them (false).
|
# values (a schema) or forbids them (false).
|
||||||
{
|
{
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||||
"unevaluatedProperties": {"type": "integer"},
|
"unevaluatedProperties": False,
|
||||||
},
|
},
|
||||||
|
# also keep in mind that unevaluatedProperties can be a schema
|
||||||
{
|
{
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||||
"unevaluatedProperties": False,
|
"unevaluatedProperties": {"type": "integer"},
|
||||||
},
|
},
|
||||||
|
# ========================================================
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def unsupported_pattern_properties_combinations():
|
def unsupported_patternProperties_combinations():
|
||||||
return [
|
return [
|
||||||
|
# ========================================================
|
||||||
|
# patternProperties + properties is unsupported by xgrammar
|
||||||
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/964
|
||||||
|
#
|
||||||
# JSON Schema requires a property matched by both `properties` and
|
# JSON Schema requires a property matched by both `properties` and
|
||||||
# `patternProperties` to satisfy both (conjunction), but xgrammar
|
# `patternProperties` to satisfy both (conjunction), but xgrammar
|
||||||
# compiles them as alternative branches, so satisfying either one is
|
# compiles them as alternative branches, so satisfying either one is
|
||||||
# enough.
|
# enough.
|
||||||
{
|
{
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {"x": {"type": "string"}},
|
|
||||||
"patternProperties": {"^x$": {"type": "integer"}},
|
"patternProperties": {"^x$": {"type": "integer"}},
|
||||||
|
"properties": {"x": {"type": "string"}},
|
||||||
},
|
},
|
||||||
|
# ========================================================
|
||||||
|
#
|
||||||
|
# ========================================================
|
||||||
|
# patternProperties + patternProperties is unsupported by xgrammar
|
||||||
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/965
|
||||||
|
#
|
||||||
# The same alternative-branches problem applies to overlapping
|
# The same alternative-branches problem applies to overlapping
|
||||||
# patternProperties patterns: a key matching both patterns only has to
|
# patternProperties patterns: a key matching both patterns only has to
|
||||||
# satisfy one of them.
|
# satisfy one of them.
|
||||||
@@ -145,6 +171,7 @@ def unsupported_pattern_properties_combinations():
|
|||||||
"^[a-z]*z$": {"type": "integer"},
|
"^[a-z]*z$": {"type": "integer"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
# ========================================================
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -273,6 +300,24 @@ def property_names_schema():
|
|||||||
return {"type": "object", "propertyNames": {"pattern": "^[a-z_]+$"}}
|
return {"type": "object", "propertyNames": {"pattern": "^[a-z_]+$"}}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def property_names_with_additional_properties_schema():
|
||||||
|
"""This combination propertyNames + additionalProperties
|
||||||
|
was unsupported some time ago by xgrammar.
|
||||||
|
So, this test case was in unsupported category before.
|
||||||
|
But this xgrammar's PR
|
||||||
|
https://github.com/mlc-ai/xgrammar/pull/836
|
||||||
|
added support of propertyNames + additionalProperties
|
||||||
|
This fix was released in xgrammar v0.2.7
|
||||||
|
which vLLM pins in PR #57272
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"type": "object",
|
||||||
|
"propertyNames": {"pattern": "^[a-z_]+$"},
|
||||||
|
"additionalProperties": {"type": "integer"},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def supported_allof_anyof_and_oneof():
|
def supported_allof_anyof_and_oneof():
|
||||||
"""Xgrammar DO support some variants of anyOf, allOf, oneOf
|
"""Xgrammar DO support some variants of anyOf, allOf, oneOf
|
||||||
@@ -334,8 +379,8 @@ class TestHasXGrammarUnsupportedJsonFeatures:
|
|||||||
"unsupported_string_schemas",
|
"unsupported_string_schemas",
|
||||||
"unsupported_multipleOf_schemas",
|
"unsupported_multipleOf_schemas",
|
||||||
"unsupported_array_schemas",
|
"unsupported_array_schemas",
|
||||||
"unsupported_property_names_combinations",
|
"unsupported_propertyNames_combinations",
|
||||||
"unsupported_pattern_properties_combinations",
|
"unsupported_patternProperties_combinations",
|
||||||
# vLLM issue #56556
|
# vLLM issue #56556
|
||||||
"unsupported_multibranch_allof",
|
"unsupported_multibranch_allof",
|
||||||
"unsupported_vllm_issue_56556_schema",
|
"unsupported_vllm_issue_56556_schema",
|
||||||
@@ -354,6 +399,7 @@ class TestHasXGrammarUnsupportedJsonFeatures:
|
|||||||
"supported_frankenstein_schema",
|
"supported_frankenstein_schema",
|
||||||
"pattern_properties_schema",
|
"pattern_properties_schema",
|
||||||
"property_names_schema",
|
"property_names_schema",
|
||||||
|
"property_names_with_additional_properties_schema",
|
||||||
# Additional test cases implemented during work on
|
# Additional test cases implemented during work on
|
||||||
# vLLM issue #56556
|
# vLLM issue #56556
|
||||||
"supported_allof_anyof_and_oneof",
|
"supported_allof_anyof_and_oneof",
|
||||||
@@ -421,6 +467,14 @@ class TestIsGrammarAcceptString:
|
|||||||
assert not grammar_accepts(property_names_schema, '{"Score": 5}')
|
assert not grammar_accepts(property_names_schema, '{"Score": 5}')
|
||||||
assert not grammar_accepts(property_names_schema, '{"score_1": 5}')
|
assert not grammar_accepts(property_names_schema, '{"score_1": 5}')
|
||||||
|
|
||||||
|
def test_property_names_keeps_additional_properties_value_schema(
|
||||||
|
self, property_names_with_additional_properties_schema
|
||||||
|
):
|
||||||
|
schema = property_names_with_additional_properties_schema
|
||||||
|
assert grammar_accepts(schema, '{"score": 5}')
|
||||||
|
assert not grammar_accepts(schema, '{"Score": 5}')
|
||||||
|
assert not grammar_accepts(schema, '{"score": "seven"}')
|
||||||
|
|
||||||
def test_pattern_properties_constrains_keys_and_values(
|
def test_pattern_properties_constrains_keys_and_values(
|
||||||
self, pattern_properties_schema
|
self, pattern_properties_schema
|
||||||
):
|
):
|
||||||
|
|||||||
@@ -296,18 +296,29 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool:
|
|||||||
|
|
||||||
schema_types = _schema_types(obj)
|
schema_types = _schema_types(obj)
|
||||||
|
|
||||||
# Check for numeric ranges
|
# integer/number + multipleOf is unsupported by xgrammar
|
||||||
|
# This is known behavior and xgrammar emits warning in logs:
|
||||||
|
# [21:18:08] /project/cpp/json_schema_converter.cc:1053:
|
||||||
|
# Warning: multipleOf is not supported for type:number; ignoring multipleOf
|
||||||
|
# This warning was added in PR
|
||||||
|
# https://github.com/mlc-ai/xgrammar/pull/670
|
||||||
|
# So, no need to track progress on this
|
||||||
if (schema_types & {"integer", "number"}) and ("multipleOf" in obj):
|
if (schema_types & {"integer", "number"}) and ("multipleOf" in obj):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
# Check for array unsupported keywords
|
# array + some constraints is unsupported by xgrammar
|
||||||
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/968
|
||||||
if "array" in schema_types and any(
|
if "array" in schema_types and any(
|
||||||
key in obj
|
key in obj
|
||||||
for key in ("uniqueItems", "contains", "minContains", "maxContains")
|
for key in ("uniqueItems", "contains", "minContains", "maxContains")
|
||||||
):
|
):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
# Unsupported keywords for strings
|
# string + format with unsupported keywords
|
||||||
|
# is unsupported by xgrammar
|
||||||
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/967
|
||||||
|
# See tests on this in test_backend_xgrammar.py
|
||||||
|
# unsupported_string_schemas
|
||||||
if (
|
if (
|
||||||
"string" in schema_types
|
"string" in schema_types
|
||||||
and "format" in obj
|
and "format" in obj
|
||||||
@@ -315,48 +326,51 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool:
|
|||||||
):
|
):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
# A string mixing a generative constraint (pattern or format) with
|
# string + format/pattern + length constraint is unsupported by xgrammar
|
||||||
# explicit length bounds. xgrammar compiles the pattern/format side
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/966
|
||||||
# and silently drops minLength/maxLength from the grammar, so output
|
# See tests on this in test_backend_xgrammar.py
|
||||||
# can violate the bound without any error surfacing. Verified against
|
# unsupported_string_schemas
|
||||||
# the compiled EBNF: pattern/format grammars come out byte-identical
|
|
||||||
# with and without the length keywords, while maxLength alone lowers
|
|
||||||
# to {0, N} correctly.
|
|
||||||
if "string" in schema_types and _has_pattern_and_length_bounds(obj):
|
if "string" in schema_types and _has_pattern_and_length_bounds(obj):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
# propertyNames validates names, so it is a string schema even when it
|
# propertyNames is not supported in pair with some constraints
|
||||||
# omits "type", which is the form that escapes the check above.
|
# in xgrammar
|
||||||
if (
|
# See tests on this in test_backend_xgrammar.py
|
||||||
"object" in schema_types
|
# unsupported_propertyNames_combinations
|
||||||
and isinstance(obj.get("propertyNames"), dict)
|
if "object" in schema_types and "propertyNames" in obj:
|
||||||
and _has_pattern_and_length_bounds(obj["propertyNames"])
|
# propertyNames + maxLength is unsupported by xgrammar
|
||||||
):
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/960
|
||||||
return True
|
property_names = obj.get("propertyNames")
|
||||||
|
if isinstance(property_names, dict) and _has_pattern_and_length_bounds(
|
||||||
|
property_names
|
||||||
|
):
|
||||||
|
return True
|
||||||
|
# propertyNames + patternProperties is unsupported by xgrammar
|
||||||
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/959
|
||||||
|
if "patternProperties" in obj:
|
||||||
|
return True
|
||||||
|
# propertyNames + properties is unsupported by xgrammar
|
||||||
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/958
|
||||||
|
if "properties" in obj:
|
||||||
|
return True
|
||||||
|
# propertyNames + unevaluatedProperties is unsupported by xgrammar
|
||||||
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/961
|
||||||
|
if obj.get("unevaluatedProperties", True) is not True:
|
||||||
|
return True
|
||||||
|
|
||||||
# FIXME: propertyNames conflicts with properties/patternProperties/
|
# patternProperties is not supported in pair with some constraints
|
||||||
# additionalProperties/unevaluatedProperties under xgrammar.
|
# in xgrammar
|
||||||
# https://github.com/mlc-ai/xgrammar/issues/826
|
# See tests on this in test_backend_xgrammar.py
|
||||||
if (
|
# unsupported_patternProperties_combinations
|
||||||
"object" in schema_types
|
if "object" in schema_types and isinstance(obj.get("patternProperties"), dict):
|
||||||
and "propertyNames" in obj
|
# patternProperties + properties is unsupported by xgrammar
|
||||||
and (
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/964
|
||||||
"properties" in obj
|
if "properties" in obj:
|
||||||
or "patternProperties" in obj
|
return True
|
||||||
or isinstance(obj.get("additionalProperties"), dict)
|
# patternProperties + patternProperties is unsupported by xgrammar
|
||||||
or obj.get("unevaluatedProperties", True) is not True
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/965
|
||||||
)
|
if len(obj["patternProperties"]) > 1:
|
||||||
):
|
return True
|
||||||
return True
|
|
||||||
|
|
||||||
# FIXME: multiple patternProperties, or patternProperties alongside
|
|
||||||
# properties, conflict under xgrammar.
|
|
||||||
if (
|
|
||||||
"object" in schema_types
|
|
||||||
and isinstance(obj.get("patternProperties"), dict)
|
|
||||||
and ("properties" in obj or len(obj["patternProperties"]) > 1)
|
|
||||||
):
|
|
||||||
return True
|
|
||||||
|
|
||||||
# Note(arpera):
|
# Note(arpera):
|
||||||
# Xgrammar lacks support of multi-branch allOf
|
# Xgrammar lacks support of multi-branch allOf
|
||||||
@@ -370,8 +384,7 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool:
|
|||||||
# would accept any kind of json, such as
|
# would accept any kind of json, such as
|
||||||
# "maybe", "", 42, {}, [], {"a": 1}, etc.
|
# "maybe", "", 42, {}, [], {"a": 1}, etc.
|
||||||
# which is NOT what is expected.
|
# which is NOT what is expected.
|
||||||
# Reported this issue to xgrammar team to track progress on resolving:
|
# Tracking issue: https://github.com/mlc-ai/xgrammar/issues/937
|
||||||
# https://github.com/mlc-ai/xgrammar/issues/937
|
|
||||||
allof = obj.get("allOf")
|
allof = obj.get("allOf")
|
||||||
if isinstance(allof, list) and len(allof) >= 2:
|
if isinstance(allof, list) and len(allof) >= 2:
|
||||||
return True
|
return True
|
||||||
|
|||||||
Reference in New Issue
Block a user