[feature] [xgrammar] support patternProperties/propertyNames/unevaluatedProperties kw for object types (#42904)
Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com> Co-authored-by: Artem Perevedentsev <aperevedents@nvidia.com> Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Artem Perevedentsev
Claude Sonnet 5
parent
7b7e117c29
commit
cd10ed6f9f
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from xgrammar import Grammar
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.v1.structured_output.backend_xgrammar import (
|
||||
has_xgrammar_unsupported_json_features,
|
||||
@@ -10,13 +12,19 @@ from vllm.v1.structured_output.backend_xgrammar import (
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def grammar_accepts(schema: dict, text: str) -> bool:
|
||||
return _is_grammar_accept_string(Grammar.from_json_schema(schema), text)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_string_schemas():
|
||||
return [
|
||||
{"type": "string", "format": "non_existing_format"},
|
||||
# TODO(arpera):
|
||||
# pattern/format is compiled but length bounds are silently dropped,
|
||||
# so the combination must be rejected instead of producing quietly
|
||||
# wrong output
|
||||
# https://github.com/mlc-ai/xgrammar/issues/749
|
||||
{"type": "string", "pattern": "^a+$", "maxLength": 2},
|
||||
{"type": "string", "pattern": "^a+$", "minLength": 3},
|
||||
{"type": "string", "format": "email", "maxLength": 10},
|
||||
@@ -48,15 +56,93 @@ def unsupported_array_schemas():
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unsupported_object_schemas():
|
||||
def unsupported_property_names_combinations():
|
||||
return [
|
||||
{"type": "object", "propertyNames": {"pattern": "^[a-z]+$"}},
|
||||
{"type": "object", "patternProperties": {"^S": {"type": "string"}}},
|
||||
# TODO(arpera):
|
||||
# this case is not covered by xgrammar, so we should report this bug to xgrammar
|
||||
# xgrammar drops propertyNames whenever patternProperties is present,
|
||||
# so "grade_12" matches the pattern and escapes the name constraint
|
||||
{
|
||||
"type": "object",
|
||||
"patternProperties": {"^grade_[0-9]+$": {"type": "integer"}},
|
||||
"propertyNames": {"pattern": "^grade_[0-9]$"}, # does NOT match "grade_12"
|
||||
},
|
||||
# propertyNames makes xgrammar discard the sibling additionalProperties
|
||||
# value schema: https://github.com/mlc-ai/xgrammar/issues/826
|
||||
{
|
||||
"type": "object",
|
||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||
"additionalProperties": {"type": "integer"},
|
||||
},
|
||||
# propertyNames is a string schema that conventionally omits "type", so
|
||||
# it escapes the string check while xgrammar drops its length bound all
|
||||
# the same: https://github.com/mlc-ai/xgrammar/issues/749
|
||||
{
|
||||
"type": "object",
|
||||
"propertyNames": {"pattern": "^a+$", "maxLength": 2},
|
||||
},
|
||||
# xgrammar emits named `properties` separately from `propertyNames` and
|
||||
# only applies `propertyNames` in its additional-properties branch, so a
|
||||
# key declared in `properties` escapes the name constraint regardless of
|
||||
# what `propertyNames` contains.
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"Bad": {"type": "integer"}},
|
||||
"required": ["Bad"],
|
||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"Bad": {"type": "integer"}},
|
||||
"propertyNames": {"enum": ["good"]},
|
||||
},
|
||||
# xgrammar's propertyNames-only branch falls back to an unconstrained
|
||||
# value type, so it drops unevaluatedProperties whether it restricts
|
||||
# values (a schema) or forbids them (false).
|
||||
{
|
||||
"type": "object",
|
||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||
"unevaluatedProperties": {"type": "integer"},
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"propertyNames": {"pattern": "^[a-z]+$"},
|
||||
"unevaluatedProperties": False,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def supported_schema():
|
||||
def unsupported_pattern_properties_combinations():
|
||||
return [
|
||||
# JSON Schema requires a property matched by both `properties` and
|
||||
# `patternProperties` to satisfy both (conjunction), but xgrammar
|
||||
# compiles them as alternative branches, so satisfying either one is
|
||||
# enough.
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"x": {"type": "string"}},
|
||||
"patternProperties": {"^x$": {"type": "integer"}},
|
||||
},
|
||||
# The same alternative-branches problem applies to overlapping
|
||||
# patternProperties patterns: a key matching both patterns only has to
|
||||
# satisfy one of them.
|
||||
{
|
||||
"type": "object",
|
||||
"patternProperties": {
|
||||
"^a[a-z]*$": {"type": "string"},
|
||||
"^[a-z]*z$": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def supported_frankenstein_schema():
|
||||
# IMPORTANT(arpera):
|
||||
# Do NOT add more keywords here! This schema is overcrowded enough that a new
|
||||
# keyword can end up having no effect on the compiled grammar while the
|
||||
# test still passes. Give a new keyword its own fixture and named test.
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -88,6 +174,19 @@ def supported_schema():
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def property_names_schema():
|
||||
return {"type": "object", "propertyNames": {"pattern": "^[a-z_]+$"}}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pattern_properties_schema():
|
||||
return {
|
||||
"type": "object",
|
||||
"patternProperties": {"^grade_[0-9]+$": {"type": "integer"}},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"schema_type",
|
||||
[
|
||||
@@ -95,7 +194,8 @@ def supported_schema():
|
||||
"unsupported_integer_schemas",
|
||||
"unsupported_number_schemas",
|
||||
"unsupported_array_schemas",
|
||||
"unsupported_object_schemas",
|
||||
"unsupported_property_names_combinations",
|
||||
"unsupported_pattern_properties_combinations",
|
||||
],
|
||||
)
|
||||
def test_unsupported_json_features_by_type(schema_type, request):
|
||||
@@ -106,7 +206,29 @@ def test_unsupported_json_features_by_type(schema_type, request):
|
||||
)
|
||||
|
||||
|
||||
def test_supported_json_features(supported_schema):
|
||||
assert not has_xgrammar_unsupported_json_features(supported_schema), (
|
||||
"Schema should be supported"
|
||||
@pytest.mark.parametrize(
|
||||
"schema_type",
|
||||
[
|
||||
"supported_frankenstein_schema",
|
||||
"pattern_properties_schema",
|
||||
"property_names_schema",
|
||||
],
|
||||
)
|
||||
def test_supported_json_features(schema_type, request):
|
||||
schema = request.getfixturevalue(schema_type)
|
||||
assert not has_xgrammar_unsupported_json_features(schema), (
|
||||
f"Schema should be supported: {schema}"
|
||||
)
|
||||
|
||||
|
||||
def test_property_names_constrains_keys(property_names_schema):
|
||||
assert grammar_accepts(property_names_schema, '{"score": 5}')
|
||||
assert grammar_accepts(property_names_schema, '{"score": "seven"}')
|
||||
assert not grammar_accepts(property_names_schema, '{"Score": 5}')
|
||||
assert not grammar_accepts(property_names_schema, '{"score_1": 5}')
|
||||
|
||||
|
||||
def test_pattern_properties_constrains_keys_and_values(pattern_properties_schema):
|
||||
assert grammar_accepts(pattern_properties_schema, '{"grade_1": 5}')
|
||||
assert not grammar_accepts(pattern_properties_schema, '{"other": 5}')
|
||||
assert not grammar_accepts(pattern_properties_schema, '{"grade_1": "five"}')
|
||||
|
||||
@@ -137,7 +137,7 @@ def test_unsupported_grammar_is_a_client_error(backend, structured_outputs):
|
||||
@pytest.mark.parametrize(
|
||||
"schema, expected_backend",
|
||||
[
|
||||
# multipleOf is unsupported by xgrammar, patternProperties also by guidance.
|
||||
# multipleOf is unsupported by xgrammar.
|
||||
(
|
||||
{
|
||||
"type": "object",
|
||||
@@ -145,8 +145,13 @@ def test_unsupported_grammar_is_a_client_error(backend, structured_outputs):
|
||||
},
|
||||
"guidance",
|
||||
),
|
||||
# patternProperties + properties is also unsupported by guidance.
|
||||
(
|
||||
{"type": "object", "patternProperties": {"^a": {"type": "string"}}},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"a": {"type": "string"}},
|
||||
"patternProperties": {"^a$": {"type": "string"}},
|
||||
},
|
||||
"outlines",
|
||||
),
|
||||
],
|
||||
|
||||
@@ -236,6 +236,12 @@ STRING_SUPPORTED_FORMATS = {
|
||||
}
|
||||
|
||||
|
||||
def _has_pattern_and_length_bounds(schema: dict[str, Any]) -> bool:
|
||||
return ("pattern" in schema or "format" in schema) and (
|
||||
"minLength" in schema or "maxLength" in schema
|
||||
)
|
||||
|
||||
|
||||
def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool:
|
||||
"""Check if JSON schema contains features unsupported by xgrammar."""
|
||||
|
||||
@@ -269,16 +275,39 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool:
|
||||
# the compiled EBNF: pattern/format grammars come out byte-identical
|
||||
# with and without the length keywords, while maxLength alone lowers
|
||||
# to {0, N} correctly.
|
||||
if obj.get("type") == "string" and _has_pattern_and_length_bounds(obj):
|
||||
return True
|
||||
|
||||
# propertyNames validates names, so it is a string schema even when it
|
||||
# omits "type", which is the form that escapes the check above.
|
||||
if (
|
||||
obj.get("type") == "string"
|
||||
and ("pattern" in obj or "format" in obj)
|
||||
and ("minLength" in obj or "maxLength" in obj)
|
||||
obj.get("type") == "object"
|
||||
and isinstance(obj.get("propertyNames"), dict)
|
||||
and _has_pattern_and_length_bounds(obj["propertyNames"])
|
||||
):
|
||||
return True
|
||||
|
||||
# Unsupported keywords for objects
|
||||
if obj.get("type") == "object" and any(
|
||||
key in obj for key in ("patternProperties", "propertyNames")
|
||||
# FIXME: propertyNames conflicts with properties/patternProperties/
|
||||
# additionalProperties/unevaluatedProperties under xgrammar.
|
||||
# https://github.com/mlc-ai/xgrammar/issues/826
|
||||
if (
|
||||
obj.get("type") == "object"
|
||||
and "propertyNames" in obj
|
||||
and (
|
||||
"properties" in obj
|
||||
or "patternProperties" in obj
|
||||
or isinstance(obj.get("additionalProperties"), dict)
|
||||
or obj.get("unevaluatedProperties", True) is not True
|
||||
)
|
||||
):
|
||||
return True
|
||||
|
||||
# FIXME: multiple patternProperties, or patternProperties alongside
|
||||
# properties, conflict under xgrammar.
|
||||
if (
|
||||
obj.get("type") == "object"
|
||||
and isinstance(obj.get("patternProperties"), dict)
|
||||
and ("properties" in obj or len(obj["patternProperties"]) > 1)
|
||||
):
|
||||
return True
|
||||
|
||||
|
||||
Reference in New Issue
Block a user