diff --git a/tests/v1/structured_output/test_utils.py b/tests/v1/structured_output/test_utils.py index 11b3e5c01b..f5f8020220 100644 --- a/tests/v1/structured_output/test_utils.py +++ b/tests/v1/structured_output/test_utils.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest +from xgrammar import Grammar +from xgrammar.testing import _is_grammar_accept_string from vllm.v1.structured_output.backend_xgrammar import ( has_xgrammar_unsupported_json_features, @@ -10,13 +12,19 @@ from vllm.v1.structured_output.backend_xgrammar import ( pytestmark = pytest.mark.cpu_test +def grammar_accepts(schema: dict, text: str) -> bool: + return _is_grammar_accept_string(Grammar.from_json_schema(schema), text) + + @pytest.fixture def unsupported_string_schemas(): return [ {"type": "string", "format": "non_existing_format"}, + # TODO(arpera): # pattern/format is compiled but length bounds are silently dropped, # so the combination must be rejected instead of producing quietly # wrong output + # https://github.com/mlc-ai/xgrammar/issues/749 {"type": "string", "pattern": "^a+$", "maxLength": 2}, {"type": "string", "pattern": "^a+$", "minLength": 3}, {"type": "string", "format": "email", "maxLength": 10}, @@ -48,15 +56,93 @@ def unsupported_array_schemas(): @pytest.fixture -def unsupported_object_schemas(): +def unsupported_property_names_combinations(): return [ - {"type": "object", "propertyNames": {"pattern": "^[a-z]+$"}}, - {"type": "object", "patternProperties": {"^S": {"type": "string"}}}, + # TODO(arpera): + # this case is not covered by xgrammar, so we should report this bug to xgrammar + # xgrammar drops propertyNames whenever patternProperties is present, + # so "grade_12" matches the pattern and escapes the name constraint + { + "type": "object", + "patternProperties": {"^grade_[0-9]+$": {"type": "integer"}}, + "propertyNames": {"pattern": "^grade_[0-9]$"}, # does NOT match "grade_12" + }, + # propertyNames makes xgrammar discard the sibling additionalProperties + # value schema: https://github.com/mlc-ai/xgrammar/issues/826 + { + "type": "object", + "propertyNames": {"pattern": "^[a-z]+$"}, + "additionalProperties": {"type": "integer"}, + }, + # propertyNames is a string schema that conventionally omits "type", so + # it escapes the string check while xgrammar drops its length bound all + # the same: https://github.com/mlc-ai/xgrammar/issues/749 + { + "type": "object", + "propertyNames": {"pattern": "^a+$", "maxLength": 2}, + }, + # xgrammar emits named `properties` separately from `propertyNames` and + # only applies `propertyNames` in its additional-properties branch, so a + # key declared in `properties` escapes the name constraint regardless of + # what `propertyNames` contains. + { + "type": "object", + "properties": {"Bad": {"type": "integer"}}, + "required": ["Bad"], + "propertyNames": {"pattern": "^[a-z]+$"}, + }, + { + "type": "object", + "properties": {"Bad": {"type": "integer"}}, + "propertyNames": {"enum": ["good"]}, + }, + # xgrammar's propertyNames-only branch falls back to an unconstrained + # value type, so it drops unevaluatedProperties whether it restricts + # values (a schema) or forbids them (false). + { + "type": "object", + "propertyNames": {"pattern": "^[a-z]+$"}, + "unevaluatedProperties": {"type": "integer"}, + }, + { + "type": "object", + "propertyNames": {"pattern": "^[a-z]+$"}, + "unevaluatedProperties": False, + }, ] @pytest.fixture -def supported_schema(): +def unsupported_pattern_properties_combinations(): + return [ + # JSON Schema requires a property matched by both `properties` and + # `patternProperties` to satisfy both (conjunction), but xgrammar + # compiles them as alternative branches, so satisfying either one is + # enough. + { + "type": "object", + "properties": {"x": {"type": "string"}}, + "patternProperties": {"^x$": {"type": "integer"}}, + }, + # The same alternative-branches problem applies to overlapping + # patternProperties patterns: a key matching both patterns only has to + # satisfy one of them. + { + "type": "object", + "patternProperties": { + "^a[a-z]*$": {"type": "string"}, + "^[a-z]*z$": {"type": "integer"}, + }, + }, + ] + + +@pytest.fixture +def supported_frankenstein_schema(): + # IMPORTANT(arpera): + # Do NOT add more keywords here! This schema is overcrowded enough that a new + # keyword can end up having no effect on the compiled grammar while the + # test still passes. Give a new keyword its own fixture and named test. return { "type": "object", "properties": { @@ -88,6 +174,19 @@ def supported_schema(): } +@pytest.fixture +def property_names_schema(): + return {"type": "object", "propertyNames": {"pattern": "^[a-z_]+$"}} + + +@pytest.fixture +def pattern_properties_schema(): + return { + "type": "object", + "patternProperties": {"^grade_[0-9]+$": {"type": "integer"}}, + } + + @pytest.mark.parametrize( "schema_type", [ @@ -95,7 +194,8 @@ def supported_schema(): "unsupported_integer_schemas", "unsupported_number_schemas", "unsupported_array_schemas", - "unsupported_object_schemas", + "unsupported_property_names_combinations", + "unsupported_pattern_properties_combinations", ], ) def test_unsupported_json_features_by_type(schema_type, request): @@ -106,7 +206,29 @@ def test_unsupported_json_features_by_type(schema_type, request): ) -def test_supported_json_features(supported_schema): - assert not has_xgrammar_unsupported_json_features(supported_schema), ( - "Schema should be supported" +@pytest.mark.parametrize( + "schema_type", + [ + "supported_frankenstein_schema", + "pattern_properties_schema", + "property_names_schema", + ], +) +def test_supported_json_features(schema_type, request): + schema = request.getfixturevalue(schema_type) + assert not has_xgrammar_unsupported_json_features(schema), ( + f"Schema should be supported: {schema}" ) + + +def test_property_names_constrains_keys(property_names_schema): + assert grammar_accepts(property_names_schema, '{"score": 5}') + assert grammar_accepts(property_names_schema, '{"score": "seven"}') + assert not grammar_accepts(property_names_schema, '{"Score": 5}') + assert not grammar_accepts(property_names_schema, '{"score_1": 5}') + + +def test_pattern_properties_constrains_keys_and_values(pattern_properties_schema): + assert grammar_accepts(pattern_properties_schema, '{"grade_1": 5}') + assert not grammar_accepts(pattern_properties_schema, '{"other": 5}') + assert not grammar_accepts(pattern_properties_schema, '{"grade_1": "five"}') diff --git a/tests/v1/structured_output/test_validation.py b/tests/v1/structured_output/test_validation.py index af417bc03e..a231894384 100644 --- a/tests/v1/structured_output/test_validation.py +++ b/tests/v1/structured_output/test_validation.py @@ -137,7 +137,7 @@ def test_unsupported_grammar_is_a_client_error(backend, structured_outputs): @pytest.mark.parametrize( "schema, expected_backend", [ - # multipleOf is unsupported by xgrammar, patternProperties also by guidance. + # multipleOf is unsupported by xgrammar. ( { "type": "object", @@ -145,8 +145,13 @@ def test_unsupported_grammar_is_a_client_error(backend, structured_outputs): }, "guidance", ), + # patternProperties + properties is also unsupported by guidance. ( - {"type": "object", "patternProperties": {"^a": {"type": "string"}}}, + { + "type": "object", + "properties": {"a": {"type": "string"}}, + "patternProperties": {"^a$": {"type": "string"}}, + }, "outlines", ), ], diff --git a/vllm/v1/structured_output/backend_xgrammar.py b/vllm/v1/structured_output/backend_xgrammar.py index 3809ba2d91..1ba7e2734d 100644 --- a/vllm/v1/structured_output/backend_xgrammar.py +++ b/vllm/v1/structured_output/backend_xgrammar.py @@ -236,6 +236,12 @@ STRING_SUPPORTED_FORMATS = { } +def _has_pattern_and_length_bounds(schema: dict[str, Any]) -> bool: + return ("pattern" in schema or "format" in schema) and ( + "minLength" in schema or "maxLength" in schema + ) + + def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool: """Check if JSON schema contains features unsupported by xgrammar.""" @@ -269,16 +275,39 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool: # the compiled EBNF: pattern/format grammars come out byte-identical # with and without the length keywords, while maxLength alone lowers # to {0, N} correctly. + if obj.get("type") == "string" and _has_pattern_and_length_bounds(obj): + return True + + # propertyNames validates names, so it is a string schema even when it + # omits "type", which is the form that escapes the check above. if ( - obj.get("type") == "string" - and ("pattern" in obj or "format" in obj) - and ("minLength" in obj or "maxLength" in obj) + obj.get("type") == "object" + and isinstance(obj.get("propertyNames"), dict) + and _has_pattern_and_length_bounds(obj["propertyNames"]) ): return True - # Unsupported keywords for objects - if obj.get("type") == "object" and any( - key in obj for key in ("patternProperties", "propertyNames") + # FIXME: propertyNames conflicts with properties/patternProperties/ + # additionalProperties/unevaluatedProperties under xgrammar. + # https://github.com/mlc-ai/xgrammar/issues/826 + if ( + obj.get("type") == "object" + and "propertyNames" in obj + and ( + "properties" in obj + or "patternProperties" in obj + or isinstance(obj.get("additionalProperties"), dict) + or obj.get("unevaluatedProperties", True) is not True + ) + ): + return True + + # FIXME: multiple patternProperties, or patternProperties alongside + # properties, conflict under xgrammar. + if ( + obj.get("type") == "object" + and isinstance(obj.get("patternProperties"), dict) + and ("properties" in obj or len(obj["patternProperties"]) > 1) ): return True