diff --git a/tests/v1/structured_output/test_backend_xgrammar.py b/tests/v1/structured_output/test_backend_xgrammar.py index bc29112077..9024b614dd 100644 --- a/tests/v1/structured_output/test_backend_xgrammar.py +++ b/tests/v1/structured_output/test_backend_xgrammar.py @@ -31,21 +31,6 @@ def grammar_accepts(schema: dict, text: str) -> bool: # ================================================ -@pytest.fixture -def unsupported_string_schemas(): - return [ - {"type": "string", "format": "non_existing_format"}, - # TODO(arpera): - # pattern/format is compiled but length bounds are silently dropped, - # so the combination must be rejected instead of producing quietly - # wrong output - # https://github.com/mlc-ai/xgrammar/issues/749 - {"type": "string", "pattern": "^a+$", "maxLength": 2}, - {"type": "string", "pattern": "^a+$", "minLength": 3}, - {"type": "string", "format": "email", "maxLength": 10}, - ] - - @pytest.fixture def unsupported_multipleOf_schemas(): return [ @@ -59,6 +44,8 @@ def unsupported_multipleOf_schemas(): @pytest.fixture def unsupported_array_schemas(): return [ + # array + some constraints is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/968 {"type": "array", "uniqueItems": True}, {"type": "array", "contains": {"type": "string"}}, {"type": "array", "minContains": 1}, @@ -67,74 +54,113 @@ def unsupported_array_schemas(): @pytest.fixture -def unsupported_property_names_combinations(): +def unsupported_string_schemas(): return [ - # TODO(arpera): - # this case is not covered by xgrammar, so we should report this bug to xgrammar + # ======================================================== + # string + format is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/967 + {"type": "string", "format": "non_existing_format"}, + # ======================================================== + # + # ======================================================== + # string + format/pattern + length constraint is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/966 + # + # pattern/format is compiled but length bounds are silently dropped, + # so the combination must be rejected instead of producing quietly + # wrong output + {"type": "string", "pattern": "^a+$", "maxLength": 2}, + {"type": "string", "pattern": "^a+$", "minLength": 3}, + {"type": "string", "format": "email", "maxLength": 10}, + # ======================================================== + ] + + +@pytest.fixture +def unsupported_propertyNames_combinations(): + return [ + # ======================================================== + # propertyNames + patternProperties is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/959 + # # xgrammar drops propertyNames whenever patternProperties is present, # so "grade_12" matches the pattern and escapes the name constraint { "type": "object", - "patternProperties": {"^grade_[0-9]+$": {"type": "integer"}}, "propertyNames": {"pattern": "^grade_[0-9]$"}, # does NOT match "grade_12" + "patternProperties": {"^grade_[0-9]+$": {"type": "integer"}}, }, - # propertyNames makes xgrammar discard the sibling additionalProperties - # value schema: https://github.com/mlc-ai/xgrammar/issues/826 - { - "type": "object", - "propertyNames": {"pattern": "^[a-z]+$"}, - "additionalProperties": {"type": "integer"}, - }, - # propertyNames is a string schema that conventionally omits "type", so - # it escapes the string check while xgrammar drops its length bound all - # the same: https://github.com/mlc-ai/xgrammar/issues/749 + # ======================================================== + # + # ======================================================== + # propertyNames + maxLength is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/960 { "type": "object", "propertyNames": {"pattern": "^a+$", "maxLength": 2}, }, - # xgrammar emits named `properties` separately from `propertyNames` and - # only applies `propertyNames` in its additional-properties branch, so a - # key declared in `properties` escapes the name constraint regardless of - # what `propertyNames` contains. + # ======================================================== + # + # ======================================================== + # propertyNames + properties is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/958 { "type": "object", + "propertyNames": {"pattern": "^[a-z]+$"}, "properties": {"Bad": {"type": "integer"}}, "required": ["Bad"], - "propertyNames": {"pattern": "^[a-z]+$"}, }, { "type": "object", - "properties": {"Bad": {"type": "integer"}}, "propertyNames": {"enum": ["good"]}, + "properties": {"Bad": {"type": "integer"}}, }, + # ======================================================== + # + # ======================================================== + # propertyNames + unevaluatedProperties is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/961 + # # xgrammar's propertyNames-only branch falls back to an unconstrained # value type, so it drops unevaluatedProperties whether it restricts # values (a schema) or forbids them (false). { "type": "object", "propertyNames": {"pattern": "^[a-z]+$"}, - "unevaluatedProperties": {"type": "integer"}, + "unevaluatedProperties": False, }, + # also keep in mind that unevaluatedProperties can be a schema { "type": "object", "propertyNames": {"pattern": "^[a-z]+$"}, - "unevaluatedProperties": False, + "unevaluatedProperties": {"type": "integer"}, }, + # ======================================================== ] @pytest.fixture -def unsupported_pattern_properties_combinations(): +def unsupported_patternProperties_combinations(): return [ + # ======================================================== + # patternProperties + properties is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/964 + # # JSON Schema requires a property matched by both `properties` and # `patternProperties` to satisfy both (conjunction), but xgrammar # compiles them as alternative branches, so satisfying either one is # enough. { "type": "object", - "properties": {"x": {"type": "string"}}, "patternProperties": {"^x$": {"type": "integer"}}, + "properties": {"x": {"type": "string"}}, }, + # ======================================================== + # + # ======================================================== + # patternProperties + patternProperties is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/965 + # # The same alternative-branches problem applies to overlapping # patternProperties patterns: a key matching both patterns only has to # satisfy one of them. @@ -145,6 +171,7 @@ def unsupported_pattern_properties_combinations(): "^[a-z]*z$": {"type": "integer"}, }, }, + # ======================================================== ] @@ -273,6 +300,24 @@ def property_names_schema(): return {"type": "object", "propertyNames": {"pattern": "^[a-z_]+$"}} +@pytest.fixture +def property_names_with_additional_properties_schema(): + """This combination propertyNames + additionalProperties + was unsupported some time ago by xgrammar. + So, this test case was in unsupported category before. + But this xgrammar's PR + https://github.com/mlc-ai/xgrammar/pull/836 + added support of propertyNames + additionalProperties + This fix was released in xgrammar v0.2.7 + which vLLM pins in PR #57272 + """ + return { + "type": "object", + "propertyNames": {"pattern": "^[a-z_]+$"}, + "additionalProperties": {"type": "integer"}, + } + + @pytest.fixture def supported_allof_anyof_and_oneof(): """Xgrammar DO support some variants of anyOf, allOf, oneOf @@ -334,8 +379,8 @@ class TestHasXGrammarUnsupportedJsonFeatures: "unsupported_string_schemas", "unsupported_multipleOf_schemas", "unsupported_array_schemas", - "unsupported_property_names_combinations", - "unsupported_pattern_properties_combinations", + "unsupported_propertyNames_combinations", + "unsupported_patternProperties_combinations", # vLLM issue #56556 "unsupported_multibranch_allof", "unsupported_vllm_issue_56556_schema", @@ -354,6 +399,7 @@ class TestHasXGrammarUnsupportedJsonFeatures: "supported_frankenstein_schema", "pattern_properties_schema", "property_names_schema", + "property_names_with_additional_properties_schema", # Additional test cases implemented during work on # vLLM issue #56556 "supported_allof_anyof_and_oneof", @@ -421,6 +467,14 @@ class TestIsGrammarAcceptString: assert not grammar_accepts(property_names_schema, '{"Score": 5}') assert not grammar_accepts(property_names_schema, '{"score_1": 5}') + def test_property_names_keeps_additional_properties_value_schema( + self, property_names_with_additional_properties_schema + ): + schema = property_names_with_additional_properties_schema + assert grammar_accepts(schema, '{"score": 5}') + assert not grammar_accepts(schema, '{"Score": 5}') + assert not grammar_accepts(schema, '{"score": "seven"}') + def test_pattern_properties_constrains_keys_and_values( self, pattern_properties_schema ): diff --git a/vllm/v1/structured_output/backend_xgrammar.py b/vllm/v1/structured_output/backend_xgrammar.py index b675a9a3f1..0199c5f75d 100644 --- a/vllm/v1/structured_output/backend_xgrammar.py +++ b/vllm/v1/structured_output/backend_xgrammar.py @@ -296,18 +296,29 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool: schema_types = _schema_types(obj) - # Check for numeric ranges + # integer/number + multipleOf is unsupported by xgrammar + # This is known behavior and xgrammar emits warning in logs: + # [21:18:08] /project/cpp/json_schema_converter.cc:1053: + # Warning: multipleOf is not supported for type:number; ignoring multipleOf + # This warning was added in PR + # https://github.com/mlc-ai/xgrammar/pull/670 + # So, no need to track progress on this if (schema_types & {"integer", "number"}) and ("multipleOf" in obj): return True - # Check for array unsupported keywords + # array + some constraints is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/968 if "array" in schema_types and any( key in obj for key in ("uniqueItems", "contains", "minContains", "maxContains") ): return True - # Unsupported keywords for strings + # string + format with unsupported keywords + # is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/967 + # See tests on this in test_backend_xgrammar.py + # unsupported_string_schemas if ( "string" in schema_types and "format" in obj @@ -315,48 +326,51 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool: ): return True - # A string mixing a generative constraint (pattern or format) with - # explicit length bounds. xgrammar compiles the pattern/format side - # and silently drops minLength/maxLength from the grammar, so output - # can violate the bound without any error surfacing. Verified against - # the compiled EBNF: pattern/format grammars come out byte-identical - # with and without the length keywords, while maxLength alone lowers - # to {0, N} correctly. + # string + format/pattern + length constraint is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/966 + # See tests on this in test_backend_xgrammar.py + # unsupported_string_schemas if "string" in schema_types and _has_pattern_and_length_bounds(obj): return True - # propertyNames validates names, so it is a string schema even when it - # omits "type", which is the form that escapes the check above. - if ( - "object" in schema_types - and isinstance(obj.get("propertyNames"), dict) - and _has_pattern_and_length_bounds(obj["propertyNames"]) - ): - return True + # propertyNames is not supported in pair with some constraints + # in xgrammar + # See tests on this in test_backend_xgrammar.py + # unsupported_propertyNames_combinations + if "object" in schema_types and "propertyNames" in obj: + # propertyNames + maxLength is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/960 + property_names = obj.get("propertyNames") + if isinstance(property_names, dict) and _has_pattern_and_length_bounds( + property_names + ): + return True + # propertyNames + patternProperties is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/959 + if "patternProperties" in obj: + return True + # propertyNames + properties is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/958 + if "properties" in obj: + return True + # propertyNames + unevaluatedProperties is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/961 + if obj.get("unevaluatedProperties", True) is not True: + return True - # FIXME: propertyNames conflicts with properties/patternProperties/ - # additionalProperties/unevaluatedProperties under xgrammar. - # https://github.com/mlc-ai/xgrammar/issues/826 - if ( - "object" in schema_types - and "propertyNames" in obj - and ( - "properties" in obj - or "patternProperties" in obj - or isinstance(obj.get("additionalProperties"), dict) - or obj.get("unevaluatedProperties", True) is not True - ) - ): - return True - - # FIXME: multiple patternProperties, or patternProperties alongside - # properties, conflict under xgrammar. - if ( - "object" in schema_types - and isinstance(obj.get("patternProperties"), dict) - and ("properties" in obj or len(obj["patternProperties"]) > 1) - ): - return True + # patternProperties is not supported in pair with some constraints + # in xgrammar + # See tests on this in test_backend_xgrammar.py + # unsupported_patternProperties_combinations + if "object" in schema_types and isinstance(obj.get("patternProperties"), dict): + # patternProperties + properties is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/964 + if "properties" in obj: + return True + # patternProperties + patternProperties is unsupported by xgrammar + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/965 + if len(obj["patternProperties"]) > 1: + return True # Note(arpera): # Xgrammar lacks support of multi-branch allOf @@ -370,8 +384,7 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool: # would accept any kind of json, such as # "maybe", "", 42, {}, [], {"a": 1}, etc. # which is NOT what is expected. - # Reported this issue to xgrammar team to track progress on resolving: - # https://github.com/mlc-ai/xgrammar/issues/937 + # Tracking issue: https://github.com/mlc-ai/xgrammar/issues/937 allof = obj.get("allOf") if isinstance(allof, list) and len(allof) >= 2: return True