diff --git a/tests/v1/structured_output/test_validation.py b/tests/v1/structured_output/test_validation.py index 276056b501..41166c2602 100644 --- a/tests/v1/structured_output/test_validation.py +++ b/tests/v1/structured_output/test_validation.py @@ -17,6 +17,13 @@ from vllm.sampling_params import ( ) from vllm.tokenizers import mistral as mistral_tokenizers from vllm.v1.structured_output import backend_guidance +from vllm.v1.structured_output.backend_lm_format_enforcer import ( + validate_structured_output_request_lm_format_enforcer, +) +from vllm.v1.structured_output.backend_outlines import ( + validate_structured_output_request_outlines, +) +from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar pytestmark = pytest.mark.cpu_test @@ -213,6 +220,9 @@ def test_schema_at_nesting_limit_accepted(field): INVALID_JSON_SCHEMA = {"type": "object", "properties": {"name": {"type": "str"}}} +STRUCTURAL_TAG = ( + '{"type": "structural_tag", "format": {"type": "const_string", "value": "hi"}}' +) @pytest.mark.parametrize( @@ -227,6 +237,9 @@ INVALID_JSON_SCHEMA = {"type": "object", "properties": {"name": {"type": "str"}} ("lm-format-enforcer", StructuredOutputsParams(grammar="not a grammar")), ("outlines", StructuredOutputsParams(regex="(")), ("guidance", StructuredOutputsParams(structural_tag='{"nope": 1}')), + ("outlines", StructuredOutputsParams(structural_tag=STRUCTURAL_TAG)), + ("lm-format-enforcer", StructuredOutputsParams(structural_tag=STRUCTURAL_TAG)), + ("outlines", StructuredOutputsParams(json_object=True)), ], ) def test_unsupported_grammar_is_a_client_error(backend, structured_outputs): @@ -241,6 +254,50 @@ def test_unsupported_grammar_is_a_client_error(backend, structured_outputs): ) +@pytest.mark.parametrize( + "backend, structured_outputs", + [ + ("xgrammar", StructuredOutputsParams(json_object=True)), + ("xgrammar", StructuredOutputsParams(regex="[0-9]+")), + ("xgrammar", StructuredOutputsParams(regex="")), + ("xgrammar", StructuredOutputsParams(json={})), + ("lm-format-enforcer", StructuredOutputsParams(json_object=True)), + ("lm-format-enforcer", StructuredOutputsParams(regex="")), + ("lm-format-enforcer", StructuredOutputsParams(json={})), + ("outlines", StructuredOutputsParams(regex="")), + ("outlines", StructuredOutputsParams(json={})), + ], +) +def test_supported_spec_is_not_rejected(backend, structured_outputs): + """Supported specs, including the falsy but valid regex="" and json={}, must + not reach the validators' final rejection.""" + params = SamplingParams(structured_outputs=structured_outputs) + params._validate_structured_outputs( + _StubModelConfig(is_diffusion=False), + StructuredOutputsConfig(backend=backend), + tokenizer=object(), + ) + + +@pytest.mark.parametrize( + "validate", + [ + validate_structured_output_request_outlines, + validate_structured_output_request_lm_format_enforcer, + validate_xgrammar_grammar, + ], + ids=["outlines", "lm-format-enforcer", "xgrammar"], +) +def test_unhandled_spec_is_a_validation_error(validate): + """json_object=False is rejected before backend selection, so only a direct + validator call reaches the final branch; it must still be a 400.""" + params = SamplingParams( + structured_outputs=StructuredOutputsParams(json_object=False) + ) + with pytest.raises(VLLMValidationError, match="does not support this"): + validate(params) + + @pytest.mark.parametrize( "schema, expected_backend", [ diff --git a/vllm/v1/structured_output/backend_lm_format_enforcer.py b/vllm/v1/structured_output/backend_lm_format_enforcer.py index 9a16ddc1fe..b48cdc7916 100644 --- a/vllm/v1/structured_output/backend_lm_format_enforcer.py +++ b/vllm/v1/structured_output/backend_lm_format_enforcer.py @@ -159,7 +159,7 @@ def validate_structured_output_request_lm_format_enforcer(params: SamplingParams so_params = params.structured_outputs - if so_params.regex: + if so_params.regex is not None: try: compile_regex_with_timeout( lmformatenforcer.RegexParser, @@ -170,7 +170,7 @@ def validate_structured_output_request_lm_format_enforcer(params: SamplingParams f"Failed to compile regex for lm-format-enforcer: {err}" ) from err return - elif so_params.json: + elif so_params.json is not None: if isinstance(so_params.json, str): try: # make sure schema is valid json @@ -185,10 +185,20 @@ def validate_structured_output_request_lm_format_enforcer(params: SamplingParams f"Error serializing structured outputs jsonschema: {e}" ) from e return - elif so_params.choice: + elif so_params.choice or so_params.json_object: return elif so_params.grammar: raise VLLMValidationError( "LM Format Enforcer structured outputs backend " "does not support grammar specifications" ) + elif so_params.structural_tag: + raise VLLMValidationError( + "LM Format Enforcer structured outputs backend " + "does not support structural_tag specifications" + ) + else: + raise VLLMValidationError( + "LM Format Enforcer structured outputs backend " + "does not support this structured outputs specification" + ) diff --git a/vllm/v1/structured_output/backend_outlines.py b/vllm/v1/structured_output/backend_outlines.py index 176d1a46e6..09588a680a 100644 --- a/vllm/v1/structured_output/backend_outlines.py +++ b/vllm/v1/structured_output/backend_outlines.py @@ -198,9 +198,9 @@ def validate_structured_output_request_outlines(params: SamplingParams): so_params = params.structured_outputs - if so_params.regex: + if so_params.regex is not None: validate_regex_is_buildable(so_params.regex) - elif so_params.json: + elif so_params.json is not None: if isinstance(so_params.json, str): try: # make sure schema is valid json @@ -236,6 +236,16 @@ def validate_structured_output_request_outlines(params: SamplingParams): "Outlines structured outputs backend " "does not support grammar specifications" ) + elif so_params.structural_tag: + raise VLLMValidationError( + "Outlines structured outputs backend " + "does not support structural_tag specifications" + ) + else: + raise VLLMValidationError( + "Outlines structured outputs backend " + "does not support this structured outputs specification" + ) def _prefix_needs_context(parsed) -> bool: diff --git a/vllm/v1/structured_output/backend_xgrammar.py b/vllm/v1/structured_output/backend_xgrammar.py index 0199c5f75d..9631766d92 100644 --- a/vllm/v1/structured_output/backend_xgrammar.py +++ b/vllm/v1/structured_output/backend_xgrammar.py @@ -414,7 +414,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: so_params = sampling_params.structured_outputs - if so_params.regex: + if so_params.regex is not None: # A NUL byte is never meaningful in a regex pattern and is not handled # by xgrammar's native regex converter. Reject it here, before the # pattern reaches that native code; the try/except below does not cover @@ -432,6 +432,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: raise VLLMValidationError( f"Failed to transform regex into a grammar: {err}" ) from err + return if so_params.choice: choice_grammar = choice_as_grammar(so_params.choice) @@ -445,7 +446,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: so_params.grammar = choice_grammar return - if so_params.json: + if so_params.json is not None: check_json_nesting(so_params.json) if isinstance(so_params.json, str): try: @@ -470,6 +471,9 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: ) from err return + if so_params.json_object: + return + if so_params.grammar: # Parse the grammar with the same syntax `compile_grammar` will use, # but don't compile it. The grammar is passed on unchanged. @@ -502,3 +506,9 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: xgr.Grammar.from_structural_tag(so_params.structural_tag) except Exception as e: raise VLLMValidationError("Invalid structural tag specification.") from e + return + + raise VLLMValidationError( + "XGrammar structured outputs backend " + "does not support this structured outputs specification" + )