From 41f12a0daf193c80f2e748f6a65f0735623be3a5 Mon Sep 17 00:00:00 2001 From: Jeffrey Wang Date: Sun, 16 Aug 2026 01:04:51 -0700 Subject: [PATCH] [Bugfix] Raise `VLLMValidationError` from structured output validators (#52394) Signed-off-by: Jeffrey Wang Co-authored-by: Claude Opus 5 (1M context) --- .../llm/test_struct_output_generate.py | 7 ++- tests/v1/structured_output/test_validation.py | 59 ++++++++++++++++++- vllm/entrypoints/openai/engine/protocol.py | 2 +- vllm/sampling_params.py | 9 ++- vllm/v1/structured_output/backend_guidance.py | 14 +++-- .../backend_lm_format_enforcer.py | 9 +-- vllm/v1/structured_output/backend_outlines.py | 20 ++++--- vllm/v1/structured_output/backend_xgrammar.py | 19 +++--- 8 files changed, 109 insertions(+), 30 deletions(-) diff --git a/tests/entrypoints/llm/test_struct_output_generate.py b/tests/entrypoints/llm/test_struct_output_generate.py index 7b90aeacda..0f55ad7adc 100644 --- a/tests/entrypoints/llm/test_struct_output_generate.py +++ b/tests/entrypoints/llm/test_struct_output_generate.py @@ -13,6 +13,7 @@ from pydantic import BaseModel from tests.reasoning.utils import run_reasoning_extraction from vllm.config import StructuredOutputsConfig +from vllm.exceptions import VLLMValidationError from vllm.outputs import RequestOutput from vllm.platforms import current_platform from vllm.reasoning.abs_reasoning_parsers import ReasoningParserManager @@ -326,7 +327,7 @@ def test_structured_output( ) if backend.startswith("xgrammar"): with pytest.raises( - ValueError, + VLLMValidationError, match="The provided JSON schema contains features " "not supported by xgrammar.", ): @@ -453,7 +454,9 @@ def test_structured_output( max_tokens=1000, structured_outputs=StructuredOutputsParams(grammar="not a grammar"), ) - with pytest.raises(ValueError, match="Failed to convert the grammar "): + with pytest.raises( + VLLMValidationError, match="Failed to convert the grammar " + ): runner.llm.generate( ( "Generate a sql statement that selects col_1 from " diff --git a/tests/v1/structured_output/test_validation.py b/tests/v1/structured_output/test_validation.py index 08be56613c..af417bc03e 100644 --- a/tests/v1/structured_output/test_validation.py +++ b/tests/v1/structured_output/test_validation.py @@ -5,7 +5,7 @@ import pytest from vllm.config import StructuredOutputsConfig -from vllm.exceptions import VLLMValidationError +from vllm.exceptions import VLLMClientError, VLLMValidationError from vllm.sampling_params import SamplingParams, StructuredOutputsParams pytestmark = pytest.mark.cpu_test @@ -103,3 +103,60 @@ def test_regex_with_nul_byte_rejected(regex): with pytest.raises(ValueError, match="NUL"): validate_xgrammar_grammar(params) + + +INVALID_JSON_SCHEMA = {"type": "object", "properties": {"name": {"type": "str"}}} + + +@pytest.mark.parametrize( + "backend, structured_outputs", + [ + ("xgrammar", StructuredOutputsParams(json=INVALID_JSON_SCHEMA)), + ("outlines", StructuredOutputsParams(json=INVALID_JSON_SCHEMA)), + ("auto", StructuredOutputsParams(json=INVALID_JSON_SCHEMA)), + ("auto", StructuredOutputsParams(json='{"type": ')), + ("xgrammar", StructuredOutputsParams(grammar="not a grammar")), + ("guidance", StructuredOutputsParams(grammar="not a grammar")), + ("lm-format-enforcer", StructuredOutputsParams(grammar="not a grammar")), + ("outlines", StructuredOutputsParams(regex="(")), + ("guidance", StructuredOutputsParams(structural_tag='{"nope": 1}')), + ], +) +def test_unsupported_grammar_is_a_client_error(backend, structured_outputs): + """Only `VLLMClientError` survives `AsyncLLM.generate` untouched; anything else + is wrapped in `EngineGenerateError` and served as a 500 instead of a 400.""" + params = SamplingParams(structured_outputs=structured_outputs) + with pytest.raises(VLLMClientError): + params._validate_structured_outputs( + _StubModelConfig(is_diffusion=False), + StructuredOutputsConfig(backend=backend), + tokenizer=object(), + ) + + +@pytest.mark.parametrize( + "schema, expected_backend", + [ + # multipleOf is unsupported by xgrammar, patternProperties also by guidance. + ( + { + "type": "object", + "properties": {"n": {"type": "integer", "multipleOf": 2}}, + }, + "guidance", + ), + ( + {"type": "object", "patternProperties": {"^a": {"type": "string"}}}, + "outlines", + ), + ], +) +def test_auto_backend_falls_back_on_unsupported_schema(schema, expected_backend): + """`auto` falls back on rejection, so it must catch what the validators raise.""" + params = SamplingParams(structured_outputs=StructuredOutputsParams(json=schema)) + params._validate_structured_outputs( + _StubModelConfig(is_diffusion=False), + StructuredOutputsConfig(backend="auto"), + tokenizer=object(), + ) + assert params.structured_outputs._backend == expected_backend diff --git a/vllm/entrypoints/openai/engine/protocol.py b/vllm/entrypoints/openai/engine/protocol.py index dded1261dc..c32be4459c 100644 --- a/vllm/entrypoints/openai/engine/protocol.py +++ b/vllm/entrypoints/openai/engine/protocol.py @@ -266,7 +266,7 @@ def validate_structural_tag_payload(payload: Any, *, parameter: str) -> None: structured_outputs=StructuredOutputsParams(structural_tag=payload) ) ) - except (TypeError, ValueError) as exc: + except (TypeError, ValueError, VLLMValidationError) as exc: raise VLLMValidationError( f"Invalid {parameter} structural_tag specification.", parameter=parameter, diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 46c4046082..2f0af9e1b4 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -1092,7 +1092,7 @@ class SamplingParams( try: validate_xgrammar_grammar(self) self.structured_outputs._backend = "xgrammar" - except ValueError: + except VLLMValidationError: # The request either failed validation # or includes some jsonschema feature(s) that # are not supported in xgrammar. @@ -1103,7 +1103,12 @@ class SamplingParams( so_params = self.structured_outputs if not skip_guidance and so_params.json: if isinstance(so_params.json, str): - schema = json_mod.loads(so_params.json) + try: + schema = json_mod.loads(so_params.json) + except json_mod.JSONDecodeError as e: + raise VLLMValidationError( + "Invalid JSON grammar specification." + ) from e else: schema = so_params.json skip_guidance = has_guidance_unsupported_json_features(schema) diff --git a/vllm/v1/structured_output/backend_guidance.py b/vllm/v1/structured_output/backend_guidance.py index 30ecbfa065..310576cd99 100644 --- a/vllm/v1/structured_output/backend_guidance.py +++ b/vllm/v1/structured_output/backend_guidance.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any import torch from transformers import MistralCommonBackend +from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger from vllm.sampling_params import SamplingParams from vllm.utils.import_utils import LazyLoader @@ -269,7 +270,7 @@ def serialize_guidance_grammar( begin: str = s["begin"] trig = next((t for t in triggers if begin.startswith(t)), None) if trig is None: - raise ValueError( + raise VLLMValidationError( f"Trigger {begin} not found in triggers {triggers}" ) tags.append( @@ -281,7 +282,9 @@ def serialize_guidance_grammar( ) ) if not tags: - raise ValueError("No structural tags found in the grammar spec.") + raise VLLMValidationError( + "No structural tags found in the grammar spec." + ) return llguidance.StructTag.to_grammar(tags) else: logger.error( @@ -300,7 +303,10 @@ def validate_guidance_grammar( if sampling_params.structured_outputs is None: return tp, grm = get_structured_output_key(sampling_params.structured_outputs) - guidance_grm = serialize_guidance_grammar(tp, grm) + try: + guidance_grm = serialize_guidance_grammar(tp, grm) + except (ValueError, KeyError, TypeError) as e: + raise VLLMValidationError(f"Invalid grammar specification: {e}") from e err = llguidance.LLMatcher.validate_grammar(guidance_grm, tokenizer) if err: - raise ValueError(f"Grammar error: {err}") + raise VLLMValidationError(f"Grammar error: {err}") diff --git a/vllm/v1/structured_output/backend_lm_format_enforcer.py b/vllm/v1/structured_output/backend_lm_format_enforcer.py index 898aaa136e..36eb5eb159 100644 --- a/vllm/v1/structured_output/backend_lm_format_enforcer.py +++ b/vllm/v1/structured_output/backend_lm_format_enforcer.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING import torch from transformers import PreTrainedTokenizerBase +from vllm.exceptions import VLLMValidationError from vllm.sampling_params import SamplingParams from vllm.utils.import_utils import LazyLoader from vllm.utils.torch_utils import PIN_MEMORY @@ -166,7 +167,7 @@ def validate_structured_output_request_lm_format_enforcer(params: SamplingParams so_params.regex, ) except Exception as err: - raise ValueError( + raise VLLMValidationError( f"Failed to compile regex for lm-format-enforcer: {err}" ) from err return @@ -176,19 +177,19 @@ def validate_structured_output_request_lm_format_enforcer(params: SamplingParams # make sure schema is valid json json.loads(so_params.json) except json.JSONDecodeError as e: - raise ValueError("Invalid JSON grammar specification.") from e + raise VLLMValidationError("Invalid JSON grammar specification.") from e else: try: json.dumps(so_params.json) except Exception as e: - raise ValueError( + raise VLLMValidationError( f"Error serializing structured outputs jsonschema: {e}" ) from e return elif so_params.choice: return elif so_params.grammar: - raise ValueError( + raise VLLMValidationError( "LM Format Enforcer structured outputs backend " "does not support grammar specifications" ) diff --git a/vllm/v1/structured_output/backend_outlines.py b/vllm/v1/structured_output/backend_outlines.py index e66ef6361a..dd460f96ec 100644 --- a/vllm/v1/structured_output/backend_outlines.py +++ b/vllm/v1/structured_output/backend_outlines.py @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING import torch from regex import escape as regex_escape +from vllm.exceptions import VLLMValidationError from vllm.sampling_params import SamplingParams from vllm.utils.import_utils import LazyLoader from vllm.utils.torch_utils import PIN_MEMORY @@ -186,22 +187,27 @@ def validate_structured_output_request_outlines(params: SamplingParams): json.loads(so_params.json) schema = so_params.json except json.JSONDecodeError as e: - raise ValueError("Invalid JSON grammar specification.") from e + raise VLLMValidationError("Invalid JSON grammar specification.") from e else: try: schema = json.dumps(so_params.json) except Exception as e: - raise ValueError( + raise VLLMValidationError( f"Error serializing structured outputs jsonschema: {e}" ) from e - pattern = json_schema.build_regex_from_schema(schema) + try: + pattern = json_schema.build_regex_from_schema(schema) + except Exception as e: + raise VLLMValidationError( + f"Failed to transform json schema into a regex: {e}" + ) from e validate_regex_is_buildable(pattern) elif so_params.choice: choices = [regex_escape(str(choice)) for choice in so_params.choice] regex = "(" + "|".join(choices) + ")" validate_regex_is_buildable(regex) elif so_params.grammar: - raise ValueError( + raise VLLMValidationError( "Outlines structured outputs backend " "does not support grammar specifications" ) @@ -315,19 +321,19 @@ def validate_regex_is_buildable(pattern: str) -> None: parsed = sre_parse.parse(pattern) except sre_constants.error as e: - raise ValueError(f"Error parsing regex: {e}") from e + raise VLLMValidationError(f"Error parsing regex: {e}") from e try: _check_unsupported(parsed) except ValueError as e: - raise ValueError( + raise VLLMValidationError( f"Regex uses unsupported feature for structured outputs: {e}. " "Only basic matching constructs are supported—lookarounds, " "backreferences, and unicode boundaries are not." ) from e if _prefix_needs_context(parsed): - raise ValueError( + raise VLLMValidationError( "Regex does not have a anchored universal start state" "This means that the Regex uses anchors (^) or look-arounds " "in a way which requires context before any token is matched." diff --git a/vllm/v1/structured_output/backend_xgrammar.py b/vllm/v1/structured_output/backend_xgrammar.py index e11b1601e1..258b1dff32 100644 --- a/vllm/v1/structured_output/backend_xgrammar.py +++ b/vllm/v1/structured_output/backend_xgrammar.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any import torch import vllm.envs +from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger from vllm.sampling_params import SamplingParams from vllm.utils.import_utils import LazyLoader @@ -276,7 +277,7 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool: def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: """Validate that the request is supported by structured output. - Raises ValueError if the request is not supported. + Raises VLLMValidationError if the request is not supported. """ if sampling_params.structured_outputs is None: return @@ -298,7 +299,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: so_params.regex, ) except Exception as err: - raise ValueError( + raise VLLMValidationError( f"Failed to transform regex into a grammar: {err}" ) from err @@ -307,7 +308,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: try: xgr.Grammar.from_ebnf(choice_grammar) except Exception as err: - raise ValueError( + raise VLLMValidationError( f"Failed to transform choices into a grammar: {err}" ) from err so_params.choice = None @@ -319,19 +320,19 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: try: schema = json.loads(so_params.json) except json.JSONDecodeError as e: - raise ValueError("Invalid JSON grammar specification.") from e + raise VLLMValidationError("Invalid JSON grammar specification.") from e else: schema = so_params.json if has_xgrammar_unsupported_json_features(schema): - raise ValueError( + raise VLLMValidationError( "The provided JSON schema contains features not supported by xgrammar." ) try: xgr.Grammar.from_json_schema(schema) except Exception as err: - raise ValueError( + raise VLLMValidationError( f"Failed to transform json schema into a grammar: {err}" ) from err return @@ -342,7 +343,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: try: so_params.grammar = convert_lark_to_ebnf(so_params.grammar) except ValueError as e: - raise ValueError( + raise VLLMValidationError( "Failed to convert the grammar from Lark to EBNF. " ) from e @@ -351,7 +352,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: # parse the grammar, but we aren't compiling it. xgr.Grammar.from_ebnf(so_params.grammar) except Exception as e: - raise ValueError("Invalid grammar specification.") from e + raise VLLMValidationError("Invalid grammar specification.") from e return if so_params.structural_tag: @@ -372,4 +373,4 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: else: xgr.Grammar.from_structural_tag(so_params.structural_tag) except Exception as e: - raise ValueError("Invalid structural tag specification.") from e + raise VLLMValidationError("Invalid structural tag specification.") from e