[Bugfix] Raise VLLMValidationError from structured output validators (#52394)

Signed-off-by: Jeffrey Wang <jeffreywang@anyscale.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jeffrey Wang
2026-08-16 01:04:51 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 8efa13b700
commit 41f12a0daf
8 changed files with 109 additions and 30 deletions
@@ -13,6 +13,7 @@ from pydantic import BaseModel
from tests.reasoning.utils import run_reasoning_extraction
from vllm.config import StructuredOutputsConfig
from vllm.exceptions import VLLMValidationError
from vllm.outputs import RequestOutput
from vllm.platforms import current_platform
from vllm.reasoning.abs_reasoning_parsers import ReasoningParserManager
@@ -326,7 +327,7 @@ def test_structured_output(
)
if backend.startswith("xgrammar"):
with pytest.raises(
ValueError,
VLLMValidationError,
match="The provided JSON schema contains features "
"not supported by xgrammar.",
):
@@ -453,7 +454,9 @@ def test_structured_output(
max_tokens=1000,
structured_outputs=StructuredOutputsParams(grammar="not a grammar"),
)
with pytest.raises(ValueError, match="Failed to convert the grammar "):
with pytest.raises(
VLLMValidationError, match="Failed to convert the grammar "
):
runner.llm.generate(
(
"Generate a sql statement that selects col_1 from "
+58 -1
View File
@@ -5,7 +5,7 @@
import pytest
from vllm.config import StructuredOutputsConfig
from vllm.exceptions import VLLMValidationError
from vllm.exceptions import VLLMClientError, VLLMValidationError
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
pytestmark = pytest.mark.cpu_test
@@ -103,3 +103,60 @@ def test_regex_with_nul_byte_rejected(regex):
with pytest.raises(ValueError, match="NUL"):
validate_xgrammar_grammar(params)
INVALID_JSON_SCHEMA = {"type": "object", "properties": {"name": {"type": "str"}}}
@pytest.mark.parametrize(
"backend, structured_outputs",
[
("xgrammar", StructuredOutputsParams(json=INVALID_JSON_SCHEMA)),
("outlines", StructuredOutputsParams(json=INVALID_JSON_SCHEMA)),
("auto", StructuredOutputsParams(json=INVALID_JSON_SCHEMA)),
("auto", StructuredOutputsParams(json='{"type": ')),
("xgrammar", StructuredOutputsParams(grammar="not a grammar")),
("guidance", StructuredOutputsParams(grammar="not a grammar")),
("lm-format-enforcer", StructuredOutputsParams(grammar="not a grammar")),
("outlines", StructuredOutputsParams(regex="(")),
("guidance", StructuredOutputsParams(structural_tag='{"nope": 1}')),
],
)
def test_unsupported_grammar_is_a_client_error(backend, structured_outputs):
"""Only `VLLMClientError` survives `AsyncLLM.generate` untouched; anything else
is wrapped in `EngineGenerateError` and served as a 500 instead of a 400."""
params = SamplingParams(structured_outputs=structured_outputs)
with pytest.raises(VLLMClientError):
params._validate_structured_outputs(
_StubModelConfig(is_diffusion=False),
StructuredOutputsConfig(backend=backend),
tokenizer=object(),
)
@pytest.mark.parametrize(
"schema, expected_backend",
[
# multipleOf is unsupported by xgrammar, patternProperties also by guidance.
(
{
"type": "object",
"properties": {"n": {"type": "integer", "multipleOf": 2}},
},
"guidance",
),
(
{"type": "object", "patternProperties": {"^a": {"type": "string"}}},
"outlines",
),
],
)
def test_auto_backend_falls_back_on_unsupported_schema(schema, expected_backend):
"""`auto` falls back on rejection, so it must catch what the validators raise."""
params = SamplingParams(structured_outputs=StructuredOutputsParams(json=schema))
params._validate_structured_outputs(
_StubModelConfig(is_diffusion=False),
StructuredOutputsConfig(backend="auto"),
tokenizer=object(),
)
assert params.structured_outputs._backend == expected_backend
+1 -1
View File
@@ -266,7 +266,7 @@ def validate_structural_tag_payload(payload: Any, *, parameter: str) -> None:
structured_outputs=StructuredOutputsParams(structural_tag=payload)
)
)
except (TypeError, ValueError) as exc:
except (TypeError, ValueError, VLLMValidationError) as exc:
raise VLLMValidationError(
f"Invalid {parameter} structural_tag specification.",
parameter=parameter,
+7 -2
View File
@@ -1092,7 +1092,7 @@ class SamplingParams(
try:
validate_xgrammar_grammar(self)
self.structured_outputs._backend = "xgrammar"
except ValueError:
except VLLMValidationError:
# The request either failed validation
# or includes some jsonschema feature(s) that
# are not supported in xgrammar.
@@ -1103,7 +1103,12 @@ class SamplingParams(
so_params = self.structured_outputs
if not skip_guidance and so_params.json:
if isinstance(so_params.json, str):
schema = json_mod.loads(so_params.json)
try:
schema = json_mod.loads(so_params.json)
except json_mod.JSONDecodeError as e:
raise VLLMValidationError(
"Invalid JSON grammar specification."
) from e
else:
schema = so_params.json
skip_guidance = has_guidance_unsupported_json_features(schema)
+10 -4
View File
@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any
import torch
from transformers import MistralCommonBackend
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.sampling_params import SamplingParams
from vllm.utils.import_utils import LazyLoader
@@ -269,7 +270,7 @@ def serialize_guidance_grammar(
begin: str = s["begin"]
trig = next((t for t in triggers if begin.startswith(t)), None)
if trig is None:
raise ValueError(
raise VLLMValidationError(
f"Trigger {begin} not found in triggers {triggers}"
)
tags.append(
@@ -281,7 +282,9 @@ def serialize_guidance_grammar(
)
)
if not tags:
raise ValueError("No structural tags found in the grammar spec.")
raise VLLMValidationError(
"No structural tags found in the grammar spec."
)
return llguidance.StructTag.to_grammar(tags)
else:
logger.error(
@@ -300,7 +303,10 @@ def validate_guidance_grammar(
if sampling_params.structured_outputs is None:
return
tp, grm = get_structured_output_key(sampling_params.structured_outputs)
guidance_grm = serialize_guidance_grammar(tp, grm)
try:
guidance_grm = serialize_guidance_grammar(tp, grm)
except (ValueError, KeyError, TypeError) as e:
raise VLLMValidationError(f"Invalid grammar specification: {e}") from e
err = llguidance.LLMatcher.validate_grammar(guidance_grm, tokenizer)
if err:
raise ValueError(f"Grammar error: {err}")
raise VLLMValidationError(f"Grammar error: {err}")
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING
import torch
from transformers import PreTrainedTokenizerBase
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import SamplingParams
from vllm.utils.import_utils import LazyLoader
from vllm.utils.torch_utils import PIN_MEMORY
@@ -166,7 +167,7 @@ def validate_structured_output_request_lm_format_enforcer(params: SamplingParams
so_params.regex,
)
except Exception as err:
raise ValueError(
raise VLLMValidationError(
f"Failed to compile regex for lm-format-enforcer: {err}"
) from err
return
@@ -176,19 +177,19 @@ def validate_structured_output_request_lm_format_enforcer(params: SamplingParams
# make sure schema is valid json
json.loads(so_params.json)
except json.JSONDecodeError as e:
raise ValueError("Invalid JSON grammar specification.") from e
raise VLLMValidationError("Invalid JSON grammar specification.") from e
else:
try:
json.dumps(so_params.json)
except Exception as e:
raise ValueError(
raise VLLMValidationError(
f"Error serializing structured outputs jsonschema: {e}"
) from e
return
elif so_params.choice:
return
elif so_params.grammar:
raise ValueError(
raise VLLMValidationError(
"LM Format Enforcer structured outputs backend "
"does not support grammar specifications"
)
+13 -7
View File
@@ -13,6 +13,7 @@ from typing import TYPE_CHECKING
import torch
from regex import escape as regex_escape
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import SamplingParams
from vllm.utils.import_utils import LazyLoader
from vllm.utils.torch_utils import PIN_MEMORY
@@ -186,22 +187,27 @@ def validate_structured_output_request_outlines(params: SamplingParams):
json.loads(so_params.json)
schema = so_params.json
except json.JSONDecodeError as e:
raise ValueError("Invalid JSON grammar specification.") from e
raise VLLMValidationError("Invalid JSON grammar specification.") from e
else:
try:
schema = json.dumps(so_params.json)
except Exception as e:
raise ValueError(
raise VLLMValidationError(
f"Error serializing structured outputs jsonschema: {e}"
) from e
pattern = json_schema.build_regex_from_schema(schema)
try:
pattern = json_schema.build_regex_from_schema(schema)
except Exception as e:
raise VLLMValidationError(
f"Failed to transform json schema into a regex: {e}"
) from e
validate_regex_is_buildable(pattern)
elif so_params.choice:
choices = [regex_escape(str(choice)) for choice in so_params.choice]
regex = "(" + "|".join(choices) + ")"
validate_regex_is_buildable(regex)
elif so_params.grammar:
raise ValueError(
raise VLLMValidationError(
"Outlines structured outputs backend "
"does not support grammar specifications"
)
@@ -315,19 +321,19 @@ def validate_regex_is_buildable(pattern: str) -> None:
parsed = sre_parse.parse(pattern)
except sre_constants.error as e:
raise ValueError(f"Error parsing regex: {e}") from e
raise VLLMValidationError(f"Error parsing regex: {e}") from e
try:
_check_unsupported(parsed)
except ValueError as e:
raise ValueError(
raise VLLMValidationError(
f"Regex uses unsupported feature for structured outputs: {e}. "
"Only basic matching constructs are supported—lookarounds, "
"backreferences, and unicode boundaries are not."
) from e
if _prefix_needs_context(parsed):
raise ValueError(
raise VLLMValidationError(
"Regex does not have a anchored universal start state"
"This means that the Regex uses anchors (^) or look-arounds "
"in a way which requires context before any token is matched."
+10 -9
View File
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any
import torch
import vllm.envs
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.sampling_params import SamplingParams
from vllm.utils.import_utils import LazyLoader
@@ -276,7 +277,7 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool:
def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
"""Validate that the request is supported by structured output.
Raises ValueError if the request is not supported.
Raises VLLMValidationError if the request is not supported.
"""
if sampling_params.structured_outputs is None:
return
@@ -298,7 +299,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
so_params.regex,
)
except Exception as err:
raise ValueError(
raise VLLMValidationError(
f"Failed to transform regex into a grammar: {err}"
) from err
@@ -307,7 +308,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
try:
xgr.Grammar.from_ebnf(choice_grammar)
except Exception as err:
raise ValueError(
raise VLLMValidationError(
f"Failed to transform choices into a grammar: {err}"
) from err
so_params.choice = None
@@ -319,19 +320,19 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
try:
schema = json.loads(so_params.json)
except json.JSONDecodeError as e:
raise ValueError("Invalid JSON grammar specification.") from e
raise VLLMValidationError("Invalid JSON grammar specification.") from e
else:
schema = so_params.json
if has_xgrammar_unsupported_json_features(schema):
raise ValueError(
raise VLLMValidationError(
"The provided JSON schema contains features not supported by xgrammar."
)
try:
xgr.Grammar.from_json_schema(schema)
except Exception as err:
raise ValueError(
raise VLLMValidationError(
f"Failed to transform json schema into a grammar: {err}"
) from err
return
@@ -342,7 +343,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
try:
so_params.grammar = convert_lark_to_ebnf(so_params.grammar)
except ValueError as e:
raise ValueError(
raise VLLMValidationError(
"Failed to convert the grammar from Lark to EBNF. "
) from e
@@ -351,7 +352,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
# parse the grammar, but we aren't compiling it.
xgr.Grammar.from_ebnf(so_params.grammar)
except Exception as e:
raise ValueError("Invalid grammar specification.") from e
raise VLLMValidationError("Invalid grammar specification.") from e
return
if so_params.structural_tag:
@@ -372,4 +373,4 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
else:
xgr.Grammar.from_structural_tag(so_params.structural_tag)
except Exception as e:
raise ValueError("Invalid structural tag specification.") from e
raise VLLMValidationError("Invalid structural tag specification.") from e