[Bugfix] Raise VLLMValidationError from structured output validators (#52394)
Signed-off-by: Jeffrey Wang <jeffreywang@anyscale.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
8efa13b700
commit
41f12a0daf
@@ -13,6 +13,7 @@ from pydantic import BaseModel
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction
|
||||
from vllm.config import StructuredOutputsConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParserManager
|
||||
@@ -326,7 +327,7 @@ def test_structured_output(
|
||||
)
|
||||
if backend.startswith("xgrammar"):
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
VLLMValidationError,
|
||||
match="The provided JSON schema contains features "
|
||||
"not supported by xgrammar.",
|
||||
):
|
||||
@@ -453,7 +454,9 @@ def test_structured_output(
|
||||
max_tokens=1000,
|
||||
structured_outputs=StructuredOutputsParams(grammar="not a grammar"),
|
||||
)
|
||||
with pytest.raises(ValueError, match="Failed to convert the grammar "):
|
||||
with pytest.raises(
|
||||
VLLMValidationError, match="Failed to convert the grammar "
|
||||
):
|
||||
runner.llm.generate(
|
||||
(
|
||||
"Generate a sql statement that selects col_1 from "
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
import pytest
|
||||
|
||||
from vllm.config import StructuredOutputsConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.exceptions import VLLMClientError, VLLMValidationError
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
@@ -103,3 +103,60 @@ def test_regex_with_nul_byte_rejected(regex):
|
||||
|
||||
with pytest.raises(ValueError, match="NUL"):
|
||||
validate_xgrammar_grammar(params)
|
||||
|
||||
|
||||
INVALID_JSON_SCHEMA = {"type": "object", "properties": {"name": {"type": "str"}}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend, structured_outputs",
|
||||
[
|
||||
("xgrammar", StructuredOutputsParams(json=INVALID_JSON_SCHEMA)),
|
||||
("outlines", StructuredOutputsParams(json=INVALID_JSON_SCHEMA)),
|
||||
("auto", StructuredOutputsParams(json=INVALID_JSON_SCHEMA)),
|
||||
("auto", StructuredOutputsParams(json='{"type": ')),
|
||||
("xgrammar", StructuredOutputsParams(grammar="not a grammar")),
|
||||
("guidance", StructuredOutputsParams(grammar="not a grammar")),
|
||||
("lm-format-enforcer", StructuredOutputsParams(grammar="not a grammar")),
|
||||
("outlines", StructuredOutputsParams(regex="(")),
|
||||
("guidance", StructuredOutputsParams(structural_tag='{"nope": 1}')),
|
||||
],
|
||||
)
|
||||
def test_unsupported_grammar_is_a_client_error(backend, structured_outputs):
|
||||
"""Only `VLLMClientError` survives `AsyncLLM.generate` untouched; anything else
|
||||
is wrapped in `EngineGenerateError` and served as a 500 instead of a 400."""
|
||||
params = SamplingParams(structured_outputs=structured_outputs)
|
||||
with pytest.raises(VLLMClientError):
|
||||
params._validate_structured_outputs(
|
||||
_StubModelConfig(is_diffusion=False),
|
||||
StructuredOutputsConfig(backend=backend),
|
||||
tokenizer=object(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"schema, expected_backend",
|
||||
[
|
||||
# multipleOf is unsupported by xgrammar, patternProperties also by guidance.
|
||||
(
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"n": {"type": "integer", "multipleOf": 2}},
|
||||
},
|
||||
"guidance",
|
||||
),
|
||||
(
|
||||
{"type": "object", "patternProperties": {"^a": {"type": "string"}}},
|
||||
"outlines",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_auto_backend_falls_back_on_unsupported_schema(schema, expected_backend):
|
||||
"""`auto` falls back on rejection, so it must catch what the validators raise."""
|
||||
params = SamplingParams(structured_outputs=StructuredOutputsParams(json=schema))
|
||||
params._validate_structured_outputs(
|
||||
_StubModelConfig(is_diffusion=False),
|
||||
StructuredOutputsConfig(backend="auto"),
|
||||
tokenizer=object(),
|
||||
)
|
||||
assert params.structured_outputs._backend == expected_backend
|
||||
|
||||
@@ -266,7 +266,7 @@ def validate_structural_tag_payload(payload: Any, *, parameter: str) -> None:
|
||||
structured_outputs=StructuredOutputsParams(structural_tag=payload)
|
||||
)
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
except (TypeError, ValueError, VLLMValidationError) as exc:
|
||||
raise VLLMValidationError(
|
||||
f"Invalid {parameter} structural_tag specification.",
|
||||
parameter=parameter,
|
||||
|
||||
@@ -1092,7 +1092,7 @@ class SamplingParams(
|
||||
try:
|
||||
validate_xgrammar_grammar(self)
|
||||
self.structured_outputs._backend = "xgrammar"
|
||||
except ValueError:
|
||||
except VLLMValidationError:
|
||||
# The request either failed validation
|
||||
# or includes some jsonschema feature(s) that
|
||||
# are not supported in xgrammar.
|
||||
@@ -1103,7 +1103,12 @@ class SamplingParams(
|
||||
so_params = self.structured_outputs
|
||||
if not skip_guidance and so_params.json:
|
||||
if isinstance(so_params.json, str):
|
||||
schema = json_mod.loads(so_params.json)
|
||||
try:
|
||||
schema = json_mod.loads(so_params.json)
|
||||
except json_mod.JSONDecodeError as e:
|
||||
raise VLLMValidationError(
|
||||
"Invalid JSON grammar specification."
|
||||
) from e
|
||||
else:
|
||||
schema = so_params.json
|
||||
skip_guidance = has_guidance_unsupported_json_features(schema)
|
||||
|
||||
@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any
|
||||
import torch
|
||||
from transformers import MistralCommonBackend
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
@@ -269,7 +270,7 @@ def serialize_guidance_grammar(
|
||||
begin: str = s["begin"]
|
||||
trig = next((t for t in triggers if begin.startswith(t)), None)
|
||||
if trig is None:
|
||||
raise ValueError(
|
||||
raise VLLMValidationError(
|
||||
f"Trigger {begin} not found in triggers {triggers}"
|
||||
)
|
||||
tags.append(
|
||||
@@ -281,7 +282,9 @@ def serialize_guidance_grammar(
|
||||
)
|
||||
)
|
||||
if not tags:
|
||||
raise ValueError("No structural tags found in the grammar spec.")
|
||||
raise VLLMValidationError(
|
||||
"No structural tags found in the grammar spec."
|
||||
)
|
||||
return llguidance.StructTag.to_grammar(tags)
|
||||
else:
|
||||
logger.error(
|
||||
@@ -300,7 +303,10 @@ def validate_guidance_grammar(
|
||||
if sampling_params.structured_outputs is None:
|
||||
return
|
||||
tp, grm = get_structured_output_key(sampling_params.structured_outputs)
|
||||
guidance_grm = serialize_guidance_grammar(tp, grm)
|
||||
try:
|
||||
guidance_grm = serialize_guidance_grammar(tp, grm)
|
||||
except (ValueError, KeyError, TypeError) as e:
|
||||
raise VLLMValidationError(f"Invalid grammar specification: {e}") from e
|
||||
err = llguidance.LLMatcher.validate_grammar(guidance_grm, tokenizer)
|
||||
if err:
|
||||
raise ValueError(f"Grammar error: {err}")
|
||||
raise VLLMValidationError(f"Grammar error: {err}")
|
||||
|
||||
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
@@ -166,7 +167,7 @@ def validate_structured_output_request_lm_format_enforcer(params: SamplingParams
|
||||
so_params.regex,
|
||||
)
|
||||
except Exception as err:
|
||||
raise ValueError(
|
||||
raise VLLMValidationError(
|
||||
f"Failed to compile regex for lm-format-enforcer: {err}"
|
||||
) from err
|
||||
return
|
||||
@@ -176,19 +177,19 @@ def validate_structured_output_request_lm_format_enforcer(params: SamplingParams
|
||||
# make sure schema is valid json
|
||||
json.loads(so_params.json)
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError("Invalid JSON grammar specification.") from e
|
||||
raise VLLMValidationError("Invalid JSON grammar specification.") from e
|
||||
else:
|
||||
try:
|
||||
json.dumps(so_params.json)
|
||||
except Exception as e:
|
||||
raise ValueError(
|
||||
raise VLLMValidationError(
|
||||
f"Error serializing structured outputs jsonschema: {e}"
|
||||
) from e
|
||||
return
|
||||
elif so_params.choice:
|
||||
return
|
||||
elif so_params.grammar:
|
||||
raise ValueError(
|
||||
raise VLLMValidationError(
|
||||
"LM Format Enforcer structured outputs backend "
|
||||
"does not support grammar specifications"
|
||||
)
|
||||
|
||||
@@ -13,6 +13,7 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
from regex import escape as regex_escape
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
@@ -186,22 +187,27 @@ def validate_structured_output_request_outlines(params: SamplingParams):
|
||||
json.loads(so_params.json)
|
||||
schema = so_params.json
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError("Invalid JSON grammar specification.") from e
|
||||
raise VLLMValidationError("Invalid JSON grammar specification.") from e
|
||||
else:
|
||||
try:
|
||||
schema = json.dumps(so_params.json)
|
||||
except Exception as e:
|
||||
raise ValueError(
|
||||
raise VLLMValidationError(
|
||||
f"Error serializing structured outputs jsonschema: {e}"
|
||||
) from e
|
||||
pattern = json_schema.build_regex_from_schema(schema)
|
||||
try:
|
||||
pattern = json_schema.build_regex_from_schema(schema)
|
||||
except Exception as e:
|
||||
raise VLLMValidationError(
|
||||
f"Failed to transform json schema into a regex: {e}"
|
||||
) from e
|
||||
validate_regex_is_buildable(pattern)
|
||||
elif so_params.choice:
|
||||
choices = [regex_escape(str(choice)) for choice in so_params.choice]
|
||||
regex = "(" + "|".join(choices) + ")"
|
||||
validate_regex_is_buildable(regex)
|
||||
elif so_params.grammar:
|
||||
raise ValueError(
|
||||
raise VLLMValidationError(
|
||||
"Outlines structured outputs backend "
|
||||
"does not support grammar specifications"
|
||||
)
|
||||
@@ -315,19 +321,19 @@ def validate_regex_is_buildable(pattern: str) -> None:
|
||||
parsed = sre_parse.parse(pattern)
|
||||
|
||||
except sre_constants.error as e:
|
||||
raise ValueError(f"Error parsing regex: {e}") from e
|
||||
raise VLLMValidationError(f"Error parsing regex: {e}") from e
|
||||
|
||||
try:
|
||||
_check_unsupported(parsed)
|
||||
except ValueError as e:
|
||||
raise ValueError(
|
||||
raise VLLMValidationError(
|
||||
f"Regex uses unsupported feature for structured outputs: {e}. "
|
||||
"Only basic matching constructs are supported—lookarounds, "
|
||||
"backreferences, and unicode boundaries are not."
|
||||
) from e
|
||||
|
||||
if _prefix_needs_context(parsed):
|
||||
raise ValueError(
|
||||
raise VLLMValidationError(
|
||||
"Regex does not have a anchored universal start state"
|
||||
"This means that the Regex uses anchors (^) or look-arounds "
|
||||
"in a way which requires context before any token is matched."
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any
|
||||
import torch
|
||||
|
||||
import vllm.envs
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
@@ -276,7 +277,7 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool:
|
||||
def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
|
||||
"""Validate that the request is supported by structured output.
|
||||
|
||||
Raises ValueError if the request is not supported.
|
||||
Raises VLLMValidationError if the request is not supported.
|
||||
"""
|
||||
if sampling_params.structured_outputs is None:
|
||||
return
|
||||
@@ -298,7 +299,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
|
||||
so_params.regex,
|
||||
)
|
||||
except Exception as err:
|
||||
raise ValueError(
|
||||
raise VLLMValidationError(
|
||||
f"Failed to transform regex into a grammar: {err}"
|
||||
) from err
|
||||
|
||||
@@ -307,7 +308,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
|
||||
try:
|
||||
xgr.Grammar.from_ebnf(choice_grammar)
|
||||
except Exception as err:
|
||||
raise ValueError(
|
||||
raise VLLMValidationError(
|
||||
f"Failed to transform choices into a grammar: {err}"
|
||||
) from err
|
||||
so_params.choice = None
|
||||
@@ -319,19 +320,19 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
|
||||
try:
|
||||
schema = json.loads(so_params.json)
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError("Invalid JSON grammar specification.") from e
|
||||
raise VLLMValidationError("Invalid JSON grammar specification.") from e
|
||||
else:
|
||||
schema = so_params.json
|
||||
|
||||
if has_xgrammar_unsupported_json_features(schema):
|
||||
raise ValueError(
|
||||
raise VLLMValidationError(
|
||||
"The provided JSON schema contains features not supported by xgrammar."
|
||||
)
|
||||
|
||||
try:
|
||||
xgr.Grammar.from_json_schema(schema)
|
||||
except Exception as err:
|
||||
raise ValueError(
|
||||
raise VLLMValidationError(
|
||||
f"Failed to transform json schema into a grammar: {err}"
|
||||
) from err
|
||||
return
|
||||
@@ -342,7 +343,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
|
||||
try:
|
||||
so_params.grammar = convert_lark_to_ebnf(so_params.grammar)
|
||||
except ValueError as e:
|
||||
raise ValueError(
|
||||
raise VLLMValidationError(
|
||||
"Failed to convert the grammar from Lark to EBNF. "
|
||||
) from e
|
||||
|
||||
@@ -351,7 +352,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
|
||||
# parse the grammar, but we aren't compiling it.
|
||||
xgr.Grammar.from_ebnf(so_params.grammar)
|
||||
except Exception as e:
|
||||
raise ValueError("Invalid grammar specification.") from e
|
||||
raise VLLMValidationError("Invalid grammar specification.") from e
|
||||
return
|
||||
|
||||
if so_params.structural_tag:
|
||||
@@ -372,4 +373,4 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
|
||||
else:
|
||||
xgr.Grammar.from_structural_tag(so_params.structural_tag)
|
||||
except Exception as e:
|
||||
raise ValueError("Invalid structural tag specification.") from e
|
||||
raise VLLMValidationError("Invalid structural tag specification.") from e
|
||||
|
||||
Reference in New Issue
Block a user