[Bugfix][Structured Output] Cap JSON schema nesting to prevent 500s and API-server stalls (#60036)
Signed-off-by: Mazukiri <maihoangduc04@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Flora Feng
parent
fa423b614c
commit
5cd8a2626b
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
@@ -642,7 +643,15 @@ def test_batch_structural_tag_response_format_invalid(format_value):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("structural_tag", ["not json", ""])
|
||||
@pytest.mark.parametrize(
|
||||
"structural_tag",
|
||||
[
|
||||
"not json",
|
||||
"",
|
||||
# json.loads raises a plain ValueError, not JSONDecodeError
|
||||
'{"type": "structural_tag", "x": ' + "1" * 5000 + "}",
|
||||
],
|
||||
)
|
||||
def test_structured_outputs_structural_tag_invalid(structural_tag):
|
||||
"""Malformed direct structured_outputs structural tags should be rejected."""
|
||||
with pytest.raises(
|
||||
@@ -656,6 +665,25 @@ def test_structured_outputs_structural_tag_invalid(structural_tag):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["response_format", "structured_outputs"])
|
||||
def test_deeply_nested_structural_tag_rejected(field):
|
||||
"""Structural tags are compiled during request parsing, so a deeply nested
|
||||
one must be rejected there instead of blocking the server while converting,
|
||||
and reported as too deep rather than malformed."""
|
||||
schema = '{"type": "array", "items": ' * 200 + "{}" + "}" * 200
|
||||
tag = (
|
||||
'{"type": "structural_tag", "format": {"type": "json_schema", '
|
||||
f'"json_schema": {schema}}}}}'
|
||||
)
|
||||
value = json.loads(tag) if field == "response_format" else {"structural_tag": tag}
|
||||
with pytest.raises(VLLMValidationError, match="nested too deeply"):
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
**{field: value},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field_name", ["prompt_logprobs", "top_logprobs"])
|
||||
def test_non_numeric_logprobs_rejected(field_name):
|
||||
"""A non-numeric logprobs value must be a clean 400 validation error, not a
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.parser.cohere_command import (
|
||||
CohereCommandParser,
|
||||
CohereNormalizedTool,
|
||||
@@ -339,6 +340,14 @@ class TestAdjustRequestFoldFromStructuredOutputs:
|
||||
_make_chat_request(structured_outputs={"json": json_value}),
|
||||
)
|
||||
|
||||
def test_deeply_nested_json_string_rejected(self, parser) -> None:
|
||||
"""json.loads raises RecursionError on a string nested this deeply."""
|
||||
schema = '{"type": "array", "items": ' * 20_000 + "{}" + "}" * 20_000
|
||||
with pytest.raises(VLLMValidationError, match="nested too deeply"):
|
||||
parser.adjust_request(
|
||||
_make_chat_request(structured_outputs={"json": schema}),
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"construct",
|
||||
[
|
||||
@@ -450,6 +459,15 @@ class TestCollectToolSchemaDefs:
|
||||
self._tool_calls_json("get_weather", {"location": {"city": 1}}),
|
||||
)
|
||||
|
||||
def test_deeply_nested_parameters_rejected(self) -> None:
|
||||
"""Tool schemas are compiled here, before request validation, and end up
|
||||
inside an EBNF grammar that the nesting check cannot see later."""
|
||||
parameters = json.loads('{"type": "array", "items": ' * 200 + "{}" + "}" * 200)
|
||||
with pytest.raises(VLLMValidationError, match="nested too deeply"):
|
||||
collect_tool_schema(
|
||||
[CohereNormalizedTool(name="deep", parameters=parameters)]
|
||||
)
|
||||
|
||||
def test_input_parameters_not_mutated(self) -> None:
|
||||
params = json.loads(json.dumps(NESTED_DEFS_PARAMETERS))
|
||||
collect_tool_schema(
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
|
||||
from vllm.parser.mistral import _DEFAULT_JSON_SCHEMA, MistralParser, mistral_config
|
||||
@@ -1890,6 +1891,19 @@ def test_adjust_request_structured_outputs_generates_grammar(
|
||||
assert len(result.structured_outputs.grammar) > 0
|
||||
|
||||
|
||||
def test_adjust_request_deeply_nested_json_string_rejected(
|
||||
mistral_tool_parser: MistralToolParser,
|
||||
) -> None:
|
||||
"""The schema string is parsed here, before request validation, and
|
||||
json.loads raises RecursionError on a string nested this deeply."""
|
||||
schema = '{"type": "array", "items": ' * 20_000 + "{}" + "}" * 20_000
|
||||
request = _make_request(
|
||||
structured_outputs=StructuredOutputsParams(json=schema),
|
||||
)
|
||||
with pytest.raises(VLLMValidationError, match="nested too deeply"):
|
||||
mistral_tool_parser.adjust_request(request)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response_format_kwargs,expected_json_schema",
|
||||
[
|
||||
|
||||
@@ -13,7 +13,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.parser.abstract_parser import DelegatingParser, structured_outputs_to_format
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tool_parsers.abstract_tool_parser import ToolParser
|
||||
from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser
|
||||
@@ -303,3 +304,18 @@ class TestToolChoice_Plus_ResponseFormat:
|
||||
assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
|
||||
assert not _is_grammar_accept_string(grammar, '{"text": "hi"}')
|
||||
mock_warn.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["json", "structural_tag"])
|
||||
def test_structured_outputs_to_format_rejects_deeply_nested_string(field):
|
||||
"""Parsers convert the constraint before request validation, and json.loads
|
||||
raises RecursionError on a string nested this deeply."""
|
||||
schema = '{"type": "array", "items": ' * 20_000 + "{}" + "}" * 20_000
|
||||
value = schema
|
||||
if field == "structural_tag":
|
||||
value = (
|
||||
'{"type": "structural_tag", "format": {"type": "json_schema", '
|
||||
f'"json_schema": {schema}}}}}'
|
||||
)
|
||||
with pytest.raises(VLLMValidationError, match="nested too deeply"):
|
||||
structured_outputs_to_format(StructuredOutputsParams(**{field: value}))
|
||||
|
||||
@@ -2,11 +2,18 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Request-time validation of structured output requests."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import StructuredOutputsConfig
|
||||
from vllm.exceptions import VLLMClientError, VLLMValidationError
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.sampling_params import (
|
||||
MAX_STRUCTURED_OUTPUT_JSON_NESTING,
|
||||
STRUCTURAL_TAG_WRAPPER_NESTING,
|
||||
SamplingParams,
|
||||
StructuredOutputsParams,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -110,6 +117,98 @@ def test_regex_with_nul_byte_rejected(regex):
|
||||
validate_xgrammar_grammar(params)
|
||||
|
||||
|
||||
def _nested_array_schema(levels: int) -> str:
|
||||
# Built as text: json.dumps on a deeply nested dict would itself recurse.
|
||||
wrappers = levels - 1
|
||||
return (
|
||||
'{"type": "array", "items": ' * wrappers + '{"type": "string"}' + "}" * wrappers
|
||||
)
|
||||
|
||||
|
||||
def _structural_tag(schema: str) -> str:
|
||||
return (
|
||||
'{"type": "structural_tag", "format": {"type": "json_schema", '
|
||||
f'"json_schema": {schema}}}}}'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"structured_outputs",
|
||||
[
|
||||
pytest.param(
|
||||
{"json": _nested_array_schema(MAX_STRUCTURED_OUTPUT_JSON_NESTING + 1)},
|
||||
id="json-str-over-limit",
|
||||
),
|
||||
pytest.param(
|
||||
{"json": json.loads(_nested_array_schema(2_000))},
|
||||
id="json-dict-past-recursion-limit",
|
||||
),
|
||||
pytest.param(
|
||||
{"json": _nested_array_schema(200_000)},
|
||||
id="json-str-past-json-loads-recursion-limit",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"structural_tag": _structural_tag(
|
||||
_nested_array_schema(
|
||||
MAX_STRUCTURED_OUTPUT_JSON_NESTING
|
||||
+ STRUCTURAL_TAG_WRAPPER_NESTING
|
||||
)
|
||||
)
|
||||
},
|
||||
id="structural-tag-over-limit",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"backend", ["auto", "xgrammar", "guidance", "outlines", "lm-format-enforcer"]
|
||||
)
|
||||
def test_deeply_nested_schema_rejected(structured_outputs, backend):
|
||||
"""Converting a schema to a grammar gets steeply more expensive with nesting
|
||||
and runs on the API server; past the recursion limit it used to escape as a
|
||||
RecursionError (HTTP 500 with an empty message). Deep nesting must be a
|
||||
validation error before any backend touches the schema."""
|
||||
params = SamplingParams(
|
||||
structured_outputs=StructuredOutputsParams(**structured_outputs)
|
||||
)
|
||||
with pytest.raises(VLLMValidationError, match="nested too deeply"):
|
||||
params._validate_structured_outputs(
|
||||
_StubModelConfig(is_diffusion=False),
|
||||
StructuredOutputsConfig(backend=backend),
|
||||
tokenizer=object(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["json", "structural_tag"])
|
||||
def test_xgrammar_validation_rejects_deeply_nested_schema(field):
|
||||
"""Request parsing and tool parsers call validate_xgrammar_grammar directly,
|
||||
before _validate_structured_outputs, so it must check nesting itself."""
|
||||
from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
|
||||
|
||||
schema = _nested_array_schema(2_000)
|
||||
value = _structural_tag(schema) if field == "structural_tag" else schema
|
||||
params = SamplingParams(
|
||||
structured_outputs=StructuredOutputsParams(**{field: value})
|
||||
)
|
||||
with pytest.raises(VLLMValidationError, match="nested too deeply"):
|
||||
validate_xgrammar_grammar(params)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["json", "structural_tag"])
|
||||
def test_schema_at_nesting_limit_accepted(field):
|
||||
"""A structural tag wraps its schema in a few more levels (more for the
|
||||
tags tool parsers generate), which must not count against the schema."""
|
||||
schema = _nested_array_schema(MAX_STRUCTURED_OUTPUT_JSON_NESTING)
|
||||
value = _structural_tag(schema) if field == "structural_tag" else schema
|
||||
structured_outputs = StructuredOutputsParams(**{field: value})
|
||||
SamplingParams(structured_outputs=structured_outputs)._validate_structured_outputs(
|
||||
_StubModelConfig(is_diffusion=False),
|
||||
StructuredOutputsConfig(),
|
||||
tokenizer=object(),
|
||||
)
|
||||
assert structured_outputs._backend == "xgrammar"
|
||||
|
||||
|
||||
INVALID_JSON_SCHEMA = {"type": "object", "properties": {"name": {"type": "str"}}}
|
||||
|
||||
|
||||
|
||||
@@ -230,7 +230,11 @@ def validate_structural_tag_response_format(
|
||||
|
||||
|
||||
def validate_structural_tag_payload(payload: Any, *, parameter: str) -> None:
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.sampling_params import (
|
||||
SamplingParams,
|
||||
StructuredOutputsParams,
|
||||
check_json_nesting,
|
||||
)
|
||||
from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
|
||||
|
||||
if isinstance(payload, str) and not payload:
|
||||
@@ -239,6 +243,9 @@ def validate_structural_tag_payload(payload: Any, *, parameter: str) -> None:
|
||||
parameter=parameter,
|
||||
)
|
||||
|
||||
if isinstance(payload, str):
|
||||
# Raised here so the error is not reported as a malformed tag below
|
||||
check_json_nesting(payload, structural_tag=True)
|
||||
try:
|
||||
validate_xgrammar_grammar(
|
||||
SamplingParams(
|
||||
|
||||
@@ -42,7 +42,11 @@ from vllm.parser.engine.adapters import ParserEngineToolAdapter
|
||||
from vllm.parser.metrics import record_tool_parser_invocation
|
||||
from vllm.parser.utils import count_history_tool_calls
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.sampling_params import (
|
||||
SamplingParams,
|
||||
StructuredOutputsParams,
|
||||
check_json_nesting,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
|
||||
from vllm.tool_parsers.streaming import (
|
||||
@@ -274,6 +278,7 @@ def structured_outputs_to_format(params: StructuredOutputsParams) -> Format | No
|
||||
if params.json is not None:
|
||||
schema = params.json
|
||||
if isinstance(schema, str):
|
||||
check_json_nesting(schema)
|
||||
schema = json.loads(schema)
|
||||
return JSONSchemaFormat(json_schema=schema)
|
||||
if params.regex is not None:
|
||||
@@ -295,6 +300,7 @@ def structured_outputs_to_format(params: StructuredOutputsParams) -> Format | No
|
||||
raise VLLMValidationError("Invalid grammar specification.") from e
|
||||
return GrammarFormat(grammar=grammar)
|
||||
if params.structural_tag is not None:
|
||||
check_json_nesting(params.structural_tag, structural_tag=True)
|
||||
s_tag = json.loads(params.structural_tag)
|
||||
if "structures" in s_tag:
|
||||
# LegacyStructuralTagResponseFormat
|
||||
|
||||
@@ -45,7 +45,7 @@ from vllm.reasoning.cohere_command_reasoning_parser import (
|
||||
BaseCohereCommandReasoningParser,
|
||||
)
|
||||
from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.sampling_params import StructuredOutputsParams, check_json_nesting
|
||||
from vllm.tool_parsers.cohere_command_tool_parser import BaseCohereCommandToolParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -127,6 +127,7 @@ def collect_tool_schema(tool_schema: list[CohereNormalizedTool]) -> str:
|
||||
tool_dictionary: dict[str, str] = {}
|
||||
for tool in tool_schema:
|
||||
tool_name = tool["name"]
|
||||
check_json_nesting(tool["parameters"])
|
||||
tool_parameters = dict(tool["parameters"])
|
||||
json_schema: dict[str, Any] = {
|
||||
"type": "object",
|
||||
@@ -404,6 +405,7 @@ def _schema_dict_from_structured_outputs(
|
||||
if isinstance(raw, str):
|
||||
if not raw.strip():
|
||||
raise ValueError("structured_outputs.json cannot be empty.")
|
||||
check_json_nesting(raw)
|
||||
try:
|
||||
raw = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
|
||||
@@ -48,7 +48,7 @@ from vllm.parser.engine.parser_engine_config import (
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.sampling_params import StructuredOutputsParams, check_json_nesting
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -381,6 +381,7 @@ class MistralParser(ParserEngine):
|
||||
json_schema = _DEFAULT_JSON_SCHEMA
|
||||
elif request.structured_outputs.json is not None:
|
||||
if isinstance(request.structured_outputs.json, str):
|
||||
check_json_nesting(request.structured_outputs.json)
|
||||
json_schema = json.loads(request.structured_outputs.json)
|
||||
else:
|
||||
json_schema = request.structured_outputs.json
|
||||
|
||||
@@ -33,6 +33,15 @@ MAX_LOGPROB_TOKEN_IDS = 128
|
||||
"""Upper bound on `SamplingParams.logprob_token_ids` list length. Must match
|
||||
the per-request row width allocated by the sampler's `LogprobTokenIdsState`."""
|
||||
|
||||
MAX_STRUCTURED_OUTPUT_JSON_NESTING = 64
|
||||
"""Upper bound on nested objects and arrays in a structured-output JSON schema
|
||||
or structural tag. Grammar conversion runs on the API server and its cost grows
|
||||
steeply with nesting. OpenAI allows 10 levels of schema nesting."""
|
||||
|
||||
STRUCTURAL_TAG_WRAPPER_NESTING = 16
|
||||
"""Extra nesting allowed for a structural tag, which wraps its JSON schemas in
|
||||
tag structure (up to 15 levels for tool-call tags generated by tool parsers)."""
|
||||
|
||||
|
||||
def _verify_num_sequences(value: int, parameter_name: str) -> None:
|
||||
if not isinstance(value, int):
|
||||
@@ -202,6 +211,52 @@ class RequestOutputKind(Enum):
|
||||
FINAL_ONLY = 2
|
||||
|
||||
|
||||
def _exceeds_json_nesting(spec: str | dict[str, Any], limit: int) -> bool:
|
||||
"""Whether a JSON document nests objects and arrays deeper than `limit`."""
|
||||
if isinstance(spec, str):
|
||||
try:
|
||||
spec = json_mod.loads(spec)
|
||||
except RecursionError:
|
||||
return True
|
||||
except ValueError:
|
||||
# Malformed JSON is reported by the backend validation.
|
||||
return False
|
||||
|
||||
level: list[Any] = [spec]
|
||||
for _ in range(limit):
|
||||
level = [
|
||||
child
|
||||
for node in level
|
||||
if isinstance(node, (dict, list))
|
||||
for child in (node.values() if isinstance(node, dict) else node)
|
||||
if isinstance(child, (dict, list))
|
||||
]
|
||||
if not level:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def check_json_nesting(
|
||||
spec: str | dict[str, Any] | None, *, structural_tag: bool = False
|
||||
) -> None:
|
||||
"""Reject a JSON schema nested more than `MAX_STRUCTURED_OUTPUT_JSON_NESTING`
|
||||
levels deep, or a structural tag nested deeper than that plus
|
||||
`STRUCTURAL_TAG_WRAPPER_NESTING`.
|
||||
|
||||
Raises:
|
||||
VLLMValidationError: If `spec` is nested too deeply.
|
||||
|
||||
"""
|
||||
limit = MAX_STRUCTURED_OUTPUT_JSON_NESTING
|
||||
if structural_tag:
|
||||
limit += STRUCTURAL_TAG_WRAPPER_NESTING
|
||||
if spec and _exceeds_json_nesting(spec, limit):
|
||||
raise VLLMValidationError(
|
||||
"The JSON schema is nested too deeply: objects and arrays may nest at "
|
||||
f"most {MAX_STRUCTURED_OUTPUT_JSON_NESTING} levels"
|
||||
)
|
||||
|
||||
|
||||
def _is_non_tekken_mistral(tokenizer: TokenizerLike) -> bool:
|
||||
return is_mistral_tokenizer(tokenizer) and not tokenizer.is_tekken
|
||||
|
||||
@@ -1230,6 +1285,9 @@ class SamplingParams(
|
||||
raise VLLMValidationError(
|
||||
"structured_outputs.json cannot be an empty string"
|
||||
)
|
||||
# Reject deep nesting before any backend walks or converts the schema
|
||||
check_json_nesting(self.structured_outputs.json)
|
||||
check_json_nesting(self.structured_outputs.structural_tag, structural_tag=True)
|
||||
# Reject json_object=False early to avoid engine-side crashes
|
||||
if self.structured_outputs.json_object is False:
|
||||
raise VLLMValidationError(
|
||||
|
||||
@@ -10,7 +10,7 @@ import torch
|
||||
import vllm.envs
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.sampling_params import SamplingParams, check_json_nesting
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
from vllm.v1.structured_output.backend_types import (
|
||||
@@ -392,6 +392,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
|
||||
return
|
||||
|
||||
if so_params.json:
|
||||
check_json_nesting(so_params.json)
|
||||
if isinstance(so_params.json, str):
|
||||
try:
|
||||
schema = json.loads(so_params.json)
|
||||
@@ -426,6 +427,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
|
||||
return
|
||||
|
||||
if so_params.structural_tag:
|
||||
check_json_nesting(so_params.structural_tag, structural_tag=True)
|
||||
try:
|
||||
s_tag = json.loads(so_params.structural_tag)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user