[Bugfix][Structured Output] Cap JSON schema nesting to prevent 500s and API-server stalls (#60036)

Signed-off-by: Mazukiri <maihoangduc04@gmail.com>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Mai Hoàng Đức
2026-10-06 00:06:59 +00:00
committed by GitHub
co-authored by Flora Feng
parent fa423b614c
commit 5cd8a2626b
11 changed files with 259 additions and 8 deletions
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from dataclasses import dataclass, field
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
@@ -642,7 +643,15 @@ def test_batch_structural_tag_response_format_invalid(format_value):
)
@pytest.mark.parametrize("structural_tag", ["not json", ""])
@pytest.mark.parametrize(
"structural_tag",
[
"not json",
"",
# json.loads raises a plain ValueError, not JSONDecodeError
'{"type": "structural_tag", "x": ' + "1" * 5000 + "}",
],
)
def test_structured_outputs_structural_tag_invalid(structural_tag):
"""Malformed direct structured_outputs structural tags should be rejected."""
with pytest.raises(
@@ -656,6 +665,25 @@ def test_structured_outputs_structural_tag_invalid(structural_tag):
)
@pytest.mark.parametrize("field", ["response_format", "structured_outputs"])
def test_deeply_nested_structural_tag_rejected(field):
"""Structural tags are compiled during request parsing, so a deeply nested
one must be rejected there instead of blocking the server while converting,
and reported as too deep rather than malformed."""
schema = '{"type": "array", "items": ' * 200 + "{}" + "}" * 200
tag = (
'{"type": "structural_tag", "format": {"type": "json_schema", '
f'"json_schema": {schema}}}}}'
)
value = json.loads(tag) if field == "response_format" else {"structural_tag": tag}
with pytest.raises(VLLMValidationError, match="nested too deeply"):
ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hello"}],
**{field: value},
)
@pytest.mark.parametrize("field_name", ["prompt_logprobs", "top_logprobs"])
def test_non_numeric_logprobs_rejected(field_name):
"""A non-numeric logprobs value must be a clean 400 validation error, not a
@@ -19,6 +19,7 @@ from vllm.entrypoints.generate.base.protocol import (
)
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.exceptions import VLLMValidationError
from vllm.parser.cohere_command import (
CohereCommandParser,
CohereNormalizedTool,
@@ -339,6 +340,14 @@ class TestAdjustRequestFoldFromStructuredOutputs:
_make_chat_request(structured_outputs={"json": json_value}),
)
def test_deeply_nested_json_string_rejected(self, parser) -> None:
"""json.loads raises RecursionError on a string nested this deeply."""
schema = '{"type": "array", "items": ' * 20_000 + "{}" + "}" * 20_000
with pytest.raises(VLLMValidationError, match="nested too deeply"):
parser.adjust_request(
_make_chat_request(structured_outputs={"json": schema}),
)
@pytest.mark.parametrize(
"construct",
[
@@ -450,6 +459,15 @@ class TestCollectToolSchemaDefs:
self._tool_calls_json("get_weather", {"location": {"city": 1}}),
)
def test_deeply_nested_parameters_rejected(self) -> None:
"""Tool schemas are compiled here, before request validation, and end up
inside an EBNF grammar that the nesting check cannot see later."""
parameters = json.loads('{"type": "array", "items": ' * 200 + "{}" + "}" * 200)
with pytest.raises(VLLMValidationError, match="nested too deeply"):
collect_tool_schema(
[CohereNormalizedTool(name="deep", parameters=parameters)]
)
def test_input_parameters_not_mutated(self) -> None:
params = json.loads(json.dumps(NESTED_DEFS_PARAMETERS))
collect_tool_schema(
+14
View File
@@ -35,6 +35,7 @@ from vllm.entrypoints.generate.base.protocol import (
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.exceptions import VLLMValidationError
from vllm.parser.engine.events import EventType
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
from vllm.parser.mistral import _DEFAULT_JSON_SCHEMA, MistralParser, mistral_config
@@ -1890,6 +1891,19 @@ def test_adjust_request_structured_outputs_generates_grammar(
assert len(result.structured_outputs.grammar) > 0
def test_adjust_request_deeply_nested_json_string_rejected(
mistral_tool_parser: MistralToolParser,
) -> None:
"""The schema string is parsed here, before request validation, and
json.loads raises RecursionError on a string nested this deeply."""
schema = '{"type": "array", "items": ' * 20_000 + "{}" + "}" * 20_000
request = _make_request(
structured_outputs=StructuredOutputsParams(json=schema),
)
with pytest.raises(VLLMValidationError, match="nested too deeply"):
mistral_tool_parser.adjust_request(request)
@pytest.mark.parametrize(
"response_format_kwargs,expected_json_schema",
[
+17 -1
View File
@@ -13,7 +13,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
)
from vllm.parser.abstract_parser import DelegatingParser
from vllm.exceptions import VLLMValidationError
from vllm.parser.abstract_parser import DelegatingParser, structured_outputs_to_format
from vllm.sampling_params import StructuredOutputsParams
from vllm.tool_parsers.abstract_tool_parser import ToolParser
from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser
@@ -303,3 +304,18 @@ class TestToolChoice_Plus_ResponseFormat:
assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
assert not _is_grammar_accept_string(grammar, '{"text": "hi"}')
mock_warn.assert_called_once()
@pytest.mark.parametrize("field", ["json", "structural_tag"])
def test_structured_outputs_to_format_rejects_deeply_nested_string(field):
"""Parsers convert the constraint before request validation, and json.loads
raises RecursionError on a string nested this deeply."""
schema = '{"type": "array", "items": ' * 20_000 + "{}" + "}" * 20_000
value = schema
if field == "structural_tag":
value = (
'{"type": "structural_tag", "format": {"type": "json_schema", '
f'"json_schema": {schema}}}}}'
)
with pytest.raises(VLLMValidationError, match="nested too deeply"):
structured_outputs_to_format(StructuredOutputsParams(**{field: value}))
+100 -1
View File
@@ -2,11 +2,18 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Request-time validation of structured output requests."""
import json
import pytest
from vllm.config import StructuredOutputsConfig
from vllm.exceptions import VLLMClientError, VLLMValidationError
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.sampling_params import (
MAX_STRUCTURED_OUTPUT_JSON_NESTING,
STRUCTURAL_TAG_WRAPPER_NESTING,
SamplingParams,
StructuredOutputsParams,
)
pytestmark = pytest.mark.cpu_test
@@ -110,6 +117,98 @@ def test_regex_with_nul_byte_rejected(regex):
validate_xgrammar_grammar(params)
def _nested_array_schema(levels: int) -> str:
# Built as text: json.dumps on a deeply nested dict would itself recurse.
wrappers = levels - 1
return (
'{"type": "array", "items": ' * wrappers + '{"type": "string"}' + "}" * wrappers
)
def _structural_tag(schema: str) -> str:
return (
'{"type": "structural_tag", "format": {"type": "json_schema", '
f'"json_schema": {schema}}}}}'
)
@pytest.mark.parametrize(
"structured_outputs",
[
pytest.param(
{"json": _nested_array_schema(MAX_STRUCTURED_OUTPUT_JSON_NESTING + 1)},
id="json-str-over-limit",
),
pytest.param(
{"json": json.loads(_nested_array_schema(2_000))},
id="json-dict-past-recursion-limit",
),
pytest.param(
{"json": _nested_array_schema(200_000)},
id="json-str-past-json-loads-recursion-limit",
),
pytest.param(
{
"structural_tag": _structural_tag(
_nested_array_schema(
MAX_STRUCTURED_OUTPUT_JSON_NESTING
+ STRUCTURAL_TAG_WRAPPER_NESTING
)
)
},
id="structural-tag-over-limit",
),
],
)
@pytest.mark.parametrize(
"backend", ["auto", "xgrammar", "guidance", "outlines", "lm-format-enforcer"]
)
def test_deeply_nested_schema_rejected(structured_outputs, backend):
"""Converting a schema to a grammar gets steeply more expensive with nesting
and runs on the API server; past the recursion limit it used to escape as a
RecursionError (HTTP 500 with an empty message). Deep nesting must be a
validation error before any backend touches the schema."""
params = SamplingParams(
structured_outputs=StructuredOutputsParams(**structured_outputs)
)
with pytest.raises(VLLMValidationError, match="nested too deeply"):
params._validate_structured_outputs(
_StubModelConfig(is_diffusion=False),
StructuredOutputsConfig(backend=backend),
tokenizer=object(),
)
@pytest.mark.parametrize("field", ["json", "structural_tag"])
def test_xgrammar_validation_rejects_deeply_nested_schema(field):
"""Request parsing and tool parsers call validate_xgrammar_grammar directly,
before _validate_structured_outputs, so it must check nesting itself."""
from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
schema = _nested_array_schema(2_000)
value = _structural_tag(schema) if field == "structural_tag" else schema
params = SamplingParams(
structured_outputs=StructuredOutputsParams(**{field: value})
)
with pytest.raises(VLLMValidationError, match="nested too deeply"):
validate_xgrammar_grammar(params)
@pytest.mark.parametrize("field", ["json", "structural_tag"])
def test_schema_at_nesting_limit_accepted(field):
"""A structural tag wraps its schema in a few more levels (more for the
tags tool parsers generate), which must not count against the schema."""
schema = _nested_array_schema(MAX_STRUCTURED_OUTPUT_JSON_NESTING)
value = _structural_tag(schema) if field == "structural_tag" else schema
structured_outputs = StructuredOutputsParams(**{field: value})
SamplingParams(structured_outputs=structured_outputs)._validate_structured_outputs(
_StubModelConfig(is_diffusion=False),
StructuredOutputsConfig(),
tokenizer=object(),
)
assert structured_outputs._backend == "xgrammar"
INVALID_JSON_SCHEMA = {"type": "object", "properties": {"name": {"type": "str"}}}