[Bugfix][Structured Output] Cap JSON schema nesting to prevent 500s and API-server stalls (#60036)

Signed-off-by: Mazukiri <maihoangduc04@gmail.com>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Mai Hoàng Đức
2026-10-06 00:06:59 +00:00
committed by GitHub
co-authored by Flora Feng
parent fa423b614c
commit 5cd8a2626b
11 changed files with 259 additions and 8 deletions
+58
View File
@@ -33,6 +33,15 @@ MAX_LOGPROB_TOKEN_IDS = 128
"""Upper bound on `SamplingParams.logprob_token_ids` list length. Must match
the per-request row width allocated by the sampler's `LogprobTokenIdsState`."""
MAX_STRUCTURED_OUTPUT_JSON_NESTING = 64
"""Upper bound on nested objects and arrays in a structured-output JSON schema
or structural tag. Grammar conversion runs on the API server and its cost grows
steeply with nesting. OpenAI allows 10 levels of schema nesting."""
STRUCTURAL_TAG_WRAPPER_NESTING = 16
"""Extra nesting allowed for a structural tag, which wraps its JSON schemas in
tag structure (up to 15 levels for tool-call tags generated by tool parsers)."""
def _verify_num_sequences(value: int, parameter_name: str) -> None:
if not isinstance(value, int):
@@ -202,6 +211,52 @@ class RequestOutputKind(Enum):
FINAL_ONLY = 2
def _exceeds_json_nesting(spec: str | dict[str, Any], limit: int) -> bool:
"""Whether a JSON document nests objects and arrays deeper than `limit`."""
if isinstance(spec, str):
try:
spec = json_mod.loads(spec)
except RecursionError:
return True
except ValueError:
# Malformed JSON is reported by the backend validation.
return False
level: list[Any] = [spec]
for _ in range(limit):
level = [
child
for node in level
if isinstance(node, (dict, list))
for child in (node.values() if isinstance(node, dict) else node)
if isinstance(child, (dict, list))
]
if not level:
return False
return True
def check_json_nesting(
spec: str | dict[str, Any] | None, *, structural_tag: bool = False
) -> None:
"""Reject a JSON schema nested more than `MAX_STRUCTURED_OUTPUT_JSON_NESTING`
levels deep, or a structural tag nested deeper than that plus
`STRUCTURAL_TAG_WRAPPER_NESTING`.
Raises:
VLLMValidationError: If `spec` is nested too deeply.
"""
limit = MAX_STRUCTURED_OUTPUT_JSON_NESTING
if structural_tag:
limit += STRUCTURAL_TAG_WRAPPER_NESTING
if spec and _exceeds_json_nesting(spec, limit):
raise VLLMValidationError(
"The JSON schema is nested too deeply: objects and arrays may nest at "
f"most {MAX_STRUCTURED_OUTPUT_JSON_NESTING} levels"
)
def _is_non_tekken_mistral(tokenizer: TokenizerLike) -> bool:
return is_mistral_tokenizer(tokenizer) and not tokenizer.is_tekken
@@ -1230,6 +1285,9 @@ class SamplingParams(
raise VLLMValidationError(
"structured_outputs.json cannot be an empty string"
)
# Reject deep nesting before any backend walks or converts the schema
check_json_nesting(self.structured_outputs.json)
check_json_nesting(self.structured_outputs.structural_tag, structural_tag=True)
# Reject json_object=False early to avoid engine-side crashes
if self.structured_outputs.json_object is False:
raise VLLMValidationError(