[Bugfix][Structured Output] Cap JSON schema nesting to prevent 500s and API-server stalls (#60036)
Signed-off-by: Mazukiri <maihoangduc04@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Flora Feng
parent
fa423b614c
commit
5cd8a2626b
@@ -33,6 +33,15 @@ MAX_LOGPROB_TOKEN_IDS = 128
|
||||
"""Upper bound on `SamplingParams.logprob_token_ids` list length. Must match
|
||||
the per-request row width allocated by the sampler's `LogprobTokenIdsState`."""
|
||||
|
||||
MAX_STRUCTURED_OUTPUT_JSON_NESTING = 64
|
||||
"""Upper bound on nested objects and arrays in a structured-output JSON schema
|
||||
or structural tag. Grammar conversion runs on the API server and its cost grows
|
||||
steeply with nesting. OpenAI allows 10 levels of schema nesting."""
|
||||
|
||||
STRUCTURAL_TAG_WRAPPER_NESTING = 16
|
||||
"""Extra nesting allowed for a structural tag, which wraps its JSON schemas in
|
||||
tag structure (up to 15 levels for tool-call tags generated by tool parsers)."""
|
||||
|
||||
|
||||
def _verify_num_sequences(value: int, parameter_name: str) -> None:
|
||||
if not isinstance(value, int):
|
||||
@@ -202,6 +211,52 @@ class RequestOutputKind(Enum):
|
||||
FINAL_ONLY = 2
|
||||
|
||||
|
||||
def _exceeds_json_nesting(spec: str | dict[str, Any], limit: int) -> bool:
|
||||
"""Whether a JSON document nests objects and arrays deeper than `limit`."""
|
||||
if isinstance(spec, str):
|
||||
try:
|
||||
spec = json_mod.loads(spec)
|
||||
except RecursionError:
|
||||
return True
|
||||
except ValueError:
|
||||
# Malformed JSON is reported by the backend validation.
|
||||
return False
|
||||
|
||||
level: list[Any] = [spec]
|
||||
for _ in range(limit):
|
||||
level = [
|
||||
child
|
||||
for node in level
|
||||
if isinstance(node, (dict, list))
|
||||
for child in (node.values() if isinstance(node, dict) else node)
|
||||
if isinstance(child, (dict, list))
|
||||
]
|
||||
if not level:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def check_json_nesting(
|
||||
spec: str | dict[str, Any] | None, *, structural_tag: bool = False
|
||||
) -> None:
|
||||
"""Reject a JSON schema nested more than `MAX_STRUCTURED_OUTPUT_JSON_NESTING`
|
||||
levels deep, or a structural tag nested deeper than that plus
|
||||
`STRUCTURAL_TAG_WRAPPER_NESTING`.
|
||||
|
||||
Raises:
|
||||
VLLMValidationError: If `spec` is nested too deeply.
|
||||
|
||||
"""
|
||||
limit = MAX_STRUCTURED_OUTPUT_JSON_NESTING
|
||||
if structural_tag:
|
||||
limit += STRUCTURAL_TAG_WRAPPER_NESTING
|
||||
if spec and _exceeds_json_nesting(spec, limit):
|
||||
raise VLLMValidationError(
|
||||
"The JSON schema is nested too deeply: objects and arrays may nest at "
|
||||
f"most {MAX_STRUCTURED_OUTPUT_JSON_NESTING} levels"
|
||||
)
|
||||
|
||||
|
||||
def _is_non_tekken_mistral(tokenizer: TokenizerLike) -> bool:
|
||||
return is_mistral_tokenizer(tokenizer) and not tokenizer.is_tekken
|
||||
|
||||
@@ -1230,6 +1285,9 @@ class SamplingParams(
|
||||
raise VLLMValidationError(
|
||||
"structured_outputs.json cannot be an empty string"
|
||||
)
|
||||
# Reject deep nesting before any backend walks or converts the schema
|
||||
check_json_nesting(self.structured_outputs.json)
|
||||
check_json_nesting(self.structured_outputs.structural_tag, structural_tag=True)
|
||||
# Reject json_object=False early to avoid engine-side crashes
|
||||
if self.structured_outputs.json_object is False:
|
||||
raise VLLMValidationError(
|
||||
|
||||
Reference in New Issue
Block a user