[Bugfix][Structured Output] Cap JSON schema nesting to prevent 500s and API-server stalls (#60036)
Signed-off-by: Mazukiri <maihoangduc04@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Flora Feng
parent
fa423b614c
commit
5cd8a2626b
@@ -19,6 +19,7 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.parser.cohere_command import (
|
||||
CohereCommandParser,
|
||||
CohereNormalizedTool,
|
||||
@@ -339,6 +340,14 @@ class TestAdjustRequestFoldFromStructuredOutputs:
|
||||
_make_chat_request(structured_outputs={"json": json_value}),
|
||||
)
|
||||
|
||||
def test_deeply_nested_json_string_rejected(self, parser) -> None:
|
||||
"""json.loads raises RecursionError on a string nested this deeply."""
|
||||
schema = '{"type": "array", "items": ' * 20_000 + "{}" + "}" * 20_000
|
||||
with pytest.raises(VLLMValidationError, match="nested too deeply"):
|
||||
parser.adjust_request(
|
||||
_make_chat_request(structured_outputs={"json": schema}),
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"construct",
|
||||
[
|
||||
@@ -450,6 +459,15 @@ class TestCollectToolSchemaDefs:
|
||||
self._tool_calls_json("get_weather", {"location": {"city": 1}}),
|
||||
)
|
||||
|
||||
def test_deeply_nested_parameters_rejected(self) -> None:
|
||||
"""Tool schemas are compiled here, before request validation, and end up
|
||||
inside an EBNF grammar that the nesting check cannot see later."""
|
||||
parameters = json.loads('{"type": "array", "items": ' * 200 + "{}" + "}" * 200)
|
||||
with pytest.raises(VLLMValidationError, match="nested too deeply"):
|
||||
collect_tool_schema(
|
||||
[CohereNormalizedTool(name="deep", parameters=parameters)]
|
||||
)
|
||||
|
||||
def test_input_parameters_not_mutated(self) -> None:
|
||||
params = json.loads(json.dumps(NESTED_DEFS_PARAMETERS))
|
||||
collect_tool_schema(
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
|
||||
from vllm.parser.mistral import _DEFAULT_JSON_SCHEMA, MistralParser, mistral_config
|
||||
@@ -1890,6 +1891,19 @@ def test_adjust_request_structured_outputs_generates_grammar(
|
||||
assert len(result.structured_outputs.grammar) > 0
|
||||
|
||||
|
||||
def test_adjust_request_deeply_nested_json_string_rejected(
|
||||
mistral_tool_parser: MistralToolParser,
|
||||
) -> None:
|
||||
"""The schema string is parsed here, before request validation, and
|
||||
json.loads raises RecursionError on a string nested this deeply."""
|
||||
schema = '{"type": "array", "items": ' * 20_000 + "{}" + "}" * 20_000
|
||||
request = _make_request(
|
||||
structured_outputs=StructuredOutputsParams(json=schema),
|
||||
)
|
||||
with pytest.raises(VLLMValidationError, match="nested too deeply"):
|
||||
mistral_tool_parser.adjust_request(request)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response_format_kwargs,expected_json_schema",
|
||||
[
|
||||
|
||||
@@ -13,7 +13,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.parser.abstract_parser import DelegatingParser, structured_outputs_to_format
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tool_parsers.abstract_tool_parser import ToolParser
|
||||
from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser
|
||||
@@ -303,3 +304,18 @@ class TestToolChoice_Plus_ResponseFormat:
|
||||
assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
|
||||
assert not _is_grammar_accept_string(grammar, '{"text": "hi"}')
|
||||
mock_warn.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["json", "structural_tag"])
|
||||
def test_structured_outputs_to_format_rejects_deeply_nested_string(field):
|
||||
"""Parsers convert the constraint before request validation, and json.loads
|
||||
raises RecursionError on a string nested this deeply."""
|
||||
schema = '{"type": "array", "items": ' * 20_000 + "{}" + "}" * 20_000
|
||||
value = schema
|
||||
if field == "structural_tag":
|
||||
value = (
|
||||
'{"type": "structural_tag", "format": {"type": "json_schema", '
|
||||
f'"json_schema": {schema}}}}}'
|
||||
)
|
||||
with pytest.raises(VLLMValidationError, match="nested too deeply"):
|
||||
structured_outputs_to_format(StructuredOutputsParams(**{field: value}))
|
||||
|
||||
Reference in New Issue
Block a user