diff --git a/tests/entrypoints/openai/chat_completion/test_chat_error.py b/tests/entrypoints/openai/chat_completion/test_chat_error.py index baa8bcec59..e467d1bcc5 100644 --- a/tests/entrypoints/openai/chat_completion/test_chat_error.py +++ b/tests/entrypoints/openai/chat_completion/test_chat_error.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json from dataclasses import dataclass, field from typing import Any from unittest.mock import AsyncMock, MagicMock, patch @@ -642,7 +643,15 @@ def test_batch_structural_tag_response_format_invalid(format_value): ) -@pytest.mark.parametrize("structural_tag", ["not json", ""]) +@pytest.mark.parametrize( + "structural_tag", + [ + "not json", + "", + # json.loads raises a plain ValueError, not JSONDecodeError + '{"type": "structural_tag", "x": ' + "1" * 5000 + "}", + ], +) def test_structured_outputs_structural_tag_invalid(structural_tag): """Malformed direct structured_outputs structural tags should be rejected.""" with pytest.raises( @@ -656,6 +665,25 @@ def test_structured_outputs_structural_tag_invalid(structural_tag): ) +@pytest.mark.parametrize("field", ["response_format", "structured_outputs"]) +def test_deeply_nested_structural_tag_rejected(field): + """Structural tags are compiled during request parsing, so a deeply nested + one must be rejected there instead of blocking the server while converting, + and reported as too deep rather than malformed.""" + schema = '{"type": "array", "items": ' * 200 + "{}" + "}" * 200 + tag = ( + '{"type": "structural_tag", "format": {"type": "json_schema", ' + f'"json_schema": {schema}}}}}' + ) + value = json.loads(tag) if field == "response_format" else {"structural_tag": tag} + with pytest.raises(VLLMValidationError, match="nested too deeply"): + ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "hello"}], + **{field: value}, + ) + + @pytest.mark.parametrize("field_name", ["prompt_logprobs", "top_logprobs"]) def test_non_numeric_logprobs_rejected(field_name): """A non-numeric logprobs value must be a clean 400 validation error, not a diff --git a/tests/parser/cohere/test_structural_tags.py b/tests/parser/cohere/test_structural_tags.py index 893cfc80f7..149c5229a7 100644 --- a/tests/parser/cohere/test_structural_tags.py +++ b/tests/parser/cohere/test_structural_tags.py @@ -19,6 +19,7 @@ from vllm.entrypoints.generate.base.protocol import ( ) from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.exceptions import VLLMValidationError from vllm.parser.cohere_command import ( CohereCommandParser, CohereNormalizedTool, @@ -339,6 +340,14 @@ class TestAdjustRequestFoldFromStructuredOutputs: _make_chat_request(structured_outputs={"json": json_value}), ) + def test_deeply_nested_json_string_rejected(self, parser) -> None: + """json.loads raises RecursionError on a string nested this deeply.""" + schema = '{"type": "array", "items": ' * 20_000 + "{}" + "}" * 20_000 + with pytest.raises(VLLMValidationError, match="nested too deeply"): + parser.adjust_request( + _make_chat_request(structured_outputs={"json": schema}), + ) + @pytest.mark.parametrize( "construct", [ @@ -450,6 +459,15 @@ class TestCollectToolSchemaDefs: self._tool_calls_json("get_weather", {"location": {"city": 1}}), ) + def test_deeply_nested_parameters_rejected(self) -> None: + """Tool schemas are compiled here, before request validation, and end up + inside an EBNF grammar that the nesting check cannot see later.""" + parameters = json.loads('{"type": "array", "items": ' * 200 + "{}" + "}" * 200) + with pytest.raises(VLLMValidationError, match="nested too deeply"): + collect_tool_schema( + [CohereNormalizedTool(name="deep", parameters=parameters)] + ) + def test_input_parameters_not_mutated(self) -> None: params = json.loads(json.dumps(NESTED_DEFS_PARAMETERS)) collect_tool_schema( diff --git a/tests/parser/mistral/test_tool_calls.py b/tests/parser/mistral/test_tool_calls.py index 25b41f44c2..a3359dccc6 100644 --- a/tests/parser/mistral/test_tool_calls.py +++ b/tests/parser/mistral/test_tool_calls.py @@ -35,6 +35,7 @@ from vllm.entrypoints.generate.base.protocol import ( from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ) +from vllm.exceptions import VLLMValidationError from vllm.parser.engine.events import EventType from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine from vllm.parser.mistral import _DEFAULT_JSON_SCHEMA, MistralParser, mistral_config @@ -1890,6 +1891,19 @@ def test_adjust_request_structured_outputs_generates_grammar( assert len(result.structured_outputs.grammar) > 0 +def test_adjust_request_deeply_nested_json_string_rejected( + mistral_tool_parser: MistralToolParser, +) -> None: + """The schema string is parsed here, before request validation, and + json.loads raises RecursionError on a string nested this deeply.""" + schema = '{"type": "array", "items": ' * 20_000 + "{}" + "}" * 20_000 + request = _make_request( + structured_outputs=StructuredOutputsParams(json=schema), + ) + with pytest.raises(VLLMValidationError, match="nested too deeply"): + mistral_tool_parser.adjust_request(request) + + @pytest.mark.parametrize( "response_format_kwargs,expected_json_schema", [ diff --git a/tests/parser/test_abstract_parser.py b/tests/parser/test_abstract_parser.py index 56d20417d1..7952ff4114 100644 --- a/tests/parser/test_abstract_parser.py +++ b/tests/parser/test_abstract_parser.py @@ -13,7 +13,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ChatCompletionToolsParam, ) -from vllm.parser.abstract_parser import DelegatingParser +from vllm.exceptions import VLLMValidationError +from vllm.parser.abstract_parser import DelegatingParser, structured_outputs_to_format from vllm.sampling_params import StructuredOutputsParams from vllm.tool_parsers.abstract_tool_parser import ToolParser from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser @@ -303,3 +304,18 @@ class TestToolChoice_Plus_ResponseFormat: assert _is_grammar_accept_string(grammar, self._qwen_tool_call()) assert not _is_grammar_accept_string(grammar, '{"text": "hi"}') mock_warn.assert_called_once() + + +@pytest.mark.parametrize("field", ["json", "structural_tag"]) +def test_structured_outputs_to_format_rejects_deeply_nested_string(field): + """Parsers convert the constraint before request validation, and json.loads + raises RecursionError on a string nested this deeply.""" + schema = '{"type": "array", "items": ' * 20_000 + "{}" + "}" * 20_000 + value = schema + if field == "structural_tag": + value = ( + '{"type": "structural_tag", "format": {"type": "json_schema", ' + f'"json_schema": {schema}}}}}' + ) + with pytest.raises(VLLMValidationError, match="nested too deeply"): + structured_outputs_to_format(StructuredOutputsParams(**{field: value})) diff --git a/tests/v1/structured_output/test_validation.py b/tests/v1/structured_output/test_validation.py index 767153cac7..cd5526c7b7 100644 --- a/tests/v1/structured_output/test_validation.py +++ b/tests/v1/structured_output/test_validation.py @@ -2,11 +2,18 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Request-time validation of structured output requests.""" +import json + import pytest from vllm.config import StructuredOutputsConfig from vllm.exceptions import VLLMClientError, VLLMValidationError -from vllm.sampling_params import SamplingParams, StructuredOutputsParams +from vllm.sampling_params import ( + MAX_STRUCTURED_OUTPUT_JSON_NESTING, + STRUCTURAL_TAG_WRAPPER_NESTING, + SamplingParams, + StructuredOutputsParams, +) pytestmark = pytest.mark.cpu_test @@ -110,6 +117,98 @@ def test_regex_with_nul_byte_rejected(regex): validate_xgrammar_grammar(params) +def _nested_array_schema(levels: int) -> str: + # Built as text: json.dumps on a deeply nested dict would itself recurse. + wrappers = levels - 1 + return ( + '{"type": "array", "items": ' * wrappers + '{"type": "string"}' + "}" * wrappers + ) + + +def _structural_tag(schema: str) -> str: + return ( + '{"type": "structural_tag", "format": {"type": "json_schema", ' + f'"json_schema": {schema}}}}}' + ) + + +@pytest.mark.parametrize( + "structured_outputs", + [ + pytest.param( + {"json": _nested_array_schema(MAX_STRUCTURED_OUTPUT_JSON_NESTING + 1)}, + id="json-str-over-limit", + ), + pytest.param( + {"json": json.loads(_nested_array_schema(2_000))}, + id="json-dict-past-recursion-limit", + ), + pytest.param( + {"json": _nested_array_schema(200_000)}, + id="json-str-past-json-loads-recursion-limit", + ), + pytest.param( + { + "structural_tag": _structural_tag( + _nested_array_schema( + MAX_STRUCTURED_OUTPUT_JSON_NESTING + + STRUCTURAL_TAG_WRAPPER_NESTING + ) + ) + }, + id="structural-tag-over-limit", + ), + ], +) +@pytest.mark.parametrize( + "backend", ["auto", "xgrammar", "guidance", "outlines", "lm-format-enforcer"] +) +def test_deeply_nested_schema_rejected(structured_outputs, backend): + """Converting a schema to a grammar gets steeply more expensive with nesting + and runs on the API server; past the recursion limit it used to escape as a + RecursionError (HTTP 500 with an empty message). Deep nesting must be a + validation error before any backend touches the schema.""" + params = SamplingParams( + structured_outputs=StructuredOutputsParams(**structured_outputs) + ) + with pytest.raises(VLLMValidationError, match="nested too deeply"): + params._validate_structured_outputs( + _StubModelConfig(is_diffusion=False), + StructuredOutputsConfig(backend=backend), + tokenizer=object(), + ) + + +@pytest.mark.parametrize("field", ["json", "structural_tag"]) +def test_xgrammar_validation_rejects_deeply_nested_schema(field): + """Request parsing and tool parsers call validate_xgrammar_grammar directly, + before _validate_structured_outputs, so it must check nesting itself.""" + from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar + + schema = _nested_array_schema(2_000) + value = _structural_tag(schema) if field == "structural_tag" else schema + params = SamplingParams( + structured_outputs=StructuredOutputsParams(**{field: value}) + ) + with pytest.raises(VLLMValidationError, match="nested too deeply"): + validate_xgrammar_grammar(params) + + +@pytest.mark.parametrize("field", ["json", "structural_tag"]) +def test_schema_at_nesting_limit_accepted(field): + """A structural tag wraps its schema in a few more levels (more for the + tags tool parsers generate), which must not count against the schema.""" + schema = _nested_array_schema(MAX_STRUCTURED_OUTPUT_JSON_NESTING) + value = _structural_tag(schema) if field == "structural_tag" else schema + structured_outputs = StructuredOutputsParams(**{field: value}) + SamplingParams(structured_outputs=structured_outputs)._validate_structured_outputs( + _StubModelConfig(is_diffusion=False), + StructuredOutputsConfig(), + tokenizer=object(), + ) + assert structured_outputs._backend == "xgrammar" + + INVALID_JSON_SCHEMA = {"type": "object", "properties": {"name": {"type": "str"}}} diff --git a/vllm/entrypoints/generate/base/protocol.py b/vllm/entrypoints/generate/base/protocol.py index b858ea19a5..89f6816c41 100644 --- a/vllm/entrypoints/generate/base/protocol.py +++ b/vllm/entrypoints/generate/base/protocol.py @@ -230,7 +230,11 @@ def validate_structural_tag_response_format( def validate_structural_tag_payload(payload: Any, *, parameter: str) -> None: - from vllm.sampling_params import SamplingParams, StructuredOutputsParams + from vllm.sampling_params import ( + SamplingParams, + StructuredOutputsParams, + check_json_nesting, + ) from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar if isinstance(payload, str) and not payload: @@ -239,6 +243,9 @@ def validate_structural_tag_payload(payload: Any, *, parameter: str) -> None: parameter=parameter, ) + if isinstance(payload, str): + # Raised here so the error is not reported as a malformed tag below + check_json_nesting(payload, structural_tag=True) try: validate_xgrammar_grammar( SamplingParams( diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index f6b009efde..c28563fabf 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -42,7 +42,11 @@ from vllm.parser.engine.adapters import ParserEngineToolAdapter from vllm.parser.metrics import record_tool_parser_invocation from vllm.parser.utils import count_history_tool_calls from vllm.reasoning.abs_reasoning_parsers import ReasoningParser -from vllm.sampling_params import SamplingParams, StructuredOutputsParams +from vllm.sampling_params import ( + SamplingParams, + StructuredOutputsParams, + check_json_nesting, +) from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser from vllm.tool_parsers.streaming import ( @@ -274,6 +278,7 @@ def structured_outputs_to_format(params: StructuredOutputsParams) -> Format | No if params.json is not None: schema = params.json if isinstance(schema, str): + check_json_nesting(schema) schema = json.loads(schema) return JSONSchemaFormat(json_schema=schema) if params.regex is not None: @@ -295,6 +300,7 @@ def structured_outputs_to_format(params: StructuredOutputsParams) -> Format | No raise VLLMValidationError("Invalid grammar specification.") from e return GrammarFormat(grammar=grammar) if params.structural_tag is not None: + check_json_nesting(params.structural_tag, structural_tag=True) s_tag = json.loads(params.structural_tag) if "structures" in s_tag: # LegacyStructuralTagResponseFormat diff --git a/vllm/parser/cohere_command.py b/vllm/parser/cohere_command.py index a1df491401..97a9d3d6f3 100644 --- a/vllm/parser/cohere_command.py +++ b/vllm/parser/cohere_command.py @@ -45,7 +45,7 @@ from vllm.reasoning.cohere_command_reasoning_parser import ( BaseCohereCommandReasoningParser, ) from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY -from vllm.sampling_params import StructuredOutputsParams +from vllm.sampling_params import StructuredOutputsParams, check_json_nesting from vllm.tool_parsers.cohere_command_tool_parser import BaseCohereCommandToolParser if TYPE_CHECKING: @@ -127,6 +127,7 @@ def collect_tool_schema(tool_schema: list[CohereNormalizedTool]) -> str: tool_dictionary: dict[str, str] = {} for tool in tool_schema: tool_name = tool["name"] + check_json_nesting(tool["parameters"]) tool_parameters = dict(tool["parameters"]) json_schema: dict[str, Any] = { "type": "object", @@ -404,6 +405,7 @@ def _schema_dict_from_structured_outputs( if isinstance(raw, str): if not raw.strip(): raise ValueError("structured_outputs.json cannot be empty.") + check_json_nesting(raw) try: raw = json.loads(raw) except json.JSONDecodeError as e: diff --git a/vllm/parser/mistral.py b/vllm/parser/mistral.py index 247ef2803f..61f133598c 100644 --- a/vllm/parser/mistral.py +++ b/vllm/parser/mistral.py @@ -48,7 +48,7 @@ from vllm.parser.engine.parser_engine_config import ( ParserState, Transition, ) -from vllm.sampling_params import StructuredOutputsParams +from vllm.sampling_params import StructuredOutputsParams, check_json_nesting from vllm.utils.mistral import is_mistral_tokenizer if TYPE_CHECKING: @@ -381,6 +381,7 @@ class MistralParser(ParserEngine): json_schema = _DEFAULT_JSON_SCHEMA elif request.structured_outputs.json is not None: if isinstance(request.structured_outputs.json, str): + check_json_nesting(request.structured_outputs.json) json_schema = json.loads(request.structured_outputs.json) else: json_schema = request.structured_outputs.json diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index ab9f271114..8a37c5a3fd 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -33,6 +33,15 @@ MAX_LOGPROB_TOKEN_IDS = 128 """Upper bound on `SamplingParams.logprob_token_ids` list length. Must match the per-request row width allocated by the sampler's `LogprobTokenIdsState`.""" +MAX_STRUCTURED_OUTPUT_JSON_NESTING = 64 +"""Upper bound on nested objects and arrays in a structured-output JSON schema +or structural tag. Grammar conversion runs on the API server and its cost grows +steeply with nesting. OpenAI allows 10 levels of schema nesting.""" + +STRUCTURAL_TAG_WRAPPER_NESTING = 16 +"""Extra nesting allowed for a structural tag, which wraps its JSON schemas in +tag structure (up to 15 levels for tool-call tags generated by tool parsers).""" + def _verify_num_sequences(value: int, parameter_name: str) -> None: if not isinstance(value, int): @@ -202,6 +211,52 @@ class RequestOutputKind(Enum): FINAL_ONLY = 2 +def _exceeds_json_nesting(spec: str | dict[str, Any], limit: int) -> bool: + """Whether a JSON document nests objects and arrays deeper than `limit`.""" + if isinstance(spec, str): + try: + spec = json_mod.loads(spec) + except RecursionError: + return True + except ValueError: + # Malformed JSON is reported by the backend validation. + return False + + level: list[Any] = [spec] + for _ in range(limit): + level = [ + child + for node in level + if isinstance(node, (dict, list)) + for child in (node.values() if isinstance(node, dict) else node) + if isinstance(child, (dict, list)) + ] + if not level: + return False + return True + + +def check_json_nesting( + spec: str | dict[str, Any] | None, *, structural_tag: bool = False +) -> None: + """Reject a JSON schema nested more than `MAX_STRUCTURED_OUTPUT_JSON_NESTING` + levels deep, or a structural tag nested deeper than that plus + `STRUCTURAL_TAG_WRAPPER_NESTING`. + + Raises: + VLLMValidationError: If `spec` is nested too deeply. + + """ + limit = MAX_STRUCTURED_OUTPUT_JSON_NESTING + if structural_tag: + limit += STRUCTURAL_TAG_WRAPPER_NESTING + if spec and _exceeds_json_nesting(spec, limit): + raise VLLMValidationError( + "The JSON schema is nested too deeply: objects and arrays may nest at " + f"most {MAX_STRUCTURED_OUTPUT_JSON_NESTING} levels" + ) + + def _is_non_tekken_mistral(tokenizer: TokenizerLike) -> bool: return is_mistral_tokenizer(tokenizer) and not tokenizer.is_tekken @@ -1230,6 +1285,9 @@ class SamplingParams( raise VLLMValidationError( "structured_outputs.json cannot be an empty string" ) + # Reject deep nesting before any backend walks or converts the schema + check_json_nesting(self.structured_outputs.json) + check_json_nesting(self.structured_outputs.structural_tag, structural_tag=True) # Reject json_object=False early to avoid engine-side crashes if self.structured_outputs.json_object is False: raise VLLMValidationError( diff --git a/vllm/v1/structured_output/backend_xgrammar.py b/vllm/v1/structured_output/backend_xgrammar.py index 337b2285f0..4697b00487 100644 --- a/vllm/v1/structured_output/backend_xgrammar.py +++ b/vllm/v1/structured_output/backend_xgrammar.py @@ -10,7 +10,7 @@ import torch import vllm.envs from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger -from vllm.sampling_params import SamplingParams +from vllm.sampling_params import SamplingParams, check_json_nesting from vllm.utils.import_utils import LazyLoader from vllm.utils.mistral import is_mistral_tokenizer from vllm.v1.structured_output.backend_types import ( @@ -392,6 +392,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: return if so_params.json: + check_json_nesting(so_params.json) if isinstance(so_params.json, str): try: schema = json.loads(so_params.json) @@ -426,6 +427,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: return if so_params.structural_tag: + check_json_nesting(so_params.structural_tag, structural_tag=True) try: s_tag = json.loads(so_params.structural_tag)