[Bugfix][Structured Output] Cap JSON schema nesting to prevent 500s and API-server stalls (#60036)
Signed-off-by: Mazukiri <maihoangduc04@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Flora Feng
parent
fa423b614c
commit
5cd8a2626b
@@ -42,7 +42,11 @@ from vllm.parser.engine.adapters import ParserEngineToolAdapter
|
||||
from vllm.parser.metrics import record_tool_parser_invocation
|
||||
from vllm.parser.utils import count_history_tool_calls
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.sampling_params import (
|
||||
SamplingParams,
|
||||
StructuredOutputsParams,
|
||||
check_json_nesting,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
|
||||
from vllm.tool_parsers.streaming import (
|
||||
@@ -274,6 +278,7 @@ def structured_outputs_to_format(params: StructuredOutputsParams) -> Format | No
|
||||
if params.json is not None:
|
||||
schema = params.json
|
||||
if isinstance(schema, str):
|
||||
check_json_nesting(schema)
|
||||
schema = json.loads(schema)
|
||||
return JSONSchemaFormat(json_schema=schema)
|
||||
if params.regex is not None:
|
||||
@@ -295,6 +300,7 @@ def structured_outputs_to_format(params: StructuredOutputsParams) -> Format | No
|
||||
raise VLLMValidationError("Invalid grammar specification.") from e
|
||||
return GrammarFormat(grammar=grammar)
|
||||
if params.structural_tag is not None:
|
||||
check_json_nesting(params.structural_tag, structural_tag=True)
|
||||
s_tag = json.loads(params.structural_tag)
|
||||
if "structures" in s_tag:
|
||||
# LegacyStructuralTagResponseFormat
|
||||
|
||||
@@ -45,7 +45,7 @@ from vllm.reasoning.cohere_command_reasoning_parser import (
|
||||
BaseCohereCommandReasoningParser,
|
||||
)
|
||||
from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.sampling_params import StructuredOutputsParams, check_json_nesting
|
||||
from vllm.tool_parsers.cohere_command_tool_parser import BaseCohereCommandToolParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -127,6 +127,7 @@ def collect_tool_schema(tool_schema: list[CohereNormalizedTool]) -> str:
|
||||
tool_dictionary: dict[str, str] = {}
|
||||
for tool in tool_schema:
|
||||
tool_name = tool["name"]
|
||||
check_json_nesting(tool["parameters"])
|
||||
tool_parameters = dict(tool["parameters"])
|
||||
json_schema: dict[str, Any] = {
|
||||
"type": "object",
|
||||
@@ -404,6 +405,7 @@ def _schema_dict_from_structured_outputs(
|
||||
if isinstance(raw, str):
|
||||
if not raw.strip():
|
||||
raise ValueError("structured_outputs.json cannot be empty.")
|
||||
check_json_nesting(raw)
|
||||
try:
|
||||
raw = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
|
||||
@@ -48,7 +48,7 @@ from vllm.parser.engine.parser_engine_config import (
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.sampling_params import StructuredOutputsParams, check_json_nesting
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -381,6 +381,7 @@ class MistralParser(ParserEngine):
|
||||
json_schema = _DEFAULT_JSON_SCHEMA
|
||||
elif request.structured_outputs.json is not None:
|
||||
if isinstance(request.structured_outputs.json, str):
|
||||
check_json_nesting(request.structured_outputs.json)
|
||||
json_schema = json.loads(request.structured_outputs.json)
|
||||
else:
|
||||
json_schema = request.structured_outputs.json
|
||||
|
||||
Reference in New Issue
Block a user