[CI] Stabilize OpenAI schema fuzzing for malformed structural tags (#44131)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-06-01 19:56:15 -07:00
committed by GitHub
parent 9affc17a05
commit 54d0c36fff
6 changed files with 201 additions and 3 deletions
@@ -6,9 +6,13 @@ from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from pydantic import ValidationError
from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
ChatCompletionRequest,
)
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.entrypoints.openai.models.protocol import BaseModelPath
@@ -444,3 +448,45 @@ def test_json_schema_response_format_missing_schema():
messages=[{"role": "user", "content": "hello"}],
response_format={"type": "json_schema"},
)
@pytest.mark.parametrize("format_value", [None, {}])
def test_structural_tag_response_format_invalid(format_value):
"""Malformed structural tags should be rejected during request validation."""
with pytest.raises(
ValidationError,
match="Invalid response_format structural_tag",
):
ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hello"}],
response_format={"type": "structural_tag", "format": format_value},
)
@pytest.mark.parametrize("format_value", [None, {}])
def test_batch_structural_tag_response_format_invalid(format_value):
"""Batch chat should reject malformed structural tags at request parsing."""
with pytest.raises(
ValidationError,
match="Invalid response_format structural_tag",
):
BatchChatCompletionRequest(
model=MODEL_NAME,
messages=[[{"role": "user", "content": "hello"}]],
response_format={"type": "structural_tag", "format": format_value},
)
@pytest.mark.parametrize("structural_tag", ["not json", ""])
def test_structured_outputs_structural_tag_invalid(structural_tag):
"""Malformed direct structured_outputs structural tags should be rejected."""
with pytest.raises(
ValidationError,
match="Invalid structured_outputs structural_tag",
):
ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hello"}],
structured_outputs={"structural_tag": structural_tag},
)
@@ -6,6 +6,7 @@ from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from pydantic import ValidationError
from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
@@ -302,6 +303,36 @@ def test_json_schema_response_format_missing_schema():
)
@pytest.mark.parametrize("format_value", [None, {}])
def test_structural_tag_response_format_invalid(format_value):
"""Malformed structural tags should be rejected during request validation."""
with pytest.raises(
ValidationError,
match="Invalid response_format structural_tag",
):
CompletionRequest(
model=MODEL_NAME,
prompt="Test prompt",
max_tokens=10,
response_format={"type": "structural_tag", "format": format_value},
)
@pytest.mark.parametrize("structural_tag", ["not json", ""])
def test_structured_outputs_structural_tag_invalid(structural_tag):
"""Malformed direct structured_outputs structural tags should be rejected."""
with pytest.raises(
ValidationError,
match="Invalid structured_outputs structural_tag",
):
CompletionRequest(
model=MODEL_NAME,
prompt="Test prompt",
max_tokens=10,
structured_outputs={"structural_tag": structural_tag},
)
def test_negative_prompt_token_ids_nested():
"""Negative token IDs in prompt (nested list) should raise validation error."""
with pytest.raises(Exception, match="greater than or equal to 0"):
+26 -2
View File
@@ -1382,7 +1382,20 @@ def test_adjust_request_non_mistral_tokenizer(
[
{"regex": r"\d+"},
{"choice": ["a", "b"]},
{"structural_tag": '{"key": "value"}'},
{
"structural_tag": json.dumps(
{
"structures": [
{
"begin": "<tool>",
"schema": {"type": "object"},
"end": "</tool>",
}
],
"triggers": ["<tool>"],
}
)
},
{"grammar": "start: 'hello'"},
],
ids=["regex", "choice", "structural_tag", "grammar"],
@@ -1404,7 +1417,18 @@ def test_adjust_request_unsupported_response_format(
) -> None:
request = _make_request(
response_format=StructuralTagResponseFormat(
type="structural_tag", format={"some": "config"}
type="structural_tag",
format={
"type": "triggered_tags",
"tags": [
{
"begin": "<tool>",
"content": {"type": "any_text"},
"end": "</tool>",
}
],
"triggers": ["<tool>"],
},
),
)
result = mistral_tool_parser.adjust_request(request)