[CI] Stabilize OpenAI schema fuzzing for malformed structural tags (#44131)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -6,9 +6,13 @@ from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
@@ -444,3 +448,45 @@ def test_json_schema_response_format_missing_schema():
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
response_format={"type": "json_schema"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("format_value", [None, {}])
|
||||
def test_structural_tag_response_format_invalid(format_value):
|
||||
"""Malformed structural tags should be rejected during request validation."""
|
||||
with pytest.raises(
|
||||
ValidationError,
|
||||
match="Invalid response_format structural_tag",
|
||||
):
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
response_format={"type": "structural_tag", "format": format_value},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("format_value", [None, {}])
|
||||
def test_batch_structural_tag_response_format_invalid(format_value):
|
||||
"""Batch chat should reject malformed structural tags at request parsing."""
|
||||
with pytest.raises(
|
||||
ValidationError,
|
||||
match="Invalid response_format structural_tag",
|
||||
):
|
||||
BatchChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[[{"role": "user", "content": "hello"}]],
|
||||
response_format={"type": "structural_tag", "format": format_value},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("structural_tag", ["not json", ""])
|
||||
def test_structured_outputs_structural_tag_invalid(structural_tag):
|
||||
"""Malformed direct structured_outputs structural tags should be rejected."""
|
||||
with pytest.raises(
|
||||
ValidationError,
|
||||
match="Invalid structured_outputs structural_tag",
|
||||
):
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
structured_outputs={"structural_tag": structural_tag},
|
||||
)
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
@@ -302,6 +303,36 @@ def test_json_schema_response_format_missing_schema():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("format_value", [None, {}])
|
||||
def test_structural_tag_response_format_invalid(format_value):
|
||||
"""Malformed structural tags should be rejected during request validation."""
|
||||
with pytest.raises(
|
||||
ValidationError,
|
||||
match="Invalid response_format structural_tag",
|
||||
):
|
||||
CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="Test prompt",
|
||||
max_tokens=10,
|
||||
response_format={"type": "structural_tag", "format": format_value},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("structural_tag", ["not json", ""])
|
||||
def test_structured_outputs_structural_tag_invalid(structural_tag):
|
||||
"""Malformed direct structured_outputs structural tags should be rejected."""
|
||||
with pytest.raises(
|
||||
ValidationError,
|
||||
match="Invalid structured_outputs structural_tag",
|
||||
):
|
||||
CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="Test prompt",
|
||||
max_tokens=10,
|
||||
structured_outputs={"structural_tag": structural_tag},
|
||||
)
|
||||
|
||||
|
||||
def test_negative_prompt_token_ids_nested():
|
||||
"""Negative token IDs in prompt (nested list) should raise validation error."""
|
||||
with pytest.raises(Exception, match="greater than or equal to 0"):
|
||||
|
||||
@@ -1382,7 +1382,20 @@ def test_adjust_request_non_mistral_tokenizer(
|
||||
[
|
||||
{"regex": r"\d+"},
|
||||
{"choice": ["a", "b"]},
|
||||
{"structural_tag": '{"key": "value"}'},
|
||||
{
|
||||
"structural_tag": json.dumps(
|
||||
{
|
||||
"structures": [
|
||||
{
|
||||
"begin": "<tool>",
|
||||
"schema": {"type": "object"},
|
||||
"end": "</tool>",
|
||||
}
|
||||
],
|
||||
"triggers": ["<tool>"],
|
||||
}
|
||||
)
|
||||
},
|
||||
{"grammar": "start: 'hello'"},
|
||||
],
|
||||
ids=["regex", "choice", "structural_tag", "grammar"],
|
||||
@@ -1404,7 +1417,18 @@ def test_adjust_request_unsupported_response_format(
|
||||
) -> None:
|
||||
request = _make_request(
|
||||
response_format=StructuralTagResponseFormat(
|
||||
type="structural_tag", format={"some": "config"}
|
||||
type="structural_tag",
|
||||
format={
|
||||
"type": "triggered_tags",
|
||||
"tags": [
|
||||
{
|
||||
"begin": "<tool>",
|
||||
"content": {"type": "any_text"},
|
||||
"end": "</tool>",
|
||||
}
|
||||
],
|
||||
"triggers": ["<tool>"],
|
||||
},
|
||||
),
|
||||
)
|
||||
result = mistral_tool_parser.adjust_request(request)
|
||||
|
||||
Reference in New Issue
Block a user