[Security] Gate per-request multimodal processor kwargs (#58830)

Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Juan Pérez de Algaba
2026-09-26 11:10:35 +00:00
committed by GitHub
co-authored by Cursor
parent 2f41e00235
commit 31f2e70cd3
17 changed files with 203 additions and 4 deletions
+12
View File
@@ -480,6 +480,18 @@ def test_compilation_config():
)
def test_trust_request_mm_kwargs_cli():
from vllm.entrypoints.launchers.cli_args import FrontendArgs
parser = FrontendArgs.add_cli_args(FlexibleArgumentParser())
args = parser.parse_args([])
assert not args.trust_request_mm_kwargs
args = parser.parse_args(["--trust-request-mm-kwargs"])
assert args.trust_request_mm_kwargs
def test_attention_config():
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -26,13 +26,11 @@ def server():
"--enforce-eager",
"--limit-mm-per-prompt",
json.dumps({"audio": 3, "video": 3}),
"--trust-request-mm-kwargs",
*ROCM_EXTRA_ARGS,
]
with RemoteOpenAIServer(
MODEL_NAME,
args,
) as remote_server:
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@@ -37,6 +37,7 @@ def server():
json.dumps({"video": MAXIMUM_VIDEOS}),
"--media-io-kwargs",
json.dumps({"video": {"num_frames": 32}}),
"--trust-request-mm-kwargs",
]
# ROCm: Increase timeouts to handle potential network delays and slower
@@ -475,6 +475,81 @@ def test_json_schema_response_format_missing_schema():
)
@pytest.mark.asyncio
async def test_online_renderer_rejects_mm_processor_kwargs_by_default():
model_config = MockModelConfig()
model_config.multimodal_config = MultiModalConfig()
online_renderer = OnlineRenderer(
model_config=model_config,
renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hello"}],
mm_processor_kwargs={
"patch_size": 1,
"vision_min_num_patches": 3_000_000_000,
},
)
with pytest.raises(
VLLMValidationError,
match="Per-request mm_processor_kwargs are disabled",
):
await online_renderer.preprocess_chat(
request,
request.messages,
default_template=None,
default_template_content_format="auto",
default_template_kwargs=None,
)
@pytest.mark.asyncio
async def test_online_renderer_rejects_media_io_kwargs_by_default():
model_config = MockModelConfig()
model_config.multimodal_config = MultiModalConfig()
online_renderer = OnlineRenderer(
model_config=model_config,
renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hello"}],
media_io_kwargs={"video": {"num_frames": 1_000_000}},
)
with pytest.raises(
VLLMValidationError,
match="Per-request media_io_kwargs are disabled",
):
await online_renderer.preprocess_chat(
request,
request.messages,
default_template=None,
default_template_content_format="auto",
default_template_kwargs=None,
)
def test_trust_request_mm_kwargs_opt_in_allows_overrides():
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
validate_request_mm_kwargs(
mm_processor_kwargs={"use_audio_in_video": True},
media_io_kwargs={"video": {"num_frames": 4}},
trust_request_mm_kwargs=True,
)
@pytest.mark.parametrize("format_value", [None, {}])
def test_structural_tag_response_format_invalid(format_value):
"""Malformed structural tags should be rejected during request validation."""
@@ -233,6 +233,7 @@ def online_renderer(model_config: MockModelConfig, request) -> OnlineRenderer:
enable_auto_tools=True,
tool_parser="openai",
exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none,
trust_request_mm_kwargs=True,
)
@@ -46,6 +46,7 @@ def server():
json.dumps({"image": MAXIMUM_IMAGES}),
"--chat-template",
str(vlm2vec_jinja_path),
"--trust-request-mm-kwargs",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
@@ -26,6 +26,36 @@ TEXTS_2 = [
]
@pytest.mark.asyncio
async def test_flash_late_interaction_rejects_mm_processor_kwargs_by_default():
from unittest.mock import MagicMock
from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.pooling.base.serving import PoolingBaseServing
from vllm.entrypoints.pooling.scoring.protocol import ScoreQueriesDocumentsRequest
from vllm.entrypoints.pooling.scoring.serving import ServingScores
from vllm.exceptions import VLLMValidationError
serving = MagicMock()
serving.model_config = MagicMock(multimodal_config=MultiModalConfig())
serving.trust_request_mm_kwargs = False
serving.io_processor = MagicMock()
serving._init_ctx = PoolingBaseServing._init_ctx.__get__(serving)
request = ScoreQueriesDocumentsRequest(
model=MODEL_NAME,
queries=TEXTS_1[0],
documents=TEXTS_2[0],
mm_processor_kwargs={"vision_min_num_patches": 3_000_000_000},
)
with pytest.raises(
VLLMValidationError,
match="Per-request mm_processor_kwargs are disabled",
):
await ServingScores.flash_late_interaction(serving, request)
serving._preprocessing.assert_not_called()
@pytest.fixture(scope="module", params=[True, False])
def server(request):
args = [