[Security] Gate per-request multimodal processor kwargs (#58830)
Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
2f41e00235
commit
31f2e70cd3
@@ -480,6 +480,18 @@ def test_compilation_config():
|
||||
)
|
||||
|
||||
|
||||
def test_trust_request_mm_kwargs_cli():
|
||||
from vllm.entrypoints.launchers.cli_args import FrontendArgs
|
||||
|
||||
parser = FrontendArgs.add_cli_args(FlexibleArgumentParser())
|
||||
|
||||
args = parser.parse_args([])
|
||||
assert not args.trust_request_mm_kwargs
|
||||
|
||||
args = parser.parse_args(["--trust-request-mm-kwargs"])
|
||||
assert args.trust_request_mm_kwargs
|
||||
|
||||
|
||||
def test_attention_config():
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
@@ -26,13 +26,11 @@ def server():
|
||||
"--enforce-eager",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"audio": 3, "video": 3}),
|
||||
"--trust-request-mm-kwargs",
|
||||
*ROCM_EXTRA_ARGS,
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ def server():
|
||||
json.dumps({"video": MAXIMUM_VIDEOS}),
|
||||
"--media-io-kwargs",
|
||||
json.dumps({"video": {"num_frames": 32}}),
|
||||
"--trust-request-mm-kwargs",
|
||||
]
|
||||
|
||||
# ROCm: Increase timeouts to handle potential network delays and slower
|
||||
|
||||
@@ -475,6 +475,81 @@ def test_json_schema_response_format_missing_schema():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_rejects_mm_processor_kwargs_by_default():
|
||||
model_config = MockModelConfig()
|
||||
model_config.multimodal_config = MultiModalConfig()
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=model_config,
|
||||
renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
mm_processor_kwargs={
|
||||
"patch_size": 1,
|
||||
"vision_min_num_patches": 3_000_000_000,
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Per-request mm_processor_kwargs are disabled",
|
||||
):
|
||||
await online_renderer.preprocess_chat(
|
||||
request,
|
||||
request.messages,
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
default_template_kwargs=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_rejects_media_io_kwargs_by_default():
|
||||
model_config = MockModelConfig()
|
||||
model_config.multimodal_config = MultiModalConfig()
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=model_config,
|
||||
renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
media_io_kwargs={"video": {"num_frames": 1_000_000}},
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Per-request media_io_kwargs are disabled",
|
||||
):
|
||||
await online_renderer.preprocess_chat(
|
||||
request,
|
||||
request.messages,
|
||||
default_template=None,
|
||||
default_template_content_format="auto",
|
||||
default_template_kwargs=None,
|
||||
)
|
||||
|
||||
|
||||
def test_trust_request_mm_kwargs_opt_in_allows_overrides():
|
||||
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
|
||||
|
||||
validate_request_mm_kwargs(
|
||||
mm_processor_kwargs={"use_audio_in_video": True},
|
||||
media_io_kwargs={"video": {"num_frames": 4}},
|
||||
trust_request_mm_kwargs=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("format_value", [None, {}])
|
||||
def test_structural_tag_response_format_invalid(format_value):
|
||||
"""Malformed structural tags should be rejected during request validation."""
|
||||
|
||||
@@ -233,6 +233,7 @@ def online_renderer(model_config: MockModelConfig, request) -> OnlineRenderer:
|
||||
enable_auto_tools=True,
|
||||
tool_parser="openai",
|
||||
exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none,
|
||||
trust_request_mm_kwargs=True,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ def server():
|
||||
json.dumps({"image": MAXIMUM_IMAGES}),
|
||||
"--chat-template",
|
||||
str(vlm2vec_jinja_path),
|
||||
"--trust-request-mm-kwargs",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
|
||||
@@ -26,6 +26,36 @@ TEXTS_2 = [
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flash_late_interaction_rejects_mm_processor_kwargs_by_default():
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.entrypoints.pooling.base.serving import PoolingBaseServing
|
||||
from vllm.entrypoints.pooling.scoring.protocol import ScoreQueriesDocumentsRequest
|
||||
from vllm.entrypoints.pooling.scoring.serving import ServingScores
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
serving = MagicMock()
|
||||
serving.model_config = MagicMock(multimodal_config=MultiModalConfig())
|
||||
serving.trust_request_mm_kwargs = False
|
||||
serving.io_processor = MagicMock()
|
||||
serving._init_ctx = PoolingBaseServing._init_ctx.__get__(serving)
|
||||
request = ScoreQueriesDocumentsRequest(
|
||||
model=MODEL_NAME,
|
||||
queries=TEXTS_1[0],
|
||||
documents=TEXTS_2[0],
|
||||
mm_processor_kwargs={"vision_min_num_patches": 3_000_000_000},
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Per-request mm_processor_kwargs are disabled",
|
||||
):
|
||||
await ServingScores.flash_late_interaction(serving, request)
|
||||
serving._preprocessing.assert_not_called()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=[True, False])
|
||||
def server(request):
|
||||
args = [
|
||||
|
||||
Reference in New Issue
Block a user