[Security] Gate per-request multimodal processor kwargs (#58830)

Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

docs/usage/security.md

@@ -289,6 +289,19 @@ To mitigate this, vLLM enforces a configurable upper bound on the `n` parameter
 - **Reverse proxy layer:** In addition to vLLM's built-in limit, consider enforcing request body validation and rate limiting at your reverse proxy to further constrain abusive payloads.
 - **Monitoring:** Monitor per-request resource consumption to detect anomalous patterns that may indicate abuse.
 
+### Per-request multimodal arguments
+
+API server endpoints reject non-empty per-request `mm_processor_kwargs` and
+`media_io_kwargs` by default. These arguments can change image, video, or audio
+loading, sizing, sampling, and preprocessing behavior, causing excessive CPU,
+GPU, or memory use when controlled by an untrusted client. Server-level
+`--mm-processor-kwargs` and `--media-io-kwargs` remain available for deployment
+configuration.
+
+Only deployments whose API clients are trusted should start the server with
+`--trust-request-mm-kwargs` to restore per-request overrides. Do not enable
+this option on an endpoint exposed to untrusted clients.
+
 ## Tool Server and MCP Security
 
 vLLM supports connecting to external tool servers via the `--tool-server` argument. This enables models to call tools through the Responses API (`/v1/responses`). Tool server support works with all models — it is not limited to specific model architectures.

rust/src/cmd/src/cli/unsupported.rs

@@ -409,6 +409,17 @@ pub struct ServerUnsupportedArgs {
     )]
     pub trust_request_chat_template: Option<Unsupported>,
 
+    /// Whether to trust per-request multimodal kwargs (`mm_processor_kwargs`
+    /// and `media_io_kwargs`). If False, the server rejects non-empty values
+    /// because they can change multimodal preprocessing resource usage.
+    #[arg(
+        long,
+        visible_alias = "no-trust-request-mm-kwargs",
+        default_missing_value = "true",
+        num_args = 0..=1
+    )]
+    pub trust_request_mm_kwargs: Option<Unsupported>,
+
     /// The role name to return if `request.add_generation_prompt=true`.
     #[arg(long)]
     pub response_role: Option<Unsupported>,

tests/engine/test_arg_utils.py

@@ -480,6 +480,18 @@ def test_compilation_config():
     )
 
 
+def test_trust_request_mm_kwargs_cli():
+    from vllm.entrypoints.launchers.cli_args import FrontendArgs
+
+    parser = FrontendArgs.add_cli_args(FlexibleArgumentParser())
+
+    args = parser.parse_args([])
+    assert not args.trust_request_mm_kwargs
+
+    args = parser.parse_args(["--trust-request-mm-kwargs"])
+    assert args.trust_request_mm_kwargs
+
+
 def test_attention_config():
     from vllm.v1.attention.backends.registry import AttentionBackendEnum
 

tests/entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py

@@ -26,13 +26,11 @@ def server():
         "--enforce-eager",
         "--limit-mm-per-prompt",
         json.dumps({"audio": 3, "video": 3}),
+        "--trust-request-mm-kwargs",
         *ROCM_EXTRA_ARGS,
     ]
 
-    with RemoteOpenAIServer(
-        MODEL_NAME,
-        args,
-    ) as remote_server:
+    with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
         yield remote_server
 
 

tests/entrypoints/multimodal/openai/chat_completion/test_video.py

@@ -37,6 +37,7 @@ def server():
         json.dumps({"video": MAXIMUM_VIDEOS}),
         "--media-io-kwargs",
         json.dumps({"video": {"num_frames": 32}}),
+        "--trust-request-mm-kwargs",
     ]
 
     # ROCm: Increase timeouts to handle potential network delays and slower

tests/entrypoints/openai/chat_completion/test_chat_error.py

@@ -475,6 +475,81 @@ def test_json_schema_response_format_missing_schema():
         )
 
 
+@pytest.mark.asyncio
+async def test_online_renderer_rejects_mm_processor_kwargs_by_default():
+    model_config = MockModelConfig()
+    model_config.multimodal_config = MultiModalConfig()
+    online_renderer = OnlineRenderer(
+        model_config=model_config,
+        renderer=MagicMock(),
+        request_logger=None,
+        chat_template=None,
+        chat_template_content_format="auto",
+    )
+
+    request = ChatCompletionRequest(
+        model=MODEL_NAME,
+        messages=[{"role": "user", "content": "hello"}],
+        mm_processor_kwargs={
+            "patch_size": 1,
+            "vision_min_num_patches": 3_000_000_000,
+        },
+    )
+
+    with pytest.raises(
+        VLLMValidationError,
+        match="Per-request mm_processor_kwargs are disabled",
+    ):
+        await online_renderer.preprocess_chat(
+            request,
+            request.messages,
+            default_template=None,
+            default_template_content_format="auto",
+            default_template_kwargs=None,
+        )
+
+
+@pytest.mark.asyncio
+async def test_online_renderer_rejects_media_io_kwargs_by_default():
+    model_config = MockModelConfig()
+    model_config.multimodal_config = MultiModalConfig()
+    online_renderer = OnlineRenderer(
+        model_config=model_config,
+        renderer=MagicMock(),
+        request_logger=None,
+        chat_template=None,
+        chat_template_content_format="auto",
+    )
+
+    request = ChatCompletionRequest(
+        model=MODEL_NAME,
+        messages=[{"role": "user", "content": "hello"}],
+        media_io_kwargs={"video": {"num_frames": 1_000_000}},
+    )
+
+    with pytest.raises(
+        VLLMValidationError,
+        match="Per-request media_io_kwargs are disabled",
+    ):
+        await online_renderer.preprocess_chat(
+            request,
+            request.messages,
+            default_template=None,
+            default_template_content_format="auto",
+            default_template_kwargs=None,
+        )
+
+
+def test_trust_request_mm_kwargs_opt_in_allows_overrides():
+    from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
+
+    validate_request_mm_kwargs(
+        mm_processor_kwargs={"use_audio_in_video": True},
+        media_io_kwargs={"video": {"num_frames": 4}},
+        trust_request_mm_kwargs=True,
+    )
+
+
 @pytest.mark.parametrize("format_value", [None, {}])
 def test_structural_tag_response_format_invalid(format_value):
     """Malformed structural tags should be rejected during request validation."""

tests/entrypoints/openai/test_render_parity.py

@@ -233,6 +233,7 @@ def online_renderer(model_config: MockModelConfig, request) -> OnlineRenderer:
         enable_auto_tools=True,
         tool_parser="openai",
         exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none,
+        trust_request_mm_kwargs=True,
     )
 
 

tests/entrypoints/pooling/embed/test_online_vision.py

@@ -46,6 +46,7 @@ def server():
         json.dumps({"image": MAXIMUM_IMAGES}),
         "--chat-template",
         str(vlm2vec_jinja_path),
+        "--trust-request-mm-kwargs",
     ]
 
     with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

tests/entrypoints/pooling/scoring/test_late_interaction_online.py

@@ -26,6 +26,36 @@ TEXTS_2 = [
 ]
 
 
+@pytest.mark.asyncio
+async def test_flash_late_interaction_rejects_mm_processor_kwargs_by_default():
+    from unittest.mock import MagicMock
+
+    from vllm.config.multimodal import MultiModalConfig
+    from vllm.entrypoints.pooling.base.serving import PoolingBaseServing
+    from vllm.entrypoints.pooling.scoring.protocol import ScoreQueriesDocumentsRequest
+    from vllm.entrypoints.pooling.scoring.serving import ServingScores
+    from vllm.exceptions import VLLMValidationError
+
+    serving = MagicMock()
+    serving.model_config = MagicMock(multimodal_config=MultiModalConfig())
+    serving.trust_request_mm_kwargs = False
+    serving.io_processor = MagicMock()
+    serving._init_ctx = PoolingBaseServing._init_ctx.__get__(serving)
+    request = ScoreQueriesDocumentsRequest(
+        model=MODEL_NAME,
+        queries=TEXTS_1[0],
+        documents=TEXTS_2[0],
+        mm_processor_kwargs={"vision_min_num_patches": 3_000_000_000},
+    )
+
+    with pytest.raises(
+        VLLMValidationError,
+        match="Per-request mm_processor_kwargs are disabled",
+    ):
+        await ServingScores.flash_late_interaction(serving, request)
+    serving._preprocessing.assert_not_called()
+
+
 @pytest.fixture(scope="module", params=[True, False])
 def server(request):
     args = [

vllm/entrypoints/chat_utils.py

@@ -1450,6 +1450,7 @@ class ChatTemplateConfig:
     chat_template: str | None = None
     chat_template_content_format: ChatTemplateContentFormatOption = "auto"
     trust_request_chat_template: bool = False
+    trust_request_mm_kwargs: bool = False
 
 
 def validate_chat_template(chat_template: Path | str | None):

vllm/entrypoints/generate/base/protocol.py

@@ -49,6 +49,34 @@ def validate_cache_salt(cache_salt: object) -> None:
         )
 
 
+def validate_request_mm_kwargs(
+    *,
+    mm_processor_kwargs: dict[str, Any] | None,
+    media_io_kwargs: dict[str, dict[str, Any]] | None,
+    trust_request_mm_kwargs: bool,
+) -> None:
+    """Reject untrusted per-request multimodal kwarg overrides."""
+    if trust_request_mm_kwargs:
+        return
+
+    if mm_processor_kwargs:
+        raise VLLMValidationError(
+            "Per-request mm_processor_kwargs are disabled by default because "
+            "they can change multimodal preprocessing resource usage. Start "
+            "the server with --trust-request-mm-kwargs only when clients "
+            "are trusted.",
+            parameter="mm_processor_kwargs",
+        )
+    if media_io_kwargs:
+        raise VLLMValidationError(
+            "Per-request media_io_kwargs are disabled by default because "
+            "they can change multimodal media loading resource usage. Start "
+            "the server with --trust-request-mm-kwargs only when clients "
+            "are trusted.",
+            parameter="media_io_kwargs",
+        )
+
+
 class SpeculativeDecodingMetrics(OpenAIBaseModel):
     """Per-request speculative-decoding acceptance metrics.
 

vllm/entrypoints/launchers/api_server/app_state.py

@@ -92,6 +92,7 @@ async def init_app_state(
         chat_template=resolved_chat_template,
         chat_template_content_format=args.chat_template_content_format,
         trust_request_chat_template=args.trust_request_chat_template,
+        trust_request_mm_kwargs=args.trust_request_mm_kwargs,
         enable_auto_tools=args.enable_auto_tool_choice,
         exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
         tool_parser=args.tool_call_parser,

vllm/entrypoints/launchers/cli_args.py

@@ -93,6 +93,11 @@ class BaseFrontendArgs:
     """Whether to trust the chat template provided in the request. If False,
     the server will always use the chat template specified by `--chat-template`
     or the ones from tokenizer."""
+    trust_request_mm_kwargs: bool = False
+    """Whether to trust per-request multimodal kwargs (`mm_processor_kwargs`
+    and `media_io_kwargs`). If False, the server rejects non-empty values
+    because they can change multimodal preprocessing resource usage. Only
+    enable this when API clients are trusted."""
     default_chat_template_kwargs: dict[str, Any] | None = None
     """Default keyword arguments to pass to the chat template renderer.
     These will be merged with request-level chat_template_kwargs,

vllm/entrypoints/launchers/render/app_state.py

@@ -57,6 +57,7 @@ async def init_render_app_state(
         chat_template=resolved_chat_template,
         chat_template_content_format=args.chat_template_content_format,
         trust_request_chat_template=args.trust_request_chat_template,
+        trust_request_mm_kwargs=args.trust_request_mm_kwargs,
         enable_auto_tools=args.enable_auto_tool_choice,
         exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
         tool_parser=args.tool_call_parser,

vllm/entrypoints/pooling/base/serving.py

@@ -16,6 +16,7 @@ from vllm import PoolingRequestOutput, envs
 from vllm.config import VllmConfig
 from vllm.engine.protocol import EngineClient
 from vllm.entrypoints.chat_utils import ChatTemplateConfig
+from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
 from vllm.entrypoints.serve.engine.serving import BaseServing
 from vllm.entrypoints.serve.engine.typing import AnyRequest
@@ -60,6 +61,7 @@ class PoolingBaseServing(ABC, BaseServing):
         self.return_tokens_as_token_ids = return_tokens_as_token_ids
         self.log_error_stack = log_error_stack
         self.chat_template_config = chat_template_config
+        self.trust_request_mm_kwargs = chat_template_config.trust_request_mm_kwargs
 
         # Shared thread pool executor for preprocessing and postprocessing.
         self._executor: Executor = self.renderer._executor
@@ -108,6 +110,11 @@ class PoolingBaseServing(ABC, BaseServing):
         request: AnyPoolingRequest,
         raw_request: Request | None = None,
     ):
+        validate_request_mm_kwargs(
+            mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
+            media_io_kwargs=getattr(request, "media_io_kwargs", None),
+            trust_request_mm_kwargs=self.trust_request_mm_kwargs,
+        )
         base_request_id = self._base_request_id(
             raw_request, getattr(request, "request_id", None)
         )

vllm/entrypoints/pooling/factories.py

@@ -166,6 +166,7 @@ def init_pooling_state(
         chat_template=resolved_chat_template,
         chat_template_content_format=args.chat_template_content_format,
         trust_request_chat_template=args.trust_request_chat_template,
+        trust_request_mm_kwargs=args.trust_request_mm_kwargs,
     )
 
     state.serving_pooling = (

vllm/renderers/online_renderer.py

@@ -17,6 +17,7 @@ from vllm.entrypoints.chat_utils import (
     ChatTemplateContentFormatOption,
     ConversationMessage,
 )
+from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
 from vllm.entrypoints.openai.chat_completion.protocol import (
     ChatCompletionNamedToolChoiceParam,
     ChatCompletionRequest,
@@ -123,6 +124,7 @@ class OnlineRenderer:
         chat_template: str | None,
         chat_template_content_format: ChatTemplateContentFormatOption,
         trust_request_chat_template: bool = False,
+        trust_request_mm_kwargs: bool = False,
         enable_auto_tools: bool = False,
         exclude_tools_when_tool_choice_none: bool = False,
         tool_parser: str | None = None,
@@ -155,6 +157,7 @@ class OnlineRenderer:
             default_chat_template_kwargs or {}
         )
         self.trust_request_chat_template = trust_request_chat_template
+        self.trust_request_mm_kwargs = trust_request_mm_kwargs
 
         self.log_error_stack = log_error_stack
         self.supports_browsing = False
@@ -674,6 +677,11 @@ class OnlineRenderer:
         """Copied from GenerateBaseServing._preprocess_cmpl."""
         renderer = self.renderer
         model_config = self.model_config
+        validate_request_mm_kwargs(
+            mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
+            media_io_kwargs=getattr(request, "media_io_kwargs", None),
+            trust_request_mm_kwargs=self.trust_request_mm_kwargs,
+        )
 
         parsed_prompts = [
             (
@@ -710,6 +718,11 @@ class OnlineRenderer:
     ) -> tuple[list[ConversationMessage], list[EngineInput]]:
         """Copied from GenerateBaseServing._preprocess_chat."""
         renderer = self.renderer
+        validate_request_mm_kwargs(
+            mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
+            media_io_kwargs=getattr(request, "media_io_kwargs", None),
+            trust_request_mm_kwargs=self.trust_request_mm_kwargs,
+        )
         mm_config = self.model_config.multimodal_config
 
         default_template_kwargs = merge_kwargs(