[Security] Gate per-request multimodal processor kwargs (#58830)

Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Juan Pérez de Algaba
2026-09-26 11:10:35 +00:00
committed by GitHub
co-authored by Cursor
parent 2f41e00235
commit 31f2e70cd3
17 changed files with 203 additions and 4 deletions
+13
View File
@@ -289,6 +289,19 @@ To mitigate this, vLLM enforces a configurable upper bound on the `n` parameter
- **Reverse proxy layer:** In addition to vLLM's built-in limit, consider enforcing request body validation and rate limiting at your reverse proxy to further constrain abusive payloads. - **Reverse proxy layer:** In addition to vLLM's built-in limit, consider enforcing request body validation and rate limiting at your reverse proxy to further constrain abusive payloads.
- **Monitoring:** Monitor per-request resource consumption to detect anomalous patterns that may indicate abuse. - **Monitoring:** Monitor per-request resource consumption to detect anomalous patterns that may indicate abuse.
### Per-request multimodal arguments
API server endpoints reject non-empty per-request `mm_processor_kwargs` and
`media_io_kwargs` by default. These arguments can change image, video, or audio
loading, sizing, sampling, and preprocessing behavior, causing excessive CPU,
GPU, or memory use when controlled by an untrusted client. Server-level
`--mm-processor-kwargs` and `--media-io-kwargs` remain available for deployment
configuration.
Only deployments whose API clients are trusted should start the server with
`--trust-request-mm-kwargs` to restore per-request overrides. Do not enable
this option on an endpoint exposed to untrusted clients.
## Tool Server and MCP Security ## Tool Server and MCP Security
vLLM supports connecting to external tool servers via the `--tool-server` argument. This enables models to call tools through the Responses API (`/v1/responses`). Tool server support works with all models — it is not limited to specific model architectures. vLLM supports connecting to external tool servers via the `--tool-server` argument. This enables models to call tools through the Responses API (`/v1/responses`). Tool server support works with all models — it is not limited to specific model architectures.
+11
View File
@@ -409,6 +409,17 @@ pub struct ServerUnsupportedArgs {
)] )]
pub trust_request_chat_template: Option<Unsupported>, pub trust_request_chat_template: Option<Unsupported>,
/// Whether to trust per-request multimodal kwargs (`mm_processor_kwargs`
/// and `media_io_kwargs`). If False, the server rejects non-empty values
/// because they can change multimodal preprocessing resource usage.
#[arg(
long,
visible_alias = "no-trust-request-mm-kwargs",
default_missing_value = "true",
num_args = 0..=1
)]
pub trust_request_mm_kwargs: Option<Unsupported>,
/// The role name to return if `request.add_generation_prompt=true`. /// The role name to return if `request.add_generation_prompt=true`.
#[arg(long)] #[arg(long)]
pub response_role: Option<Unsupported>, pub response_role: Option<Unsupported>,
+12
View File
@@ -480,6 +480,18 @@ def test_compilation_config():
) )
def test_trust_request_mm_kwargs_cli():
from vllm.entrypoints.launchers.cli_args import FrontendArgs
parser = FrontendArgs.add_cli_args(FlexibleArgumentParser())
args = parser.parse_args([])
assert not args.trust_request_mm_kwargs
args = parser.parse_args(["--trust-request-mm-kwargs"])
assert args.trust_request_mm_kwargs
def test_attention_config(): def test_attention_config():
from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -26,13 +26,11 @@ def server():
"--enforce-eager", "--enforce-eager",
"--limit-mm-per-prompt", "--limit-mm-per-prompt",
json.dumps({"audio": 3, "video": 3}), json.dumps({"audio": 3, "video": 3}),
"--trust-request-mm-kwargs",
*ROCM_EXTRA_ARGS, *ROCM_EXTRA_ARGS,
] ]
with RemoteOpenAIServer( with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
MODEL_NAME,
args,
) as remote_server:
yield remote_server yield remote_server
@@ -37,6 +37,7 @@ def server():
json.dumps({"video": MAXIMUM_VIDEOS}), json.dumps({"video": MAXIMUM_VIDEOS}),
"--media-io-kwargs", "--media-io-kwargs",
json.dumps({"video": {"num_frames": 32}}), json.dumps({"video": {"num_frames": 32}}),
"--trust-request-mm-kwargs",
] ]
# ROCm: Increase timeouts to handle potential network delays and slower # ROCm: Increase timeouts to handle potential network delays and slower
@@ -475,6 +475,81 @@ def test_json_schema_response_format_missing_schema():
) )
@pytest.mark.asyncio
async def test_online_renderer_rejects_mm_processor_kwargs_by_default():
model_config = MockModelConfig()
model_config.multimodal_config = MultiModalConfig()
online_renderer = OnlineRenderer(
model_config=model_config,
renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hello"}],
mm_processor_kwargs={
"patch_size": 1,
"vision_min_num_patches": 3_000_000_000,
},
)
with pytest.raises(
VLLMValidationError,
match="Per-request mm_processor_kwargs are disabled",
):
await online_renderer.preprocess_chat(
request,
request.messages,
default_template=None,
default_template_content_format="auto",
default_template_kwargs=None,
)
@pytest.mark.asyncio
async def test_online_renderer_rejects_media_io_kwargs_by_default():
model_config = MockModelConfig()
model_config.multimodal_config = MultiModalConfig()
online_renderer = OnlineRenderer(
model_config=model_config,
renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hello"}],
media_io_kwargs={"video": {"num_frames": 1_000_000}},
)
with pytest.raises(
VLLMValidationError,
match="Per-request media_io_kwargs are disabled",
):
await online_renderer.preprocess_chat(
request,
request.messages,
default_template=None,
default_template_content_format="auto",
default_template_kwargs=None,
)
def test_trust_request_mm_kwargs_opt_in_allows_overrides():
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
validate_request_mm_kwargs(
mm_processor_kwargs={"use_audio_in_video": True},
media_io_kwargs={"video": {"num_frames": 4}},
trust_request_mm_kwargs=True,
)
@pytest.mark.parametrize("format_value", [None, {}]) @pytest.mark.parametrize("format_value", [None, {}])
def test_structural_tag_response_format_invalid(format_value): def test_structural_tag_response_format_invalid(format_value):
"""Malformed structural tags should be rejected during request validation.""" """Malformed structural tags should be rejected during request validation."""
@@ -233,6 +233,7 @@ def online_renderer(model_config: MockModelConfig, request) -> OnlineRenderer:
enable_auto_tools=True, enable_auto_tools=True,
tool_parser="openai", tool_parser="openai",
exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none, exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none,
trust_request_mm_kwargs=True,
) )
@@ -46,6 +46,7 @@ def server():
json.dumps({"image": MAXIMUM_IMAGES}), json.dumps({"image": MAXIMUM_IMAGES}),
"--chat-template", "--chat-template",
str(vlm2vec_jinja_path), str(vlm2vec_jinja_path),
"--trust-request-mm-kwargs",
] ]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
@@ -26,6 +26,36 @@ TEXTS_2 = [
] ]
@pytest.mark.asyncio
async def test_flash_late_interaction_rejects_mm_processor_kwargs_by_default():
from unittest.mock import MagicMock
from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.pooling.base.serving import PoolingBaseServing
from vllm.entrypoints.pooling.scoring.protocol import ScoreQueriesDocumentsRequest
from vllm.entrypoints.pooling.scoring.serving import ServingScores
from vllm.exceptions import VLLMValidationError
serving = MagicMock()
serving.model_config = MagicMock(multimodal_config=MultiModalConfig())
serving.trust_request_mm_kwargs = False
serving.io_processor = MagicMock()
serving._init_ctx = PoolingBaseServing._init_ctx.__get__(serving)
request = ScoreQueriesDocumentsRequest(
model=MODEL_NAME,
queries=TEXTS_1[0],
documents=TEXTS_2[0],
mm_processor_kwargs={"vision_min_num_patches": 3_000_000_000},
)
with pytest.raises(
VLLMValidationError,
match="Per-request mm_processor_kwargs are disabled",
):
await ServingScores.flash_late_interaction(serving, request)
serving._preprocessing.assert_not_called()
@pytest.fixture(scope="module", params=[True, False]) @pytest.fixture(scope="module", params=[True, False])
def server(request): def server(request):
args = [ args = [
+1
View File
@@ -1450,6 +1450,7 @@ class ChatTemplateConfig:
chat_template: str | None = None chat_template: str | None = None
chat_template_content_format: ChatTemplateContentFormatOption = "auto" chat_template_content_format: ChatTemplateContentFormatOption = "auto"
trust_request_chat_template: bool = False trust_request_chat_template: bool = False
trust_request_mm_kwargs: bool = False
def validate_chat_template(chat_template: Path | str | None): def validate_chat_template(chat_template: Path | str | None):
@@ -49,6 +49,34 @@ def validate_cache_salt(cache_salt: object) -> None:
) )
def validate_request_mm_kwargs(
*,
mm_processor_kwargs: dict[str, Any] | None,
media_io_kwargs: dict[str, dict[str, Any]] | None,
trust_request_mm_kwargs: bool,
) -> None:
"""Reject untrusted per-request multimodal kwarg overrides."""
if trust_request_mm_kwargs:
return
if mm_processor_kwargs:
raise VLLMValidationError(
"Per-request mm_processor_kwargs are disabled by default because "
"they can change multimodal preprocessing resource usage. Start "
"the server with --trust-request-mm-kwargs only when clients "
"are trusted.",
parameter="mm_processor_kwargs",
)
if media_io_kwargs:
raise VLLMValidationError(
"Per-request media_io_kwargs are disabled by default because "
"they can change multimodal media loading resource usage. Start "
"the server with --trust-request-mm-kwargs only when clients "
"are trusted.",
parameter="media_io_kwargs",
)
class SpeculativeDecodingMetrics(OpenAIBaseModel): class SpeculativeDecodingMetrics(OpenAIBaseModel):
"""Per-request speculative-decoding acceptance metrics. """Per-request speculative-decoding acceptance metrics.
@@ -92,6 +92,7 @@ async def init_app_state(
chat_template=resolved_chat_template, chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format, chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template, trust_request_chat_template=args.trust_request_chat_template,
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
enable_auto_tools=args.enable_auto_tool_choice, enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser, tool_parser=args.tool_call_parser,
+5
View File
@@ -93,6 +93,11 @@ class BaseFrontendArgs:
"""Whether to trust the chat template provided in the request. If False, """Whether to trust the chat template provided in the request. If False,
the server will always use the chat template specified by `--chat-template` the server will always use the chat template specified by `--chat-template`
or the ones from tokenizer.""" or the ones from tokenizer."""
trust_request_mm_kwargs: bool = False
"""Whether to trust per-request multimodal kwargs (`mm_processor_kwargs`
and `media_io_kwargs`). If False, the server rejects non-empty values
because they can change multimodal preprocessing resource usage. Only
enable this when API clients are trusted."""
default_chat_template_kwargs: dict[str, Any] | None = None default_chat_template_kwargs: dict[str, Any] | None = None
"""Default keyword arguments to pass to the chat template renderer. """Default keyword arguments to pass to the chat template renderer.
These will be merged with request-level chat_template_kwargs, These will be merged with request-level chat_template_kwargs,
@@ -57,6 +57,7 @@ async def init_render_app_state(
chat_template=resolved_chat_template, chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format, chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template, trust_request_chat_template=args.trust_request_chat_template,
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
enable_auto_tools=args.enable_auto_tool_choice, enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser, tool_parser=args.tool_call_parser,
+7
View File
@@ -16,6 +16,7 @@ from vllm import PoolingRequestOutput, envs
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.engine.protocol import EngineClient from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import ChatTemplateConfig from vllm.entrypoints.chat_utils import ChatTemplateConfig
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.serving import BaseServing from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.engine.typing import AnyRequest from vllm.entrypoints.serve.engine.typing import AnyRequest
@@ -60,6 +61,7 @@ class PoolingBaseServing(ABC, BaseServing):
self.return_tokens_as_token_ids = return_tokens_as_token_ids self.return_tokens_as_token_ids = return_tokens_as_token_ids
self.log_error_stack = log_error_stack self.log_error_stack = log_error_stack
self.chat_template_config = chat_template_config self.chat_template_config = chat_template_config
self.trust_request_mm_kwargs = chat_template_config.trust_request_mm_kwargs
# Shared thread pool executor for preprocessing and postprocessing. # Shared thread pool executor for preprocessing and postprocessing.
self._executor: Executor = self.renderer._executor self._executor: Executor = self.renderer._executor
@@ -108,6 +110,11 @@ class PoolingBaseServing(ABC, BaseServing):
request: AnyPoolingRequest, request: AnyPoolingRequest,
raw_request: Request | None = None, raw_request: Request | None = None,
): ):
validate_request_mm_kwargs(
mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
media_io_kwargs=getattr(request, "media_io_kwargs", None),
trust_request_mm_kwargs=self.trust_request_mm_kwargs,
)
base_request_id = self._base_request_id( base_request_id = self._base_request_id(
raw_request, getattr(request, "request_id", None) raw_request, getattr(request, "request_id", None)
) )
+1
View File
@@ -166,6 +166,7 @@ def init_pooling_state(
chat_template=resolved_chat_template, chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format, chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template, trust_request_chat_template=args.trust_request_chat_template,
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
) )
state.serving_pooling = ( state.serving_pooling = (
+13
View File
@@ -17,6 +17,7 @@ from vllm.entrypoints.chat_utils import (
ChatTemplateContentFormatOption, ChatTemplateContentFormatOption,
ConversationMessage, ConversationMessage,
) )
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
from vllm.entrypoints.openai.chat_completion.protocol import ( from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedToolChoiceParam, ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest, ChatCompletionRequest,
@@ -123,6 +124,7 @@ class OnlineRenderer:
chat_template: str | None, chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption, chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False, trust_request_chat_template: bool = False,
trust_request_mm_kwargs: bool = False,
enable_auto_tools: bool = False, enable_auto_tools: bool = False,
exclude_tools_when_tool_choice_none: bool = False, exclude_tools_when_tool_choice_none: bool = False,
tool_parser: str | None = None, tool_parser: str | None = None,
@@ -155,6 +157,7 @@ class OnlineRenderer:
default_chat_template_kwargs or {} default_chat_template_kwargs or {}
) )
self.trust_request_chat_template = trust_request_chat_template self.trust_request_chat_template = trust_request_chat_template
self.trust_request_mm_kwargs = trust_request_mm_kwargs
self.log_error_stack = log_error_stack self.log_error_stack = log_error_stack
self.supports_browsing = False self.supports_browsing = False
@@ -674,6 +677,11 @@ class OnlineRenderer:
"""Copied from GenerateBaseServing._preprocess_cmpl.""" """Copied from GenerateBaseServing._preprocess_cmpl."""
renderer = self.renderer renderer = self.renderer
model_config = self.model_config model_config = self.model_config
validate_request_mm_kwargs(
mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
media_io_kwargs=getattr(request, "media_io_kwargs", None),
trust_request_mm_kwargs=self.trust_request_mm_kwargs,
)
parsed_prompts = [ parsed_prompts = [
( (
@@ -710,6 +718,11 @@ class OnlineRenderer:
) -> tuple[list[ConversationMessage], list[EngineInput]]: ) -> tuple[list[ConversationMessage], list[EngineInput]]:
"""Copied from GenerateBaseServing._preprocess_chat.""" """Copied from GenerateBaseServing._preprocess_chat."""
renderer = self.renderer renderer = self.renderer
validate_request_mm_kwargs(
mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
media_io_kwargs=getattr(request, "media_io_kwargs", None),
trust_request_mm_kwargs=self.trust_request_mm_kwargs,
)
mm_config = self.model_config.multimodal_config mm_config = self.model_config.multimodal_config
default_template_kwargs = merge_kwargs( default_template_kwargs = merge_kwargs(