[Security] Gate per-request multimodal processor kwargs (#58830)
Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
2f41e00235
commit
31f2e70cd3
@@ -1450,6 +1450,7 @@ class ChatTemplateConfig:
|
||||
chat_template: str | None = None
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto"
|
||||
trust_request_chat_template: bool = False
|
||||
trust_request_mm_kwargs: bool = False
|
||||
|
||||
|
||||
def validate_chat_template(chat_template: Path | str | None):
|
||||
|
||||
@@ -49,6 +49,34 @@ def validate_cache_salt(cache_salt: object) -> None:
|
||||
)
|
||||
|
||||
|
||||
def validate_request_mm_kwargs(
|
||||
*,
|
||||
mm_processor_kwargs: dict[str, Any] | None,
|
||||
media_io_kwargs: dict[str, dict[str, Any]] | None,
|
||||
trust_request_mm_kwargs: bool,
|
||||
) -> None:
|
||||
"""Reject untrusted per-request multimodal kwarg overrides."""
|
||||
if trust_request_mm_kwargs:
|
||||
return
|
||||
|
||||
if mm_processor_kwargs:
|
||||
raise VLLMValidationError(
|
||||
"Per-request mm_processor_kwargs are disabled by default because "
|
||||
"they can change multimodal preprocessing resource usage. Start "
|
||||
"the server with --trust-request-mm-kwargs only when clients "
|
||||
"are trusted.",
|
||||
parameter="mm_processor_kwargs",
|
||||
)
|
||||
if media_io_kwargs:
|
||||
raise VLLMValidationError(
|
||||
"Per-request media_io_kwargs are disabled by default because "
|
||||
"they can change multimodal media loading resource usage. Start "
|
||||
"the server with --trust-request-mm-kwargs only when clients "
|
||||
"are trusted.",
|
||||
parameter="media_io_kwargs",
|
||||
)
|
||||
|
||||
|
||||
class SpeculativeDecodingMetrics(OpenAIBaseModel):
|
||||
"""Per-request speculative-decoding acceptance metrics.
|
||||
|
||||
|
||||
@@ -92,6 +92,7 @@ async def init_app_state(
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
|
||||
@@ -93,6 +93,11 @@ class BaseFrontendArgs:
|
||||
"""Whether to trust the chat template provided in the request. If False,
|
||||
the server will always use the chat template specified by `--chat-template`
|
||||
or the ones from tokenizer."""
|
||||
trust_request_mm_kwargs: bool = False
|
||||
"""Whether to trust per-request multimodal kwargs (`mm_processor_kwargs`
|
||||
and `media_io_kwargs`). If False, the server rejects non-empty values
|
||||
because they can change multimodal preprocessing resource usage. Only
|
||||
enable this when API clients are trusted."""
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None
|
||||
"""Default keyword arguments to pass to the chat template renderer.
|
||||
These will be merged with request-level chat_template_kwargs,
|
||||
|
||||
@@ -57,6 +57,7 @@ async def init_render_app_state(
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm import PoolingRequestOutput, envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateConfig
|
||||
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
@@ -60,6 +61,7 @@ class PoolingBaseServing(ABC, BaseServing):
|
||||
self.return_tokens_as_token_ids = return_tokens_as_token_ids
|
||||
self.log_error_stack = log_error_stack
|
||||
self.chat_template_config = chat_template_config
|
||||
self.trust_request_mm_kwargs = chat_template_config.trust_request_mm_kwargs
|
||||
|
||||
# Shared thread pool executor for preprocessing and postprocessing.
|
||||
self._executor: Executor = self.renderer._executor
|
||||
@@ -108,6 +110,11 @@ class PoolingBaseServing(ABC, BaseServing):
|
||||
request: AnyPoolingRequest,
|
||||
raw_request: Request | None = None,
|
||||
):
|
||||
validate_request_mm_kwargs(
|
||||
mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
|
||||
media_io_kwargs=getattr(request, "media_io_kwargs", None),
|
||||
trust_request_mm_kwargs=self.trust_request_mm_kwargs,
|
||||
)
|
||||
base_request_id = self._base_request_id(
|
||||
raw_request, getattr(request, "request_id", None)
|
||||
)
|
||||
|
||||
@@ -166,6 +166,7 @@ def init_pooling_state(
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
|
||||
)
|
||||
|
||||
state.serving_pooling = (
|
||||
|
||||
Reference in New Issue
Block a user