[Security] Gate per-request multimodal processor kwargs (#58830)
Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
2f41e00235
commit
31f2e70cd3
@@ -289,6 +289,19 @@ To mitigate this, vLLM enforces a configurable upper bound on the `n` parameter
|
|||||||
- **Reverse proxy layer:** In addition to vLLM's built-in limit, consider enforcing request body validation and rate limiting at your reverse proxy to further constrain abusive payloads.
|
- **Reverse proxy layer:** In addition to vLLM's built-in limit, consider enforcing request body validation and rate limiting at your reverse proxy to further constrain abusive payloads.
|
||||||
- **Monitoring:** Monitor per-request resource consumption to detect anomalous patterns that may indicate abuse.
|
- **Monitoring:** Monitor per-request resource consumption to detect anomalous patterns that may indicate abuse.
|
||||||
|
|
||||||
|
### Per-request multimodal arguments
|
||||||
|
|
||||||
|
API server endpoints reject non-empty per-request `mm_processor_kwargs` and
|
||||||
|
`media_io_kwargs` by default. These arguments can change image, video, or audio
|
||||||
|
loading, sizing, sampling, and preprocessing behavior, causing excessive CPU,
|
||||||
|
GPU, or memory use when controlled by an untrusted client. Server-level
|
||||||
|
`--mm-processor-kwargs` and `--media-io-kwargs` remain available for deployment
|
||||||
|
configuration.
|
||||||
|
|
||||||
|
Only deployments whose API clients are trusted should start the server with
|
||||||
|
`--trust-request-mm-kwargs` to restore per-request overrides. Do not enable
|
||||||
|
this option on an endpoint exposed to untrusted clients.
|
||||||
|
|
||||||
## Tool Server and MCP Security
|
## Tool Server and MCP Security
|
||||||
|
|
||||||
vLLM supports connecting to external tool servers via the `--tool-server` argument. This enables models to call tools through the Responses API (`/v1/responses`). Tool server support works with all models — it is not limited to specific model architectures.
|
vLLM supports connecting to external tool servers via the `--tool-server` argument. This enables models to call tools through the Responses API (`/v1/responses`). Tool server support works with all models — it is not limited to specific model architectures.
|
||||||
|
|||||||
@@ -409,6 +409,17 @@ pub struct ServerUnsupportedArgs {
|
|||||||
)]
|
)]
|
||||||
pub trust_request_chat_template: Option<Unsupported>,
|
pub trust_request_chat_template: Option<Unsupported>,
|
||||||
|
|
||||||
|
/// Whether to trust per-request multimodal kwargs (`mm_processor_kwargs`
|
||||||
|
/// and `media_io_kwargs`). If False, the server rejects non-empty values
|
||||||
|
/// because they can change multimodal preprocessing resource usage.
|
||||||
|
#[arg(
|
||||||
|
long,
|
||||||
|
visible_alias = "no-trust-request-mm-kwargs",
|
||||||
|
default_missing_value = "true",
|
||||||
|
num_args = 0..=1
|
||||||
|
)]
|
||||||
|
pub trust_request_mm_kwargs: Option<Unsupported>,
|
||||||
|
|
||||||
/// The role name to return if `request.add_generation_prompt=true`.
|
/// The role name to return if `request.add_generation_prompt=true`.
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
pub response_role: Option<Unsupported>,
|
pub response_role: Option<Unsupported>,
|
||||||
|
|||||||
@@ -480,6 +480,18 @@ def test_compilation_config():
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_trust_request_mm_kwargs_cli():
|
||||||
|
from vllm.entrypoints.launchers.cli_args import FrontendArgs
|
||||||
|
|
||||||
|
parser = FrontendArgs.add_cli_args(FlexibleArgumentParser())
|
||||||
|
|
||||||
|
args = parser.parse_args([])
|
||||||
|
assert not args.trust_request_mm_kwargs
|
||||||
|
|
||||||
|
args = parser.parse_args(["--trust-request-mm-kwargs"])
|
||||||
|
assert args.trust_request_mm_kwargs
|
||||||
|
|
||||||
|
|
||||||
def test_attention_config():
|
def test_attention_config():
|
||||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||||
|
|
||||||
|
|||||||
@@ -26,13 +26,11 @@ def server():
|
|||||||
"--enforce-eager",
|
"--enforce-eager",
|
||||||
"--limit-mm-per-prompt",
|
"--limit-mm-per-prompt",
|
||||||
json.dumps({"audio": 3, "video": 3}),
|
json.dumps({"audio": 3, "video": 3}),
|
||||||
|
"--trust-request-mm-kwargs",
|
||||||
*ROCM_EXTRA_ARGS,
|
*ROCM_EXTRA_ARGS,
|
||||||
]
|
]
|
||||||
|
|
||||||
with RemoteOpenAIServer(
|
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||||
MODEL_NAME,
|
|
||||||
args,
|
|
||||||
) as remote_server:
|
|
||||||
yield remote_server
|
yield remote_server
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ def server():
|
|||||||
json.dumps({"video": MAXIMUM_VIDEOS}),
|
json.dumps({"video": MAXIMUM_VIDEOS}),
|
||||||
"--media-io-kwargs",
|
"--media-io-kwargs",
|
||||||
json.dumps({"video": {"num_frames": 32}}),
|
json.dumps({"video": {"num_frames": 32}}),
|
||||||
|
"--trust-request-mm-kwargs",
|
||||||
]
|
]
|
||||||
|
|
||||||
# ROCm: Increase timeouts to handle potential network delays and slower
|
# ROCm: Increase timeouts to handle potential network delays and slower
|
||||||
|
|||||||
@@ -475,6 +475,81 @@ def test_json_schema_response_format_missing_schema():
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_online_renderer_rejects_mm_processor_kwargs_by_default():
|
||||||
|
model_config = MockModelConfig()
|
||||||
|
model_config.multimodal_config = MultiModalConfig()
|
||||||
|
online_renderer = OnlineRenderer(
|
||||||
|
model_config=model_config,
|
||||||
|
renderer=MagicMock(),
|
||||||
|
request_logger=None,
|
||||||
|
chat_template=None,
|
||||||
|
chat_template_content_format="auto",
|
||||||
|
)
|
||||||
|
|
||||||
|
request = ChatCompletionRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
messages=[{"role": "user", "content": "hello"}],
|
||||||
|
mm_processor_kwargs={
|
||||||
|
"patch_size": 1,
|
||||||
|
"vision_min_num_patches": 3_000_000_000,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
VLLMValidationError,
|
||||||
|
match="Per-request mm_processor_kwargs are disabled",
|
||||||
|
):
|
||||||
|
await online_renderer.preprocess_chat(
|
||||||
|
request,
|
||||||
|
request.messages,
|
||||||
|
default_template=None,
|
||||||
|
default_template_content_format="auto",
|
||||||
|
default_template_kwargs=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_online_renderer_rejects_media_io_kwargs_by_default():
|
||||||
|
model_config = MockModelConfig()
|
||||||
|
model_config.multimodal_config = MultiModalConfig()
|
||||||
|
online_renderer = OnlineRenderer(
|
||||||
|
model_config=model_config,
|
||||||
|
renderer=MagicMock(),
|
||||||
|
request_logger=None,
|
||||||
|
chat_template=None,
|
||||||
|
chat_template_content_format="auto",
|
||||||
|
)
|
||||||
|
|
||||||
|
request = ChatCompletionRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
messages=[{"role": "user", "content": "hello"}],
|
||||||
|
media_io_kwargs={"video": {"num_frames": 1_000_000}},
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
VLLMValidationError,
|
||||||
|
match="Per-request media_io_kwargs are disabled",
|
||||||
|
):
|
||||||
|
await online_renderer.preprocess_chat(
|
||||||
|
request,
|
||||||
|
request.messages,
|
||||||
|
default_template=None,
|
||||||
|
default_template_content_format="auto",
|
||||||
|
default_template_kwargs=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_trust_request_mm_kwargs_opt_in_allows_overrides():
|
||||||
|
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
|
||||||
|
|
||||||
|
validate_request_mm_kwargs(
|
||||||
|
mm_processor_kwargs={"use_audio_in_video": True},
|
||||||
|
media_io_kwargs={"video": {"num_frames": 4}},
|
||||||
|
trust_request_mm_kwargs=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("format_value", [None, {}])
|
@pytest.mark.parametrize("format_value", [None, {}])
|
||||||
def test_structural_tag_response_format_invalid(format_value):
|
def test_structural_tag_response_format_invalid(format_value):
|
||||||
"""Malformed structural tags should be rejected during request validation."""
|
"""Malformed structural tags should be rejected during request validation."""
|
||||||
|
|||||||
@@ -233,6 +233,7 @@ def online_renderer(model_config: MockModelConfig, request) -> OnlineRenderer:
|
|||||||
enable_auto_tools=True,
|
enable_auto_tools=True,
|
||||||
tool_parser="openai",
|
tool_parser="openai",
|
||||||
exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none,
|
exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none,
|
||||||
|
trust_request_mm_kwargs=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -46,6 +46,7 @@ def server():
|
|||||||
json.dumps({"image": MAXIMUM_IMAGES}),
|
json.dumps({"image": MAXIMUM_IMAGES}),
|
||||||
"--chat-template",
|
"--chat-template",
|
||||||
str(vlm2vec_jinja_path),
|
str(vlm2vec_jinja_path),
|
||||||
|
"--trust-request-mm-kwargs",
|
||||||
]
|
]
|
||||||
|
|
||||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||||
|
|||||||
@@ -26,6 +26,36 @@ TEXTS_2 = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_flash_late_interaction_rejects_mm_processor_kwargs_by_default():
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from vllm.config.multimodal import MultiModalConfig
|
||||||
|
from vllm.entrypoints.pooling.base.serving import PoolingBaseServing
|
||||||
|
from vllm.entrypoints.pooling.scoring.protocol import ScoreQueriesDocumentsRequest
|
||||||
|
from vllm.entrypoints.pooling.scoring.serving import ServingScores
|
||||||
|
from vllm.exceptions import VLLMValidationError
|
||||||
|
|
||||||
|
serving = MagicMock()
|
||||||
|
serving.model_config = MagicMock(multimodal_config=MultiModalConfig())
|
||||||
|
serving.trust_request_mm_kwargs = False
|
||||||
|
serving.io_processor = MagicMock()
|
||||||
|
serving._init_ctx = PoolingBaseServing._init_ctx.__get__(serving)
|
||||||
|
request = ScoreQueriesDocumentsRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
queries=TEXTS_1[0],
|
||||||
|
documents=TEXTS_2[0],
|
||||||
|
mm_processor_kwargs={"vision_min_num_patches": 3_000_000_000},
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
VLLMValidationError,
|
||||||
|
match="Per-request mm_processor_kwargs are disabled",
|
||||||
|
):
|
||||||
|
await ServingScores.flash_late_interaction(serving, request)
|
||||||
|
serving._preprocessing.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module", params=[True, False])
|
@pytest.fixture(scope="module", params=[True, False])
|
||||||
def server(request):
|
def server(request):
|
||||||
args = [
|
args = [
|
||||||
|
|||||||
@@ -1450,6 +1450,7 @@ class ChatTemplateConfig:
|
|||||||
chat_template: str | None = None
|
chat_template: str | None = None
|
||||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto"
|
chat_template_content_format: ChatTemplateContentFormatOption = "auto"
|
||||||
trust_request_chat_template: bool = False
|
trust_request_chat_template: bool = False
|
||||||
|
trust_request_mm_kwargs: bool = False
|
||||||
|
|
||||||
|
|
||||||
def validate_chat_template(chat_template: Path | str | None):
|
def validate_chat_template(chat_template: Path | str | None):
|
||||||
|
|||||||
@@ -49,6 +49,34 @@ def validate_cache_salt(cache_salt: object) -> None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_request_mm_kwargs(
|
||||||
|
*,
|
||||||
|
mm_processor_kwargs: dict[str, Any] | None,
|
||||||
|
media_io_kwargs: dict[str, dict[str, Any]] | None,
|
||||||
|
trust_request_mm_kwargs: bool,
|
||||||
|
) -> None:
|
||||||
|
"""Reject untrusted per-request multimodal kwarg overrides."""
|
||||||
|
if trust_request_mm_kwargs:
|
||||||
|
return
|
||||||
|
|
||||||
|
if mm_processor_kwargs:
|
||||||
|
raise VLLMValidationError(
|
||||||
|
"Per-request mm_processor_kwargs are disabled by default because "
|
||||||
|
"they can change multimodal preprocessing resource usage. Start "
|
||||||
|
"the server with --trust-request-mm-kwargs only when clients "
|
||||||
|
"are trusted.",
|
||||||
|
parameter="mm_processor_kwargs",
|
||||||
|
)
|
||||||
|
if media_io_kwargs:
|
||||||
|
raise VLLMValidationError(
|
||||||
|
"Per-request media_io_kwargs are disabled by default because "
|
||||||
|
"they can change multimodal media loading resource usage. Start "
|
||||||
|
"the server with --trust-request-mm-kwargs only when clients "
|
||||||
|
"are trusted.",
|
||||||
|
parameter="media_io_kwargs",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class SpeculativeDecodingMetrics(OpenAIBaseModel):
|
class SpeculativeDecodingMetrics(OpenAIBaseModel):
|
||||||
"""Per-request speculative-decoding acceptance metrics.
|
"""Per-request speculative-decoding acceptance metrics.
|
||||||
|
|
||||||
|
|||||||
@@ -92,6 +92,7 @@ async def init_app_state(
|
|||||||
chat_template=resolved_chat_template,
|
chat_template=resolved_chat_template,
|
||||||
chat_template_content_format=args.chat_template_content_format,
|
chat_template_content_format=args.chat_template_content_format,
|
||||||
trust_request_chat_template=args.trust_request_chat_template,
|
trust_request_chat_template=args.trust_request_chat_template,
|
||||||
|
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
|
||||||
enable_auto_tools=args.enable_auto_tool_choice,
|
enable_auto_tools=args.enable_auto_tool_choice,
|
||||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||||
tool_parser=args.tool_call_parser,
|
tool_parser=args.tool_call_parser,
|
||||||
|
|||||||
@@ -93,6 +93,11 @@ class BaseFrontendArgs:
|
|||||||
"""Whether to trust the chat template provided in the request. If False,
|
"""Whether to trust the chat template provided in the request. If False,
|
||||||
the server will always use the chat template specified by `--chat-template`
|
the server will always use the chat template specified by `--chat-template`
|
||||||
or the ones from tokenizer."""
|
or the ones from tokenizer."""
|
||||||
|
trust_request_mm_kwargs: bool = False
|
||||||
|
"""Whether to trust per-request multimodal kwargs (`mm_processor_kwargs`
|
||||||
|
and `media_io_kwargs`). If False, the server rejects non-empty values
|
||||||
|
because they can change multimodal preprocessing resource usage. Only
|
||||||
|
enable this when API clients are trusted."""
|
||||||
default_chat_template_kwargs: dict[str, Any] | None = None
|
default_chat_template_kwargs: dict[str, Any] | None = None
|
||||||
"""Default keyword arguments to pass to the chat template renderer.
|
"""Default keyword arguments to pass to the chat template renderer.
|
||||||
These will be merged with request-level chat_template_kwargs,
|
These will be merged with request-level chat_template_kwargs,
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ async def init_render_app_state(
|
|||||||
chat_template=resolved_chat_template,
|
chat_template=resolved_chat_template,
|
||||||
chat_template_content_format=args.chat_template_content_format,
|
chat_template_content_format=args.chat_template_content_format,
|
||||||
trust_request_chat_template=args.trust_request_chat_template,
|
trust_request_chat_template=args.trust_request_chat_template,
|
||||||
|
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
|
||||||
enable_auto_tools=args.enable_auto_tool_choice,
|
enable_auto_tools=args.enable_auto_tool_choice,
|
||||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||||
tool_parser=args.tool_call_parser,
|
tool_parser=args.tool_call_parser,
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from vllm import PoolingRequestOutput, envs
|
|||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.chat_utils import ChatTemplateConfig
|
from vllm.entrypoints.chat_utils import ChatTemplateConfig
|
||||||
|
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||||
@@ -60,6 +61,7 @@ class PoolingBaseServing(ABC, BaseServing):
|
|||||||
self.return_tokens_as_token_ids = return_tokens_as_token_ids
|
self.return_tokens_as_token_ids = return_tokens_as_token_ids
|
||||||
self.log_error_stack = log_error_stack
|
self.log_error_stack = log_error_stack
|
||||||
self.chat_template_config = chat_template_config
|
self.chat_template_config = chat_template_config
|
||||||
|
self.trust_request_mm_kwargs = chat_template_config.trust_request_mm_kwargs
|
||||||
|
|
||||||
# Shared thread pool executor for preprocessing and postprocessing.
|
# Shared thread pool executor for preprocessing and postprocessing.
|
||||||
self._executor: Executor = self.renderer._executor
|
self._executor: Executor = self.renderer._executor
|
||||||
@@ -108,6 +110,11 @@ class PoolingBaseServing(ABC, BaseServing):
|
|||||||
request: AnyPoolingRequest,
|
request: AnyPoolingRequest,
|
||||||
raw_request: Request | None = None,
|
raw_request: Request | None = None,
|
||||||
):
|
):
|
||||||
|
validate_request_mm_kwargs(
|
||||||
|
mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
|
||||||
|
media_io_kwargs=getattr(request, "media_io_kwargs", None),
|
||||||
|
trust_request_mm_kwargs=self.trust_request_mm_kwargs,
|
||||||
|
)
|
||||||
base_request_id = self._base_request_id(
|
base_request_id = self._base_request_id(
|
||||||
raw_request, getattr(request, "request_id", None)
|
raw_request, getattr(request, "request_id", None)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -166,6 +166,7 @@ def init_pooling_state(
|
|||||||
chat_template=resolved_chat_template,
|
chat_template=resolved_chat_template,
|
||||||
chat_template_content_format=args.chat_template_content_format,
|
chat_template_content_format=args.chat_template_content_format,
|
||||||
trust_request_chat_template=args.trust_request_chat_template,
|
trust_request_chat_template=args.trust_request_chat_template,
|
||||||
|
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
state.serving_pooling = (
|
state.serving_pooling = (
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from vllm.entrypoints.chat_utils import (
|
|||||||
ChatTemplateContentFormatOption,
|
ChatTemplateContentFormatOption,
|
||||||
ConversationMessage,
|
ConversationMessage,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionNamedToolChoiceParam,
|
ChatCompletionNamedToolChoiceParam,
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
@@ -123,6 +124,7 @@ class OnlineRenderer:
|
|||||||
chat_template: str | None,
|
chat_template: str | None,
|
||||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||||
trust_request_chat_template: bool = False,
|
trust_request_chat_template: bool = False,
|
||||||
|
trust_request_mm_kwargs: bool = False,
|
||||||
enable_auto_tools: bool = False,
|
enable_auto_tools: bool = False,
|
||||||
exclude_tools_when_tool_choice_none: bool = False,
|
exclude_tools_when_tool_choice_none: bool = False,
|
||||||
tool_parser: str | None = None,
|
tool_parser: str | None = None,
|
||||||
@@ -155,6 +157,7 @@ class OnlineRenderer:
|
|||||||
default_chat_template_kwargs or {}
|
default_chat_template_kwargs or {}
|
||||||
)
|
)
|
||||||
self.trust_request_chat_template = trust_request_chat_template
|
self.trust_request_chat_template = trust_request_chat_template
|
||||||
|
self.trust_request_mm_kwargs = trust_request_mm_kwargs
|
||||||
|
|
||||||
self.log_error_stack = log_error_stack
|
self.log_error_stack = log_error_stack
|
||||||
self.supports_browsing = False
|
self.supports_browsing = False
|
||||||
@@ -674,6 +677,11 @@ class OnlineRenderer:
|
|||||||
"""Copied from GenerateBaseServing._preprocess_cmpl."""
|
"""Copied from GenerateBaseServing._preprocess_cmpl."""
|
||||||
renderer = self.renderer
|
renderer = self.renderer
|
||||||
model_config = self.model_config
|
model_config = self.model_config
|
||||||
|
validate_request_mm_kwargs(
|
||||||
|
mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
|
||||||
|
media_io_kwargs=getattr(request, "media_io_kwargs", None),
|
||||||
|
trust_request_mm_kwargs=self.trust_request_mm_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
parsed_prompts = [
|
parsed_prompts = [
|
||||||
(
|
(
|
||||||
@@ -710,6 +718,11 @@ class OnlineRenderer:
|
|||||||
) -> tuple[list[ConversationMessage], list[EngineInput]]:
|
) -> tuple[list[ConversationMessage], list[EngineInput]]:
|
||||||
"""Copied from GenerateBaseServing._preprocess_chat."""
|
"""Copied from GenerateBaseServing._preprocess_chat."""
|
||||||
renderer = self.renderer
|
renderer = self.renderer
|
||||||
|
validate_request_mm_kwargs(
|
||||||
|
mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
|
||||||
|
media_io_kwargs=getattr(request, "media_io_kwargs", None),
|
||||||
|
trust_request_mm_kwargs=self.trust_request_mm_kwargs,
|
||||||
|
)
|
||||||
mm_config = self.model_config.multimodal_config
|
mm_config = self.model_config.multimodal_config
|
||||||
|
|
||||||
default_template_kwargs = merge_kwargs(
|
default_template_kwargs = merge_kwargs(
|
||||||
|
|||||||
Reference in New Issue
Block a user