[Frontend] Replace return_mm_kwargs with an mm_payload enum on render requests (#61427)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-10-15 12:41:46 +00:00
committed by GitHub
parent c6b772cf6b
commit 252be28c18
9 changed files with 171 additions and 44 deletions
+7 -6
View File
@@ -6,6 +6,7 @@ from fastapi import Request
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.generate.base.protocol import MMPayload
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.models.serving import (
@@ -169,7 +170,7 @@ class ServingRender(BaseServing):
request_id=request_id,
token_ids=token_ids,
features=self._extract_mm_features(
engine_input, include_mm_kwargs=request.return_mm_kwargs
engine_input, mm_payload=request.mm_payload
),
sampling_params=params,
kv_transfer_params=kv_transfer_params,
@@ -198,7 +199,7 @@ class ServingRender(BaseServing):
chat_req = AnthropicServingMessages.to_chat_completion_request(
request, merge_inline_system=self._merge_inline_system
)
chat_req.return_mm_kwargs = request.return_mm_kwargs
chat_req.mm_payload = request.mm_payload
return await self.render_chat_request(chat_req, raw_request)
async def render_completion_request(
@@ -257,7 +258,7 @@ class ServingRender(BaseServing):
request_id=request_id,
token_ids=token_ids,
features=self._extract_mm_features(
engine_input, include_mm_kwargs=request.return_mm_kwargs
engine_input, mm_payload=request.mm_payload
),
sampling_params=params,
kv_transfer_params=kv_transfer_params,
@@ -322,7 +323,7 @@ class ServingRender(BaseServing):
request_id=request.request_id,
token_ids=list(token_ids),
features=self._extract_mm_features(
engine_input, include_mm_kwargs=request.return_mm_kwargs
engine_input, mm_payload=request.mm_payload
),
sampling_params=params,
model=request.model,
@@ -375,10 +376,10 @@ class ServingRender(BaseServing):
self,
engine_input: EngineInput,
*,
include_mm_kwargs: bool = True,
mm_payload: MMPayload = "tensors",
) -> MultiModalFeatures | None:
return extract_mm_features(
engine_input,
metadata_fields_for=self._placeholder_metadata_fields,
include_mm_kwargs=include_mm_kwargs,
mm_payload=mm_payload,
)
@@ -7,6 +7,7 @@ from __future__ import annotations
from collections.abc import Callable, Collection, Sequence
from typing import cast
from vllm.entrypoints.generate.base.protocol import MMPayload
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
decode_mm_kwargs_item,
encode_mm_kwargs_item,
@@ -139,7 +140,7 @@ def extract_mm_features(
engine_input: EngineInput,
*,
metadata_fields_for: Callable[[str], Collection[str]] | None = None,
include_mm_kwargs: bool = True,
mm_payload: MMPayload = "tensors",
) -> MultiModalFeatures | None:
"""Extract multimodal features from a rendered engine prompt.
@@ -147,7 +148,7 @@ def extract_mm_features(
intersection of processed kwargs that prefill needs after EC transfer:
fields declared as embedding metadata, plus fields marked
``keep_on_cpu`` (for example M-RoPE grid dims). With
``include_mm_kwargs=False`` neither is serialized.
``mm_payload="none"`` neither is serialized.
"""
if engine_input.get("type") != "multimodal":
return None
@@ -159,7 +160,7 @@ def extract_mm_features(
kwargs_data: dict[str, list[str | None]] | None = None
mm_metadata: dict[str, list[str | None]] | None = None
if include_mm_kwargs:
if mm_payload == "tensors":
raw_mm_kwargs = mm_engine_input.get("mm_kwargs")
if raw_mm_kwargs:
kwargs_data, mm_metadata = _encode_mm_kwargs_with_metadata(