[Frontend] Replace return_mm_kwargs with an mm_payload enum on render requests (#61427)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>

docs/serving/online_serving/renderer.md

@@ -154,11 +154,15 @@ should split those fields:
   continue to work.
 
 Callers that need only the token layout and item hashes, such as cache-aware
-routers, can set `"return_mm_kwargs": false` on the render request. The response
+routers, can set `"mm_payload": "none"` on the render request. The response
 then keeps `mm_hashes` and `mm_placeholders` and sets `kwargs_data` and
 `mm_metadata` to null, so the processed tensors are neither serialized nor sent.
 Do not forward such a response to `/inference/v1/generate`, which reads a null
-`kwargs_data` as every item being cached.
+`kwargs_data` as every item being cached. The default, `"tensors"`, returns
+both fields.
+
+`"return_mm_kwargs": false` is a deprecated alias for `"mm_payload": "none"`
+and is ignored when `mm_payload` is set.
 
 ## Example
 

tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py

@@ -8,6 +8,10 @@ import pytest
 import torch
 from pydantic import ValidationError
 
+from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
+from vllm.entrypoints.openai.completion.protocol import CompletionRequest
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
 from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
     extract_mm_features,
     merge_mm_kwargs_items,
@@ -173,7 +177,7 @@ def test_extract_without_kwargs_keeps_layout():
     engine_input, _pixel_values, _image_grid_thw = _image_engine_input()
 
     full = extract_mm_features(engine_input)
-    layout = extract_mm_features(engine_input, include_mm_kwargs=False)
+    layout = extract_mm_features(engine_input, mm_payload="none")
 
     assert full is not None and layout is not None
     assert layout.kwargs_data is None
@@ -182,6 +186,40 @@ def test_extract_without_kwargs_keeps_layout():
     assert layout.mm_placeholders == full.mm_placeholders
 
 
+_MESSAGES = [{"role": "user", "content": "hi"}]
+
+
+@pytest.mark.parametrize(
+    ("request_cls", "body"),
+    [
+        (ChatCompletionRequest, {"model": "m", "messages": _MESSAGES}),
+        (CompletionRequest, {"model": "m", "prompt": "hi"}),
+        (ResponsesRequest, {"model": "m", "input": "hi"}),
+        (
+            AnthropicMessagesRequest,
+            {"model": "m", "max_tokens": 8, "messages": _MESSAGES},
+        ),
+    ],
+)
+def test_return_mm_kwargs_is_alias_for_mm_payload(request_cls, body):
+    """Clients of the deprecated bool keep getting layout-only responses."""
+    assert request_cls.model_validate(body).mm_payload == "tensors"
+
+    for return_mm_kwargs, expected in [(False, "none"), (True, "tensors")]:
+        request = request_cls.model_validate(
+            {**body, "return_mm_kwargs": return_mm_kwargs}
+        )
+        assert request.mm_payload == expected
+
+    explicit = request_cls.model_validate(
+        {**body, "return_mm_kwargs": False, "mm_payload": "tensors"}
+    )
+    assert explicit.mm_payload == "tensors"
+
+    with pytest.raises(ValidationError):
+        request_cls.model_validate({**body, "mm_payload": "source"})
+
+
 def test_extract_includes_declared_placeholder_metadata_fields():
     """EC placeholder metadata is kept even when keep_on_cpu is unset."""
     engine_input, _pixel_values, image_grid_thw = _image_engine_input(

vllm/entrypoints/anthropic/protocol.py

@@ -8,6 +8,10 @@ from typing import Annotated, Any, Literal
 from pydantic import BaseModel, Field, field_validator, model_validator
 
 import vllm.envs as envs
+from vllm.entrypoints.generate.base.protocol import (
+    MMPayload,
+    resolve_return_mm_kwargs_alias,
+)
 
 
 class AnthropicError(BaseModel):
@@ -229,15 +233,25 @@ class AnthropicMessagesRequest(BaseModel):
     top_p: float | None = None
 
     # vLLM-specific fields that are not in Anthropic spec
-    return_mm_kwargs: bool = Field(
-        default=True,
+    mm_payload: MMPayload = Field(
+        default="tensors",
         description=(
-            "If false, the render response's `features` set `kwargs_data` "
-            "and `mm_metadata` to null, for callers that need only the token "
-            "layout and item hashes, such as cache-aware routers. Do not send "
-            "such a response to `/inference/v1/generate`, which reads a null "
-            "`kwargs_data` as every item being cached. Only supported on the "
-            "render endpoints; ignored on regular generation endpoints."
+            "What the render response's `features` carry for multimodal "
+            "items. 'tensors' serializes the processed tensors into "
+            "`kwargs_data` and `mm_metadata`. 'none' sets both to null for "
+            "callers that need only the token layout and item hashes, such as "
+            "cache-aware routers. Do not send a 'none' response to "
+            "`/inference/v1/generate` which reads a null `kwargs_data` as "
+            "every item being cached. Only supported on the render endpoints. "
+            "Ignored on regular generation endpoints."
+        ),
+    )
+    return_mm_kwargs: bool | None = Field(
+        default=None,
+        deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field",
+        description=(
+            "Deprecated alias: false is the same as `mm_payload='none'`. "
+            "Ignored when `mm_payload` is set."
         ),
     )
     cache_salt: str | None = Field(
@@ -304,6 +318,11 @@ class AnthropicMessagesRequest(BaseModel):
         _reject_tool_changes(v)
         return v
 
+    @model_validator(mode="before")
+    @classmethod
+    def apply_return_mm_kwargs_alias(cls, data: Any) -> Any:
+        return resolve_return_mm_kwargs_alias(data)
+
     @model_validator(mode="after")
     def validate_thinking_budget(self) -> "AnthropicMessagesRequest":
         # P/D prefill legs are sent with max_tokens=1 and never decode; the

vllm/entrypoints/generate/base/protocol.py

@@ -33,6 +33,8 @@ TopLogprobsParam: TypeAlias = Annotated[
     BeforeValidator(lambda v: 0 if v is None else v, json_schema_input_type=int | None),
 ]
 
+MMPayload: TypeAlias = Literal["tensors", "none"]
+
 _CACHE_SALT_FORBIDDEN_CHARS = frozenset("@/\\\x00")
 _MAX_CACHE_SALT_LENGTH = 128
 
@@ -56,6 +58,17 @@ def validate_cache_salt(cache_salt: object) -> None:
         )
 
 
+def resolve_return_mm_kwargs_alias(data: Any) -> Any:
+    """Map the deprecated `return_mm_kwargs=False` onto `mm_payload="none"`."""
+    if (
+        isinstance(data, dict)
+        and data.get("return_mm_kwargs") is False
+        and "mm_payload" not in data
+    ):
+        data = {**data, "mm_payload": "none"}
+    return data
+
+
 def validate_request_mm_kwargs(
     *,
     mm_processor_kwargs: dict[str, Any] | None,

vllm/entrypoints/openai/chat_completion/protocol.py

@@ -24,11 +24,13 @@ from vllm.entrypoints.generate.base.protocol import (
     DeltaMessage,
     FunctionCall,
     FunctionDefinition,
+    MMPayload,
     PerRequestMetrics,
     StopParam,
     StreamOptions,
     ToolCall,
     TopLogprobsParam,
+    resolve_return_mm_kwargs_alias,
     structured_outputs_from_response_format,
     validate_cache_salt,
     validate_structural_tag_response_format,
@@ -458,15 +460,25 @@ class ChatCompletionRequest(OpenAIBaseModel):
         ),
     )
 
-    return_mm_kwargs: bool = Field(
-        default=True,
+    mm_payload: MMPayload = Field(
+        default="tensors",
         description=(
-            "If false, the render response's `features` set `kwargs_data` "
-            "and `mm_metadata` to null, for callers that need only the token "
-            "layout and item hashes, such as cache-aware routers. Do not send "
-            "such a response to `/inference/v1/generate`, which reads a null "
-            "`kwargs_data` as every item being cached. Only supported on the "
-            "render endpoints; ignored on regular generation endpoints."
+            "What the render response's `features` carry for multimodal "
+            "items. 'tensors' serializes the processed tensors into "
+            "`kwargs_data` and `mm_metadata`. 'none' sets both to null for "
+            "callers that need only the token layout and item hashes, such as "
+            "cache-aware routers. Do not send a 'none' response to "
+            "`/inference/v1/generate` which reads a null `kwargs_data` as "
+            "every item being cached. Only supported on the render endpoints. "
+            "Ignored on regular generation endpoints."
+        ),
+    )
+    return_mm_kwargs: bool | None = Field(
+        default=None,
+        deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field",
+        description=(
+            "Deprecated alias: false is the same as `mm_payload='none'`. "
+            "Ignored when `mm_payload` is set."
         ),
     )
 
@@ -526,6 +538,11 @@ class ChatCompletionRequest(OpenAIBaseModel):
 
     # --8<-- [end:chat-completion-extra-params]
 
+    @model_validator(mode="before")
+    @classmethod
+    def apply_return_mm_kwargs_alias(cls, data: Any) -> Any:
+        return resolve_return_mm_kwargs_alias(data)
+
     @model_validator(mode="before")
     @classmethod
     def check_cache_salt_support(cls, data: Any) -> Any:

vllm/entrypoints/openai/completion/protocol.py

@@ -12,9 +12,11 @@ import vllm.envs as envs
 from vllm.config import ModelConfig
 from vllm.entrypoints.generate.base.protocol import (
     AnyResponseFormat,
+    MMPayload,
     PerRequestMetrics,
     StopParam,
     StreamOptions,
+    resolve_return_mm_kwargs_alias,
     structured_outputs_from_response_format,
     validate_cache_salt,
     validate_structural_tag_response_format,
@@ -197,15 +199,25 @@ class CompletionRequest(OpenAIBaseModel):
         ),
     )
 
-    return_mm_kwargs: bool = Field(
-        default=True,
+    mm_payload: MMPayload = Field(
+        default="tensors",
         description=(
-            "If false, the render response's `features` set `kwargs_data` "
-            "and `mm_metadata` to null, for callers that need only the token "
-            "layout and item hashes, such as cache-aware routers. Do not send "
-            "such a response to `/inference/v1/generate`, which reads a null "
-            "`kwargs_data` as every item being cached. Only supported on the "
-            "render endpoints; ignored on regular generation endpoints."
+            "What the render response's `features` carry for multimodal "
+            "items. 'tensors' serializes the processed tensors into "
+            "`kwargs_data` and `mm_metadata`. 'none' sets both to null for "
+            "callers that need only the token layout and item hashes, such as "
+            "cache-aware routers. Do not send a 'none' response to "
+            "`/inference/v1/generate` which reads a null `kwargs_data` as "
+            "every item being cached. Only supported on the render endpoints. "
+            "Ignored on regular generation endpoints."
+        ),
+    )
+    return_mm_kwargs: bool | None = Field(
+        default=None,
+        deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field",
+        description=(
+            "Deprecated alias: false is the same as `mm_payload='none'`. "
+            "Ignored when `mm_payload` is set."
         ),
     )
 
@@ -402,6 +414,11 @@ class CompletionRequest(OpenAIBaseModel):
             routed_experts_prompt_start=self.routed_experts_prompt_start,
         )
 
+    @model_validator(mode="before")
+    @classmethod
+    def apply_return_mm_kwargs_alias(cls, data: Any) -> Any:
+        return resolve_return_mm_kwargs_alias(data)
+
     @model_validator(mode="before")
     @classmethod
     def check_cache_salt_support(cls, data: Any) -> Any:

vllm/entrypoints/openai/responses/protocol.py

@@ -60,9 +60,11 @@ from pydantic import (
 
 from vllm.config import ModelConfig
 from vllm.entrypoints.generate.base.protocol import (
+    MMPayload,
     PerRequestMetrics,
     StopParam,
     TopLogprobsParam,
+    resolve_return_mm_kwargs_alias,
     validate_cache_salt,
 )
 from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
@@ -335,15 +337,25 @@ class ResponsesRequest(OpenAIBaseModel):
             "if the served model does not use priority scheduling."
         ),
     )
-    return_mm_kwargs: bool = Field(
-        default=True,
+    mm_payload: MMPayload = Field(
+        default="tensors",
         description=(
-            "If false, the render response's `features` set `kwargs_data` "
-            "and `mm_metadata` to null, for callers that need only the token "
-            "layout and item hashes, such as cache-aware routers. Do not send "
-            "such a response to `/inference/v1/generate`, which reads a null "
-            "`kwargs_data` as every item being cached. Only supported on the "
-            "render endpoints; ignored on regular generation endpoints."
+            "What the render response's `features` carry for multimodal "
+            "items. 'tensors' serializes the processed tensors into "
+            "`kwargs_data` and `mm_metadata`. 'none' sets both to null for "
+            "callers that need only the token layout and item hashes, such as "
+            "cache-aware routers. Do not send a 'none' response to "
+            "`/inference/v1/generate` which reads a null `kwargs_data` as "
+            "every item being cached. Only supported on the render endpoints. "
+            "Ignored on regular generation endpoints."
+        ),
+    )
+    return_mm_kwargs: bool | None = Field(
+        default=None,
+        deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field",
+        description=(
+            "Deprecated alias: false is the same as `mm_payload='none'`. "
+            "Ignored when `mm_payload` is set."
         ),
     )
     cache_salt: str | None = Field(
@@ -571,6 +583,11 @@ class ResponsesRequest(OpenAIBaseModel):
             and "message.output_text.logprobs" in self.include
         )
 
+    @model_validator(mode="before")
+    @classmethod
+    def apply_return_mm_kwargs_alias(cls, data: Any) -> Any:
+        return resolve_return_mm_kwargs_alias(data)
+
     @model_validator(mode="before")
     @classmethod
     def check_cache_salt_support(cls, data: Any) -> Any:

vllm/entrypoints/scale_out/render/serving.py

@@ -6,6 +6,7 @@ from fastapi import Request
 
 from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
 from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
+from vllm.entrypoints.generate.base.protocol import MMPayload
 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
 from vllm.entrypoints.openai.completion.protocol import CompletionRequest
 from vllm.entrypoints.openai.models.serving import (
@@ -169,7 +170,7 @@ class ServingRender(BaseServing):
             request_id=request_id,
             token_ids=token_ids,
             features=self._extract_mm_features(
-                engine_input, include_mm_kwargs=request.return_mm_kwargs
+                engine_input, mm_payload=request.mm_payload
             ),
             sampling_params=params,
             kv_transfer_params=kv_transfer_params,
@@ -198,7 +199,7 @@ class ServingRender(BaseServing):
         chat_req = AnthropicServingMessages.to_chat_completion_request(
             request, merge_inline_system=self._merge_inline_system
         )
-        chat_req.return_mm_kwargs = request.return_mm_kwargs
+        chat_req.mm_payload = request.mm_payload
         return await self.render_chat_request(chat_req, raw_request)
 
     async def render_completion_request(
@@ -257,7 +258,7 @@ class ServingRender(BaseServing):
                     request_id=request_id,
                     token_ids=token_ids,
                     features=self._extract_mm_features(
-                        engine_input, include_mm_kwargs=request.return_mm_kwargs
+                        engine_input, mm_payload=request.mm_payload
                     ),
                     sampling_params=params,
                     kv_transfer_params=kv_transfer_params,
@@ -322,7 +323,7 @@ class ServingRender(BaseServing):
             request_id=request.request_id,
             token_ids=list(token_ids),
             features=self._extract_mm_features(
-                engine_input, include_mm_kwargs=request.return_mm_kwargs
+                engine_input, mm_payload=request.mm_payload
             ),
             sampling_params=params,
             model=request.model,
@@ -375,10 +376,10 @@ class ServingRender(BaseServing):
         self,
         engine_input: EngineInput,
         *,
-        include_mm_kwargs: bool = True,
+        mm_payload: MMPayload = "tensors",
     ) -> MultiModalFeatures | None:
         return extract_mm_features(
             engine_input,
             metadata_fields_for=self._placeholder_metadata_fields,
-            include_mm_kwargs=include_mm_kwargs,
+            mm_payload=mm_payload,
         )

vllm/entrypoints/scale_out/token_in_token_out/mm_features.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 from collections.abc import Callable, Collection, Sequence
 from typing import cast
 
+from vllm.entrypoints.generate.base.protocol import MMPayload
 from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
     decode_mm_kwargs_item,
     encode_mm_kwargs_item,
@@ -139,7 +140,7 @@ def extract_mm_features(
     engine_input: EngineInput,
     *,
     metadata_fields_for: Callable[[str], Collection[str]] | None = None,
-    include_mm_kwargs: bool = True,
+    mm_payload: MMPayload = "tensors",
 ) -> MultiModalFeatures | None:
     """Extract multimodal features from a rendered engine prompt.
 
@@ -147,7 +148,7 @@ def extract_mm_features(
     intersection of processed kwargs that prefill needs after EC transfer:
     fields declared as embedding metadata, plus fields marked
     ``keep_on_cpu`` (for example M-RoPE grid dims). With
-    ``include_mm_kwargs=False`` neither is serialized.
+    ``mm_payload="none"`` neither is serialized.
     """
     if engine_input.get("type") != "multimodal":
         return None
@@ -159,7 +160,7 @@ def extract_mm_features(
 
     kwargs_data: dict[str, list[str | None]] | None = None
     mm_metadata: dict[str, list[str | None]] | None = None
-    if include_mm_kwargs:
+    if mm_payload == "tensors":
         raw_mm_kwargs = mm_engine_input.get("mm_kwargs")
         if raw_mm_kwargs:
             kwargs_data, mm_metadata = _encode_mm_kwargs_with_metadata(