diff --git a/docs/serving/online_serving/renderer.md b/docs/serving/online_serving/renderer.md index 93ecf34f4f..e44a9de319 100644 --- a/docs/serving/online_serving/renderer.md +++ b/docs/serving/online_serving/renderer.md @@ -154,11 +154,15 @@ should split those fields: continue to work. Callers that need only the token layout and item hashes, such as cache-aware -routers, can set `"return_mm_kwargs": false` on the render request. The response +routers, can set `"mm_payload": "none"` on the render request. The response then keeps `mm_hashes` and `mm_placeholders` and sets `kwargs_data` and `mm_metadata` to null, so the processed tensors are neither serialized nor sent. Do not forward such a response to `/inference/v1/generate`, which reads a null -`kwargs_data` as every item being cached. +`kwargs_data` as every item being cached. The default, `"tensors"`, returns +both fields. + +`"return_mm_kwargs": false` is a deprecated alias for `"mm_payload": "none"` +and is ignored when `mm_payload` is set. ## Example diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py b/tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py index 62223f1d7c..da994d7501 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py @@ -8,6 +8,10 @@ import pytest import torch from pydantic import ValidationError +from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.completion.protocol import CompletionRequest +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.entrypoints.scale_out.token_in_token_out.mm_features import ( extract_mm_features, merge_mm_kwargs_items, @@ -173,7 +177,7 @@ def test_extract_without_kwargs_keeps_layout(): engine_input, _pixel_values, _image_grid_thw = _image_engine_input() full = extract_mm_features(engine_input) - layout = extract_mm_features(engine_input, include_mm_kwargs=False) + layout = extract_mm_features(engine_input, mm_payload="none") assert full is not None and layout is not None assert layout.kwargs_data is None @@ -182,6 +186,40 @@ def test_extract_without_kwargs_keeps_layout(): assert layout.mm_placeholders == full.mm_placeholders +_MESSAGES = [{"role": "user", "content": "hi"}] + + +@pytest.mark.parametrize( + ("request_cls", "body"), + [ + (ChatCompletionRequest, {"model": "m", "messages": _MESSAGES}), + (CompletionRequest, {"model": "m", "prompt": "hi"}), + (ResponsesRequest, {"model": "m", "input": "hi"}), + ( + AnthropicMessagesRequest, + {"model": "m", "max_tokens": 8, "messages": _MESSAGES}, + ), + ], +) +def test_return_mm_kwargs_is_alias_for_mm_payload(request_cls, body): + """Clients of the deprecated bool keep getting layout-only responses.""" + assert request_cls.model_validate(body).mm_payload == "tensors" + + for return_mm_kwargs, expected in [(False, "none"), (True, "tensors")]: + request = request_cls.model_validate( + {**body, "return_mm_kwargs": return_mm_kwargs} + ) + assert request.mm_payload == expected + + explicit = request_cls.model_validate( + {**body, "return_mm_kwargs": False, "mm_payload": "tensors"} + ) + assert explicit.mm_payload == "tensors" + + with pytest.raises(ValidationError): + request_cls.model_validate({**body, "mm_payload": "source"}) + + def test_extract_includes_declared_placeholder_metadata_fields(): """EC placeholder metadata is kept even when keep_on_cpu is unset.""" engine_input, _pixel_values, image_grid_thw = _image_engine_input( diff --git a/vllm/entrypoints/anthropic/protocol.py b/vllm/entrypoints/anthropic/protocol.py index b2d29e0b34..0bf7d5c34c 100644 --- a/vllm/entrypoints/anthropic/protocol.py +++ b/vllm/entrypoints/anthropic/protocol.py @@ -8,6 +8,10 @@ from typing import Annotated, Any, Literal from pydantic import BaseModel, Field, field_validator, model_validator import vllm.envs as envs +from vllm.entrypoints.generate.base.protocol import ( + MMPayload, + resolve_return_mm_kwargs_alias, +) class AnthropicError(BaseModel): @@ -229,15 +233,25 @@ class AnthropicMessagesRequest(BaseModel): top_p: float | None = None # vLLM-specific fields that are not in Anthropic spec - return_mm_kwargs: bool = Field( - default=True, + mm_payload: MMPayload = Field( + default="tensors", description=( - "If false, the render response's `features` set `kwargs_data` " - "and `mm_metadata` to null, for callers that need only the token " - "layout and item hashes, such as cache-aware routers. Do not send " - "such a response to `/inference/v1/generate`, which reads a null " - "`kwargs_data` as every item being cached. Only supported on the " - "render endpoints; ignored on regular generation endpoints." + "What the render response's `features` carry for multimodal " + "items. 'tensors' serializes the processed tensors into " + "`kwargs_data` and `mm_metadata`. 'none' sets both to null for " + "callers that need only the token layout and item hashes, such as " + "cache-aware routers. Do not send a 'none' response to " + "`/inference/v1/generate` which reads a null `kwargs_data` as " + "every item being cached. Only supported on the render endpoints. " + "Ignored on regular generation endpoints." + ), + ) + return_mm_kwargs: bool | None = Field( + default=None, + deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field", + description=( + "Deprecated alias: false is the same as `mm_payload='none'`. " + "Ignored when `mm_payload` is set." ), ) cache_salt: str | None = Field( @@ -304,6 +318,11 @@ class AnthropicMessagesRequest(BaseModel): _reject_tool_changes(v) return v + @model_validator(mode="before") + @classmethod + def apply_return_mm_kwargs_alias(cls, data: Any) -> Any: + return resolve_return_mm_kwargs_alias(data) + @model_validator(mode="after") def validate_thinking_budget(self) -> "AnthropicMessagesRequest": # P/D prefill legs are sent with max_tokens=1 and never decode; the diff --git a/vllm/entrypoints/generate/base/protocol.py b/vllm/entrypoints/generate/base/protocol.py index 9781bf935f..a0122cadf5 100644 --- a/vllm/entrypoints/generate/base/protocol.py +++ b/vllm/entrypoints/generate/base/protocol.py @@ -33,6 +33,8 @@ TopLogprobsParam: TypeAlias = Annotated[ BeforeValidator(lambda v: 0 if v is None else v, json_schema_input_type=int | None), ] +MMPayload: TypeAlias = Literal["tensors", "none"] + _CACHE_SALT_FORBIDDEN_CHARS = frozenset("@/\\\x00") _MAX_CACHE_SALT_LENGTH = 128 @@ -56,6 +58,17 @@ def validate_cache_salt(cache_salt: object) -> None: ) +def resolve_return_mm_kwargs_alias(data: Any) -> Any: + """Map the deprecated `return_mm_kwargs=False` onto `mm_payload="none"`.""" + if ( + isinstance(data, dict) + and data.get("return_mm_kwargs") is False + and "mm_payload" not in data + ): + data = {**data, "mm_payload": "none"} + return data + + def validate_request_mm_kwargs( *, mm_processor_kwargs: dict[str, Any] | None, diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index d2e09851eb..79dd9605e6 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -24,11 +24,13 @@ from vllm.entrypoints.generate.base.protocol import ( DeltaMessage, FunctionCall, FunctionDefinition, + MMPayload, PerRequestMetrics, StopParam, StreamOptions, ToolCall, TopLogprobsParam, + resolve_return_mm_kwargs_alias, structured_outputs_from_response_format, validate_cache_salt, validate_structural_tag_response_format, @@ -458,15 +460,25 @@ class ChatCompletionRequest(OpenAIBaseModel): ), ) - return_mm_kwargs: bool = Field( - default=True, + mm_payload: MMPayload = Field( + default="tensors", description=( - "If false, the render response's `features` set `kwargs_data` " - "and `mm_metadata` to null, for callers that need only the token " - "layout and item hashes, such as cache-aware routers. Do not send " - "such a response to `/inference/v1/generate`, which reads a null " - "`kwargs_data` as every item being cached. Only supported on the " - "render endpoints; ignored on regular generation endpoints." + "What the render response's `features` carry for multimodal " + "items. 'tensors' serializes the processed tensors into " + "`kwargs_data` and `mm_metadata`. 'none' sets both to null for " + "callers that need only the token layout and item hashes, such as " + "cache-aware routers. Do not send a 'none' response to " + "`/inference/v1/generate` which reads a null `kwargs_data` as " + "every item being cached. Only supported on the render endpoints. " + "Ignored on regular generation endpoints." + ), + ) + return_mm_kwargs: bool | None = Field( + default=None, + deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field", + description=( + "Deprecated alias: false is the same as `mm_payload='none'`. " + "Ignored when `mm_payload` is set." ), ) @@ -526,6 +538,11 @@ class ChatCompletionRequest(OpenAIBaseModel): # --8<-- [end:chat-completion-extra-params] + @model_validator(mode="before") + @classmethod + def apply_return_mm_kwargs_alias(cls, data: Any) -> Any: + return resolve_return_mm_kwargs_alias(data) + @model_validator(mode="before") @classmethod def check_cache_salt_support(cls, data: Any) -> Any: diff --git a/vllm/entrypoints/openai/completion/protocol.py b/vllm/entrypoints/openai/completion/protocol.py index e99de1e286..4da67e607a 100644 --- a/vllm/entrypoints/openai/completion/protocol.py +++ b/vllm/entrypoints/openai/completion/protocol.py @@ -12,9 +12,11 @@ import vllm.envs as envs from vllm.config import ModelConfig from vllm.entrypoints.generate.base.protocol import ( AnyResponseFormat, + MMPayload, PerRequestMetrics, StopParam, StreamOptions, + resolve_return_mm_kwargs_alias, structured_outputs_from_response_format, validate_cache_salt, validate_structural_tag_response_format, @@ -197,15 +199,25 @@ class CompletionRequest(OpenAIBaseModel): ), ) - return_mm_kwargs: bool = Field( - default=True, + mm_payload: MMPayload = Field( + default="tensors", description=( - "If false, the render response's `features` set `kwargs_data` " - "and `mm_metadata` to null, for callers that need only the token " - "layout and item hashes, such as cache-aware routers. Do not send " - "such a response to `/inference/v1/generate`, which reads a null " - "`kwargs_data` as every item being cached. Only supported on the " - "render endpoints; ignored on regular generation endpoints." + "What the render response's `features` carry for multimodal " + "items. 'tensors' serializes the processed tensors into " + "`kwargs_data` and `mm_metadata`. 'none' sets both to null for " + "callers that need only the token layout and item hashes, such as " + "cache-aware routers. Do not send a 'none' response to " + "`/inference/v1/generate` which reads a null `kwargs_data` as " + "every item being cached. Only supported on the render endpoints. " + "Ignored on regular generation endpoints." + ), + ) + return_mm_kwargs: bool | None = Field( + default=None, + deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field", + description=( + "Deprecated alias: false is the same as `mm_payload='none'`. " + "Ignored when `mm_payload` is set." ), ) @@ -402,6 +414,11 @@ class CompletionRequest(OpenAIBaseModel): routed_experts_prompt_start=self.routed_experts_prompt_start, ) + @model_validator(mode="before") + @classmethod + def apply_return_mm_kwargs_alias(cls, data: Any) -> Any: + return resolve_return_mm_kwargs_alias(data) + @model_validator(mode="before") @classmethod def check_cache_salt_support(cls, data: Any) -> Any: diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py index 0e94bca51f..3ddf81b288 100644 --- a/vllm/entrypoints/openai/responses/protocol.py +++ b/vllm/entrypoints/openai/responses/protocol.py @@ -60,9 +60,11 @@ from pydantic import ( from vllm.config import ModelConfig from vllm.entrypoints.generate.base.protocol import ( + MMPayload, PerRequestMetrics, StopParam, TopLogprobsParam, + resolve_return_mm_kwargs_alias, validate_cache_salt, ) from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel @@ -335,15 +337,25 @@ class ResponsesRequest(OpenAIBaseModel): "if the served model does not use priority scheduling." ), ) - return_mm_kwargs: bool = Field( - default=True, + mm_payload: MMPayload = Field( + default="tensors", description=( - "If false, the render response's `features` set `kwargs_data` " - "and `mm_metadata` to null, for callers that need only the token " - "layout and item hashes, such as cache-aware routers. Do not send " - "such a response to `/inference/v1/generate`, which reads a null " - "`kwargs_data` as every item being cached. Only supported on the " - "render endpoints; ignored on regular generation endpoints." + "What the render response's `features` carry for multimodal " + "items. 'tensors' serializes the processed tensors into " + "`kwargs_data` and `mm_metadata`. 'none' sets both to null for " + "callers that need only the token layout and item hashes, such as " + "cache-aware routers. Do not send a 'none' response to " + "`/inference/v1/generate` which reads a null `kwargs_data` as " + "every item being cached. Only supported on the render endpoints. " + "Ignored on regular generation endpoints." + ), + ) + return_mm_kwargs: bool | None = Field( + default=None, + deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field", + description=( + "Deprecated alias: false is the same as `mm_payload='none'`. " + "Ignored when `mm_payload` is set." ), ) cache_salt: str | None = Field( @@ -571,6 +583,11 @@ class ResponsesRequest(OpenAIBaseModel): and "message.output_text.logprobs" in self.include ) + @model_validator(mode="before") + @classmethod + def apply_return_mm_kwargs_alias(cls, data: Any) -> Any: + return resolve_return_mm_kwargs_alias(data) + @model_validator(mode="before") @classmethod def check_cache_salt_support(cls, data: Any) -> Any: diff --git a/vllm/entrypoints/scale_out/render/serving.py b/vllm/entrypoints/scale_out/render/serving.py index 54d6606427..64486d017a 100644 --- a/vllm/entrypoints/scale_out/render/serving.py +++ b/vllm/entrypoints/scale_out/render/serving.py @@ -6,6 +6,7 @@ from fastapi import Request from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest from vllm.entrypoints.anthropic.serving import AnthropicServingMessages +from vllm.entrypoints.generate.base.protocol import MMPayload from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.completion.protocol import CompletionRequest from vllm.entrypoints.openai.models.serving import ( @@ -169,7 +170,7 @@ class ServingRender(BaseServing): request_id=request_id, token_ids=token_ids, features=self._extract_mm_features( - engine_input, include_mm_kwargs=request.return_mm_kwargs + engine_input, mm_payload=request.mm_payload ), sampling_params=params, kv_transfer_params=kv_transfer_params, @@ -198,7 +199,7 @@ class ServingRender(BaseServing): chat_req = AnthropicServingMessages.to_chat_completion_request( request, merge_inline_system=self._merge_inline_system ) - chat_req.return_mm_kwargs = request.return_mm_kwargs + chat_req.mm_payload = request.mm_payload return await self.render_chat_request(chat_req, raw_request) async def render_completion_request( @@ -257,7 +258,7 @@ class ServingRender(BaseServing): request_id=request_id, token_ids=token_ids, features=self._extract_mm_features( - engine_input, include_mm_kwargs=request.return_mm_kwargs + engine_input, mm_payload=request.mm_payload ), sampling_params=params, kv_transfer_params=kv_transfer_params, @@ -322,7 +323,7 @@ class ServingRender(BaseServing): request_id=request.request_id, token_ids=list(token_ids), features=self._extract_mm_features( - engine_input, include_mm_kwargs=request.return_mm_kwargs + engine_input, mm_payload=request.mm_payload ), sampling_params=params, model=request.model, @@ -375,10 +376,10 @@ class ServingRender(BaseServing): self, engine_input: EngineInput, *, - include_mm_kwargs: bool = True, + mm_payload: MMPayload = "tensors", ) -> MultiModalFeatures | None: return extract_mm_features( engine_input, metadata_fields_for=self._placeholder_metadata_fields, - include_mm_kwargs=include_mm_kwargs, + mm_payload=mm_payload, ) diff --git a/vllm/entrypoints/scale_out/token_in_token_out/mm_features.py b/vllm/entrypoints/scale_out/token_in_token_out/mm_features.py index a0cb0d4b5e..124d602867 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/mm_features.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/mm_features.py @@ -7,6 +7,7 @@ from __future__ import annotations from collections.abc import Callable, Collection, Sequence from typing import cast +from vllm.entrypoints.generate.base.protocol import MMPayload from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import ( decode_mm_kwargs_item, encode_mm_kwargs_item, @@ -139,7 +140,7 @@ def extract_mm_features( engine_input: EngineInput, *, metadata_fields_for: Callable[[str], Collection[str]] | None = None, - include_mm_kwargs: bool = True, + mm_payload: MMPayload = "tensors", ) -> MultiModalFeatures | None: """Extract multimodal features from a rendered engine prompt. @@ -147,7 +148,7 @@ def extract_mm_features( intersection of processed kwargs that prefill needs after EC transfer: fields declared as embedding metadata, plus fields marked ``keep_on_cpu`` (for example M-RoPE grid dims). With - ``include_mm_kwargs=False`` neither is serialized. + ``mm_payload="none"`` neither is serialized. """ if engine_input.get("type") != "multimodal": return None @@ -159,7 +160,7 @@ def extract_mm_features( kwargs_data: dict[str, list[str | None]] | None = None mm_metadata: dict[str, list[str | None]] | None = None - if include_mm_kwargs: + if mm_payload == "tensors": raw_mm_kwargs = mm_engine_input.get("mm_kwargs") if raw_mm_kwargs: kwargs_data, mm_metadata = _encode_mm_kwargs_with_metadata(