diff --git a/docs/serving/online_serving/renderer.md b/docs/serving/online_serving/renderer.md index 79d0ade85b..93ecf34f4f 100644 --- a/docs/serving/online_serving/renderer.md +++ b/docs/serving/online_serving/renderer.md @@ -153,6 +153,13 @@ should split those fields: - Legacy clients that ignore `mm_metadata` and keep sending `kwargs_data` continue to work. +Callers that need only the token layout and item hashes, such as cache-aware +routers, can set `"return_mm_kwargs": false` on the render request. The response +then keeps `mm_hashes` and `mm_placeholders` and sets `kwargs_data` and +`mm_metadata` to null, so the processed tensors are neither serialized nor sent. +Do not forward such a response to `/inference/v1/generate`, which reads a null +`kwargs_data` as every item being cached. + ## Example The example below shows how a disaggregated encode / prefill coordinator can diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py b/tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py index 5cd60a959f..62223f1d7c 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py @@ -168,6 +168,20 @@ def test_render_extracts_metadata_fields_separately(): assert torch.equal(full_data["pixel_values"].data, pixel_values.data) +def test_extract_without_kwargs_keeps_layout(): + """Layout-only callers get hashes and placeholders, no serialized tensors.""" + engine_input, _pixel_values, _image_grid_thw = _image_engine_input() + + full = extract_mm_features(engine_input) + layout = extract_mm_features(engine_input, include_mm_kwargs=False) + + assert full is not None and layout is not None + assert layout.kwargs_data is None + assert layout.mm_metadata is None + assert layout.mm_hashes == full.mm_hashes + assert layout.mm_placeholders == full.mm_placeholders + + def test_extract_includes_declared_placeholder_metadata_fields(): """EC placeholder metadata is kept even when keep_on_cpu is unset.""" engine_input, _pixel_values, image_grid_thw = _image_engine_input( diff --git a/vllm/entrypoints/anthropic/protocol.py b/vllm/entrypoints/anthropic/protocol.py index 5997d2e3d2..b2d29e0b34 100644 --- a/vllm/entrypoints/anthropic/protocol.py +++ b/vllm/entrypoints/anthropic/protocol.py @@ -229,6 +229,17 @@ class AnthropicMessagesRequest(BaseModel): top_p: float | None = None # vLLM-specific fields that are not in Anthropic spec + return_mm_kwargs: bool = Field( + default=True, + description=( + "If false, the render response's `features` set `kwargs_data` " + "and `mm_metadata` to null, for callers that need only the token " + "layout and item hashes, such as cache-aware routers. Do not send " + "such a response to `/inference/v1/generate`, which reads a null " + "`kwargs_data` as every item being cached. Only supported on the " + "render endpoints; ignored on regular generation endpoints." + ), + ) cache_salt: str | None = Field( default=None, min_length=1, diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 15905b8a3d..dee8f283cf 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -458,6 +458,18 @@ class ChatCompletionRequest(OpenAIBaseModel): ), ) + return_mm_kwargs: bool = Field( + default=True, + description=( + "If false, the render response's `features` set `kwargs_data` " + "and `mm_metadata` to null, for callers that need only the token " + "layout and item hashes, such as cache-aware routers. Do not send " + "such a response to `/inference/v1/generate`, which reads a null " + "`kwargs_data` as every item being cached. Only supported on the " + "render endpoints; ignored on regular generation endpoints." + ), + ) + cache_salt: str | None = Field( default=None, min_length=1, diff --git a/vllm/entrypoints/openai/completion/protocol.py b/vllm/entrypoints/openai/completion/protocol.py index 78722b45ec..f26dea2e1f 100644 --- a/vllm/entrypoints/openai/completion/protocol.py +++ b/vllm/entrypoints/openai/completion/protocol.py @@ -197,6 +197,18 @@ class CompletionRequest(OpenAIBaseModel): ), ) + return_mm_kwargs: bool = Field( + default=True, + description=( + "If false, the render response's `features` set `kwargs_data` " + "and `mm_metadata` to null, for callers that need only the token " + "layout and item hashes, such as cache-aware routers. Do not send " + "such a response to `/inference/v1/generate`, which reads a null " + "`kwargs_data` as every item being cached. Only supported on the " + "render endpoints; ignored on regular generation endpoints." + ), + ) + cache_salt: str | None = Field( default=None, min_length=1, diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py index d1a4ea4f6c..406ab2d081 100644 --- a/vllm/entrypoints/openai/responses/protocol.py +++ b/vllm/entrypoints/openai/responses/protocol.py @@ -330,6 +330,17 @@ class ResponsesRequest(OpenAIBaseModel): "if the served model does not use priority scheduling." ), ) + return_mm_kwargs: bool = Field( + default=True, + description=( + "If false, the render response's `features` set `kwargs_data` " + "and `mm_metadata` to null, for callers that need only the token " + "layout and item hashes, such as cache-aware routers. Do not send " + "such a response to `/inference/v1/generate`, which reads a null " + "`kwargs_data` as every item being cached. Only supported on the " + "render endpoints; ignored on regular generation endpoints." + ), + ) cache_salt: str | None = Field( default=None, min_length=1, diff --git a/vllm/entrypoints/scale_out/render/serving.py b/vllm/entrypoints/scale_out/render/serving.py index bddbe90f28..84771c5f73 100644 --- a/vllm/entrypoints/scale_out/render/serving.py +++ b/vllm/entrypoints/scale_out/render/serving.py @@ -144,7 +144,9 @@ class ServingRender(BaseServing): return GenerateRequest( request_id=request_id, token_ids=token_ids, - features=self._extract_mm_features(engine_input), + features=self._extract_mm_features( + engine_input, include_mm_kwargs=request.return_mm_kwargs + ), sampling_params=params, model=request.model, reasoning_ended=reasoning_ended, @@ -169,6 +171,7 @@ class ServingRender(BaseServing): chat_req = AnthropicServingMessages.to_chat_completion_request( request, merge_inline_system=self._merge_inline_system ) + chat_req.return_mm_kwargs = request.return_mm_kwargs return await self.render_chat_request(chat_req) async def render_completion_request( @@ -217,7 +220,9 @@ class ServingRender(BaseServing): GenerateRequest( request_id=request_id, token_ids=token_ids, - features=self._extract_mm_features(engine_input), + features=self._extract_mm_features( + engine_input, include_mm_kwargs=request.return_mm_kwargs + ), sampling_params=params, model=request.model, stream=bool(request.stream), @@ -277,7 +282,9 @@ class ServingRender(BaseServing): return GenerateRequest( request_id=request.request_id, token_ids=list(token_ids), - features=self._extract_mm_features(engine_input), + features=self._extract_mm_features( + engine_input, include_mm_kwargs=request.return_mm_kwargs + ), sampling_params=params, model=request.model, reasoning_parser_kwargs=self._reasoning_parser_kwargs(request), @@ -328,8 +335,11 @@ class ServingRender(BaseServing): def _extract_mm_features( self, engine_input: EngineInput, + *, + include_mm_kwargs: bool = True, ) -> MultiModalFeatures | None: return extract_mm_features( engine_input, metadata_fields_for=self._placeholder_metadata_fields, + include_mm_kwargs=include_mm_kwargs, ) diff --git a/vllm/entrypoints/scale_out/token_in_token_out/mm_features.py b/vllm/entrypoints/scale_out/token_in_token_out/mm_features.py index 62d006783a..a0cb0d4b5e 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/mm_features.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/mm_features.py @@ -139,13 +139,15 @@ def extract_mm_features( engine_input: EngineInput, *, metadata_fields_for: Callable[[str], Collection[str]] | None = None, + include_mm_kwargs: bool = True, ) -> MultiModalFeatures | None: """Extract multimodal features from a rendered engine prompt. Returns ``None`` for text-only prompts. ``mm_metadata`` keeps the intersection of processed kwargs that prefill needs after EC transfer: fields declared as embedding metadata, plus fields marked - ``keep_on_cpu`` (for example M-RoPE grid dims). + ``keep_on_cpu`` (for example M-RoPE grid dims). With + ``include_mm_kwargs=False`` neither is serialized. """ if engine_input.get("type") != "multimodal": return None @@ -157,11 +159,13 @@ def extract_mm_features( kwargs_data: dict[str, list[str | None]] | None = None mm_metadata: dict[str, list[str | None]] | None = None - if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"): - kwargs_data, mm_metadata = _encode_mm_kwargs_with_metadata( - raw_mm_kwargs, - metadata_fields_for=metadata_fields_for, - ) + if include_mm_kwargs: + raw_mm_kwargs = mm_engine_input.get("mm_kwargs") + if raw_mm_kwargs: + kwargs_data, mm_metadata = _encode_mm_kwargs_with_metadata( + raw_mm_kwargs, + metadata_fields_for=metadata_fields_for, + ) return MultiModalFeatures( mm_hashes=mm_hashes,