[Frontend] Add return_mm_kwargs to render requests (#60728)

Signed-off-by: Maroon Ayoub <mayoub@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Maroon Ayoub
2026-10-09 08:14:13 +00:00
committed by GitHub
co-authored by Claude
parent f43e4c39e3
commit 8a03ccf7b3
8 changed files with 90 additions and 9 deletions
+11
View File
@@ -229,6 +229,17 @@ class AnthropicMessagesRequest(BaseModel):
top_p: float | None = None
# vLLM-specific fields that are not in Anthropic spec
return_mm_kwargs: bool = Field(
default=True,
description=(
"If false, the render response's `features` set `kwargs_data` "
"and `mm_metadata` to null, for callers that need only the token "
"layout and item hashes, such as cache-aware routers. Do not send "
"such a response to `/inference/v1/generate`, which reads a null "
"`kwargs_data` as every item being cached. Only supported on the "
"render endpoints; ignored on regular generation endpoints."
),
)
cache_salt: str | None = Field(
default=None,
min_length=1,
@@ -458,6 +458,18 @@ class ChatCompletionRequest(OpenAIBaseModel):
),
)
return_mm_kwargs: bool = Field(
default=True,
description=(
"If false, the render response's `features` set `kwargs_data` "
"and `mm_metadata` to null, for callers that need only the token "
"layout and item hashes, such as cache-aware routers. Do not send "
"such a response to `/inference/v1/generate`, which reads a null "
"`kwargs_data` as every item being cached. Only supported on the "
"render endpoints; ignored on regular generation endpoints."
),
)
cache_salt: str | None = Field(
default=None,
min_length=1,
@@ -197,6 +197,18 @@ class CompletionRequest(OpenAIBaseModel):
),
)
return_mm_kwargs: bool = Field(
default=True,
description=(
"If false, the render response's `features` set `kwargs_data` "
"and `mm_metadata` to null, for callers that need only the token "
"layout and item hashes, such as cache-aware routers. Do not send "
"such a response to `/inference/v1/generate`, which reads a null "
"`kwargs_data` as every item being cached. Only supported on the "
"render endpoints; ignored on regular generation endpoints."
),
)
cache_salt: str | None = Field(
default=None,
min_length=1,
@@ -330,6 +330,17 @@ class ResponsesRequest(OpenAIBaseModel):
"if the served model does not use priority scheduling."
),
)
return_mm_kwargs: bool = Field(
default=True,
description=(
"If false, the render response's `features` set `kwargs_data` "
"and `mm_metadata` to null, for callers that need only the token "
"layout and item hashes, such as cache-aware routers. Do not send "
"such a response to `/inference/v1/generate`, which reads a null "
"`kwargs_data` as every item being cached. Only supported on the "
"render endpoints; ignored on regular generation endpoints."
),
)
cache_salt: str | None = Field(
default=None,
min_length=1,
+13 -3
View File
@@ -144,7 +144,9 @@ class ServingRender(BaseServing):
return GenerateRequest(
request_id=request_id,
token_ids=token_ids,
features=self._extract_mm_features(engine_input),
features=self._extract_mm_features(
engine_input, include_mm_kwargs=request.return_mm_kwargs
),
sampling_params=params,
model=request.model,
reasoning_ended=reasoning_ended,
@@ -169,6 +171,7 @@ class ServingRender(BaseServing):
chat_req = AnthropicServingMessages.to_chat_completion_request(
request, merge_inline_system=self._merge_inline_system
)
chat_req.return_mm_kwargs = request.return_mm_kwargs
return await self.render_chat_request(chat_req)
async def render_completion_request(
@@ -217,7 +220,9 @@ class ServingRender(BaseServing):
GenerateRequest(
request_id=request_id,
token_ids=token_ids,
features=self._extract_mm_features(engine_input),
features=self._extract_mm_features(
engine_input, include_mm_kwargs=request.return_mm_kwargs
),
sampling_params=params,
model=request.model,
stream=bool(request.stream),
@@ -277,7 +282,9 @@ class ServingRender(BaseServing):
return GenerateRequest(
request_id=request.request_id,
token_ids=list(token_ids),
features=self._extract_mm_features(engine_input),
features=self._extract_mm_features(
engine_input, include_mm_kwargs=request.return_mm_kwargs
),
sampling_params=params,
model=request.model,
reasoning_parser_kwargs=self._reasoning_parser_kwargs(request),
@@ -328,8 +335,11 @@ class ServingRender(BaseServing):
def _extract_mm_features(
self,
engine_input: EngineInput,
*,
include_mm_kwargs: bool = True,
) -> MultiModalFeatures | None:
return extract_mm_features(
engine_input,
metadata_fields_for=self._placeholder_metadata_fields,
include_mm_kwargs=include_mm_kwargs,
)
@@ -139,13 +139,15 @@ def extract_mm_features(
engine_input: EngineInput,
*,
metadata_fields_for: Callable[[str], Collection[str]] | None = None,
include_mm_kwargs: bool = True,
) -> MultiModalFeatures | None:
"""Extract multimodal features from a rendered engine prompt.
Returns ``None`` for text-only prompts. ``mm_metadata`` keeps the
intersection of processed kwargs that prefill needs after EC transfer:
fields declared as embedding metadata, plus fields marked
``keep_on_cpu`` (for example M-RoPE grid dims).
``keep_on_cpu`` (for example M-RoPE grid dims). With
``include_mm_kwargs=False`` neither is serialized.
"""
if engine_input.get("type") != "multimodal":
return None
@@ -157,11 +159,13 @@ def extract_mm_features(
kwargs_data: dict[str, list[str | None]] | None = None
mm_metadata: dict[str, list[str | None]] | None = None
if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"):
kwargs_data, mm_metadata = _encode_mm_kwargs_with_metadata(
raw_mm_kwargs,
metadata_fields_for=metadata_fields_for,
)
if include_mm_kwargs:
raw_mm_kwargs = mm_engine_input.get("mm_kwargs")
if raw_mm_kwargs:
kwargs_data, mm_metadata = _encode_mm_kwargs_with_metadata(
raw_mm_kwargs,
metadata_fields_for=metadata_fields_for,
)
return MultiModalFeatures(
mm_hashes=mm_hashes,