[Frontend] Add return_mm_kwargs to render requests (#60728)
Signed-off-by: Maroon Ayoub <mayoub@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -153,6 +153,13 @@ should split those fields:
|
|||||||
- Legacy clients that ignore `mm_metadata` and keep sending `kwargs_data`
|
- Legacy clients that ignore `mm_metadata` and keep sending `kwargs_data`
|
||||||
continue to work.
|
continue to work.
|
||||||
|
|
||||||
|
Callers that need only the token layout and item hashes, such as cache-aware
|
||||||
|
routers, can set `"return_mm_kwargs": false` on the render request. The response
|
||||||
|
then keeps `mm_hashes` and `mm_placeholders` and sets `kwargs_data` and
|
||||||
|
`mm_metadata` to null, so the processed tensors are neither serialized nor sent.
|
||||||
|
Do not forward such a response to `/inference/v1/generate`, which reads a null
|
||||||
|
`kwargs_data` as every item being cached.
|
||||||
|
|
||||||
## Example
|
## Example
|
||||||
|
|
||||||
The example below shows how a disaggregated encode / prefill coordinator can
|
The example below shows how a disaggregated encode / prefill coordinator can
|
||||||
|
|||||||
@@ -168,6 +168,20 @@ def test_render_extracts_metadata_fields_separately():
|
|||||||
assert torch.equal(full_data["pixel_values"].data, pixel_values.data)
|
assert torch.equal(full_data["pixel_values"].data, pixel_values.data)
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_without_kwargs_keeps_layout():
|
||||||
|
"""Layout-only callers get hashes and placeholders, no serialized tensors."""
|
||||||
|
engine_input, _pixel_values, _image_grid_thw = _image_engine_input()
|
||||||
|
|
||||||
|
full = extract_mm_features(engine_input)
|
||||||
|
layout = extract_mm_features(engine_input, include_mm_kwargs=False)
|
||||||
|
|
||||||
|
assert full is not None and layout is not None
|
||||||
|
assert layout.kwargs_data is None
|
||||||
|
assert layout.mm_metadata is None
|
||||||
|
assert layout.mm_hashes == full.mm_hashes
|
||||||
|
assert layout.mm_placeholders == full.mm_placeholders
|
||||||
|
|
||||||
|
|
||||||
def test_extract_includes_declared_placeholder_metadata_fields():
|
def test_extract_includes_declared_placeholder_metadata_fields():
|
||||||
"""EC placeholder metadata is kept even when keep_on_cpu is unset."""
|
"""EC placeholder metadata is kept even when keep_on_cpu is unset."""
|
||||||
engine_input, _pixel_values, image_grid_thw = _image_engine_input(
|
engine_input, _pixel_values, image_grid_thw = _image_engine_input(
|
||||||
|
|||||||
@@ -229,6 +229,17 @@ class AnthropicMessagesRequest(BaseModel):
|
|||||||
top_p: float | None = None
|
top_p: float | None = None
|
||||||
|
|
||||||
# vLLM-specific fields that are not in Anthropic spec
|
# vLLM-specific fields that are not in Anthropic spec
|
||||||
|
return_mm_kwargs: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description=(
|
||||||
|
"If false, the render response's `features` set `kwargs_data` "
|
||||||
|
"and `mm_metadata` to null, for callers that need only the token "
|
||||||
|
"layout and item hashes, such as cache-aware routers. Do not send "
|
||||||
|
"such a response to `/inference/v1/generate`, which reads a null "
|
||||||
|
"`kwargs_data` as every item being cached. Only supported on the "
|
||||||
|
"render endpoints; ignored on regular generation endpoints."
|
||||||
|
),
|
||||||
|
)
|
||||||
cache_salt: str | None = Field(
|
cache_salt: str | None = Field(
|
||||||
default=None,
|
default=None,
|
||||||
min_length=1,
|
min_length=1,
|
||||||
|
|||||||
@@ -458,6 +458,18 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
return_mm_kwargs: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description=(
|
||||||
|
"If false, the render response's `features` set `kwargs_data` "
|
||||||
|
"and `mm_metadata` to null, for callers that need only the token "
|
||||||
|
"layout and item hashes, such as cache-aware routers. Do not send "
|
||||||
|
"such a response to `/inference/v1/generate`, which reads a null "
|
||||||
|
"`kwargs_data` as every item being cached. Only supported on the "
|
||||||
|
"render endpoints; ignored on regular generation endpoints."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
cache_salt: str | None = Field(
|
cache_salt: str | None = Field(
|
||||||
default=None,
|
default=None,
|
||||||
min_length=1,
|
min_length=1,
|
||||||
|
|||||||
@@ -197,6 +197,18 @@ class CompletionRequest(OpenAIBaseModel):
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
return_mm_kwargs: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description=(
|
||||||
|
"If false, the render response's `features` set `kwargs_data` "
|
||||||
|
"and `mm_metadata` to null, for callers that need only the token "
|
||||||
|
"layout and item hashes, such as cache-aware routers. Do not send "
|
||||||
|
"such a response to `/inference/v1/generate`, which reads a null "
|
||||||
|
"`kwargs_data` as every item being cached. Only supported on the "
|
||||||
|
"render endpoints; ignored on regular generation endpoints."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
cache_salt: str | None = Field(
|
cache_salt: str | None = Field(
|
||||||
default=None,
|
default=None,
|
||||||
min_length=1,
|
min_length=1,
|
||||||
|
|||||||
@@ -330,6 +330,17 @@ class ResponsesRequest(OpenAIBaseModel):
|
|||||||
"if the served model does not use priority scheduling."
|
"if the served model does not use priority scheduling."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
return_mm_kwargs: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description=(
|
||||||
|
"If false, the render response's `features` set `kwargs_data` "
|
||||||
|
"and `mm_metadata` to null, for callers that need only the token "
|
||||||
|
"layout and item hashes, such as cache-aware routers. Do not send "
|
||||||
|
"such a response to `/inference/v1/generate`, which reads a null "
|
||||||
|
"`kwargs_data` as every item being cached. Only supported on the "
|
||||||
|
"render endpoints; ignored on regular generation endpoints."
|
||||||
|
),
|
||||||
|
)
|
||||||
cache_salt: str | None = Field(
|
cache_salt: str | None = Field(
|
||||||
default=None,
|
default=None,
|
||||||
min_length=1,
|
min_length=1,
|
||||||
|
|||||||
@@ -144,7 +144,9 @@ class ServingRender(BaseServing):
|
|||||||
return GenerateRequest(
|
return GenerateRequest(
|
||||||
request_id=request_id,
|
request_id=request_id,
|
||||||
token_ids=token_ids,
|
token_ids=token_ids,
|
||||||
features=self._extract_mm_features(engine_input),
|
features=self._extract_mm_features(
|
||||||
|
engine_input, include_mm_kwargs=request.return_mm_kwargs
|
||||||
|
),
|
||||||
sampling_params=params,
|
sampling_params=params,
|
||||||
model=request.model,
|
model=request.model,
|
||||||
reasoning_ended=reasoning_ended,
|
reasoning_ended=reasoning_ended,
|
||||||
@@ -169,6 +171,7 @@ class ServingRender(BaseServing):
|
|||||||
chat_req = AnthropicServingMessages.to_chat_completion_request(
|
chat_req = AnthropicServingMessages.to_chat_completion_request(
|
||||||
request, merge_inline_system=self._merge_inline_system
|
request, merge_inline_system=self._merge_inline_system
|
||||||
)
|
)
|
||||||
|
chat_req.return_mm_kwargs = request.return_mm_kwargs
|
||||||
return await self.render_chat_request(chat_req)
|
return await self.render_chat_request(chat_req)
|
||||||
|
|
||||||
async def render_completion_request(
|
async def render_completion_request(
|
||||||
@@ -217,7 +220,9 @@ class ServingRender(BaseServing):
|
|||||||
GenerateRequest(
|
GenerateRequest(
|
||||||
request_id=request_id,
|
request_id=request_id,
|
||||||
token_ids=token_ids,
|
token_ids=token_ids,
|
||||||
features=self._extract_mm_features(engine_input),
|
features=self._extract_mm_features(
|
||||||
|
engine_input, include_mm_kwargs=request.return_mm_kwargs
|
||||||
|
),
|
||||||
sampling_params=params,
|
sampling_params=params,
|
||||||
model=request.model,
|
model=request.model,
|
||||||
stream=bool(request.stream),
|
stream=bool(request.stream),
|
||||||
@@ -277,7 +282,9 @@ class ServingRender(BaseServing):
|
|||||||
return GenerateRequest(
|
return GenerateRequest(
|
||||||
request_id=request.request_id,
|
request_id=request.request_id,
|
||||||
token_ids=list(token_ids),
|
token_ids=list(token_ids),
|
||||||
features=self._extract_mm_features(engine_input),
|
features=self._extract_mm_features(
|
||||||
|
engine_input, include_mm_kwargs=request.return_mm_kwargs
|
||||||
|
),
|
||||||
sampling_params=params,
|
sampling_params=params,
|
||||||
model=request.model,
|
model=request.model,
|
||||||
reasoning_parser_kwargs=self._reasoning_parser_kwargs(request),
|
reasoning_parser_kwargs=self._reasoning_parser_kwargs(request),
|
||||||
@@ -328,8 +335,11 @@ class ServingRender(BaseServing):
|
|||||||
def _extract_mm_features(
|
def _extract_mm_features(
|
||||||
self,
|
self,
|
||||||
engine_input: EngineInput,
|
engine_input: EngineInput,
|
||||||
|
*,
|
||||||
|
include_mm_kwargs: bool = True,
|
||||||
) -> MultiModalFeatures | None:
|
) -> MultiModalFeatures | None:
|
||||||
return extract_mm_features(
|
return extract_mm_features(
|
||||||
engine_input,
|
engine_input,
|
||||||
metadata_fields_for=self._placeholder_metadata_fields,
|
metadata_fields_for=self._placeholder_metadata_fields,
|
||||||
|
include_mm_kwargs=include_mm_kwargs,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -139,13 +139,15 @@ def extract_mm_features(
|
|||||||
engine_input: EngineInput,
|
engine_input: EngineInput,
|
||||||
*,
|
*,
|
||||||
metadata_fields_for: Callable[[str], Collection[str]] | None = None,
|
metadata_fields_for: Callable[[str], Collection[str]] | None = None,
|
||||||
|
include_mm_kwargs: bool = True,
|
||||||
) -> MultiModalFeatures | None:
|
) -> MultiModalFeatures | None:
|
||||||
"""Extract multimodal features from a rendered engine prompt.
|
"""Extract multimodal features from a rendered engine prompt.
|
||||||
|
|
||||||
Returns ``None`` for text-only prompts. ``mm_metadata`` keeps the
|
Returns ``None`` for text-only prompts. ``mm_metadata`` keeps the
|
||||||
intersection of processed kwargs that prefill needs after EC transfer:
|
intersection of processed kwargs that prefill needs after EC transfer:
|
||||||
fields declared as embedding metadata, plus fields marked
|
fields declared as embedding metadata, plus fields marked
|
||||||
``keep_on_cpu`` (for example M-RoPE grid dims).
|
``keep_on_cpu`` (for example M-RoPE grid dims). With
|
||||||
|
``include_mm_kwargs=False`` neither is serialized.
|
||||||
"""
|
"""
|
||||||
if engine_input.get("type") != "multimodal":
|
if engine_input.get("type") != "multimodal":
|
||||||
return None
|
return None
|
||||||
@@ -157,11 +159,13 @@ def extract_mm_features(
|
|||||||
|
|
||||||
kwargs_data: dict[str, list[str | None]] | None = None
|
kwargs_data: dict[str, list[str | None]] | None = None
|
||||||
mm_metadata: dict[str, list[str | None]] | None = None
|
mm_metadata: dict[str, list[str | None]] | None = None
|
||||||
if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"):
|
if include_mm_kwargs:
|
||||||
kwargs_data, mm_metadata = _encode_mm_kwargs_with_metadata(
|
raw_mm_kwargs = mm_engine_input.get("mm_kwargs")
|
||||||
raw_mm_kwargs,
|
if raw_mm_kwargs:
|
||||||
metadata_fields_for=metadata_fields_for,
|
kwargs_data, mm_metadata = _encode_mm_kwargs_with_metadata(
|
||||||
)
|
raw_mm_kwargs,
|
||||||
|
metadata_fields_for=metadata_fields_for,
|
||||||
|
)
|
||||||
|
|
||||||
return MultiModalFeatures(
|
return MultiModalFeatures(
|
||||||
mm_hashes=mm_hashes,
|
mm_hashes=mm_hashes,
|
||||||
|
|||||||
Reference in New Issue
Block a user