[Frontend] Add return_mm_kwargs to render requests (#60728)
Signed-off-by: Maroon Ayoub <mayoub@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -144,7 +144,9 @@ class ServingRender(BaseServing):
|
||||
return GenerateRequest(
|
||||
request_id=request_id,
|
||||
token_ids=token_ids,
|
||||
features=self._extract_mm_features(engine_input),
|
||||
features=self._extract_mm_features(
|
||||
engine_input, include_mm_kwargs=request.return_mm_kwargs
|
||||
),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
reasoning_ended=reasoning_ended,
|
||||
@@ -169,6 +171,7 @@ class ServingRender(BaseServing):
|
||||
chat_req = AnthropicServingMessages.to_chat_completion_request(
|
||||
request, merge_inline_system=self._merge_inline_system
|
||||
)
|
||||
chat_req.return_mm_kwargs = request.return_mm_kwargs
|
||||
return await self.render_chat_request(chat_req)
|
||||
|
||||
async def render_completion_request(
|
||||
@@ -217,7 +220,9 @@ class ServingRender(BaseServing):
|
||||
GenerateRequest(
|
||||
request_id=request_id,
|
||||
token_ids=token_ids,
|
||||
features=self._extract_mm_features(engine_input),
|
||||
features=self._extract_mm_features(
|
||||
engine_input, include_mm_kwargs=request.return_mm_kwargs
|
||||
),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
stream=bool(request.stream),
|
||||
@@ -277,7 +282,9 @@ class ServingRender(BaseServing):
|
||||
return GenerateRequest(
|
||||
request_id=request.request_id,
|
||||
token_ids=list(token_ids),
|
||||
features=self._extract_mm_features(engine_input),
|
||||
features=self._extract_mm_features(
|
||||
engine_input, include_mm_kwargs=request.return_mm_kwargs
|
||||
),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
reasoning_parser_kwargs=self._reasoning_parser_kwargs(request),
|
||||
@@ -328,8 +335,11 @@ class ServingRender(BaseServing):
|
||||
def _extract_mm_features(
|
||||
self,
|
||||
engine_input: EngineInput,
|
||||
*,
|
||||
include_mm_kwargs: bool = True,
|
||||
) -> MultiModalFeatures | None:
|
||||
return extract_mm_features(
|
||||
engine_input,
|
||||
metadata_fields_for=self._placeholder_metadata_fields,
|
||||
include_mm_kwargs=include_mm_kwargs,
|
||||
)
|
||||
|
||||
@@ -139,13 +139,15 @@ def extract_mm_features(
|
||||
engine_input: EngineInput,
|
||||
*,
|
||||
metadata_fields_for: Callable[[str], Collection[str]] | None = None,
|
||||
include_mm_kwargs: bool = True,
|
||||
) -> MultiModalFeatures | None:
|
||||
"""Extract multimodal features from a rendered engine prompt.
|
||||
|
||||
Returns ``None`` for text-only prompts. ``mm_metadata`` keeps the
|
||||
intersection of processed kwargs that prefill needs after EC transfer:
|
||||
fields declared as embedding metadata, plus fields marked
|
||||
``keep_on_cpu`` (for example M-RoPE grid dims).
|
||||
``keep_on_cpu`` (for example M-RoPE grid dims). With
|
||||
``include_mm_kwargs=False`` neither is serialized.
|
||||
"""
|
||||
if engine_input.get("type") != "multimodal":
|
||||
return None
|
||||
@@ -157,11 +159,13 @@ def extract_mm_features(
|
||||
|
||||
kwargs_data: dict[str, list[str | None]] | None = None
|
||||
mm_metadata: dict[str, list[str | None]] | None = None
|
||||
if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"):
|
||||
kwargs_data, mm_metadata = _encode_mm_kwargs_with_metadata(
|
||||
raw_mm_kwargs,
|
||||
metadata_fields_for=metadata_fields_for,
|
||||
)
|
||||
if include_mm_kwargs:
|
||||
raw_mm_kwargs = mm_engine_input.get("mm_kwargs")
|
||||
if raw_mm_kwargs:
|
||||
kwargs_data, mm_metadata = _encode_mm_kwargs_with_metadata(
|
||||
raw_mm_kwargs,
|
||||
metadata_fields_for=metadata_fields_for,
|
||||
)
|
||||
|
||||
return MultiModalFeatures(
|
||||
mm_hashes=mm_hashes,
|
||||
|
||||
Reference in New Issue
Block a user