[Frontend][EPD] Use JSON arrays for multimodal metadata (#56090)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-09-10 05:04:14 +00:00
committed by GitHub
parent 2a02f6efe3
commit 479611c0c6
6 changed files with 167 additions and 62 deletions
@@ -22,7 +22,6 @@ from __future__ import annotations
import argparse
import asyncio
import hashlib
import io
import itertools
import json
import logging
@@ -34,7 +33,6 @@ from collections.abc import AsyncIterator
from typing import Any
import aiohttp
import pybase64 as base64
import uvicorn
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse, StreamingResponse
@@ -125,19 +123,6 @@ def content_uuid(item: dict) -> str:
return hashlib.sha256(payload.encode()).hexdigest()
def _b64_tensor(values: list) -> str:
import torch
buf = io.BytesIO()
flat = [v for item in values for v in (item if isinstance(item, list) else [item])]
# Floats stay float64 so timestamp strings format exactly as the
# encoder computed them.
dtype = torch.float64 if any(isinstance(v, float) for v in flat) else None
# Downstream stacks per item, so hand over a flat vector.
torch.save(torch.tensor(flat, dtype=dtype), buf)
return base64.b64encode(buf.getvalue()).decode()
def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
"""Replace each media item with a metadata-only reference for the decoder.
@@ -172,7 +157,15 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
# Whatever keys the encoder reported are the metadata its model
# declared as needed to size the placeholder range; the proxy does
# not need to know their names.
metadata = {k: _b64_tensor(v) for k, v in meta.items()}
# Downstream stacks per item; keep the existing per-item vector shape.
metadata = {
k: [
x
for item in v
for x in (item if isinstance(item, list) else [item])
]
for k, v in meta.items()
}
if not metadata or not item_uuid:
# Nothing to size the placeholder range with. A processor cache
# hit is not a cause on its own: with the default `lru` type the
@@ -18,6 +18,8 @@ from vllm.entrypoints.chat_utils import (
MEDIA_CONNECTOR_REGISTRY,
AsyncMultiModalItemTracker,
ConversationMessage,
_load_embeds_dict,
_parse_metadata_array,
_postprocess_messages,
parse_chat_messages,
parse_chat_messages_async,
@@ -39,6 +41,45 @@ QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B"
MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
@pytest.mark.parametrize("values", [[1, 32, 48], [0.0, 0.12345678912345678]])
def test_json_metadata_preserves_numeric_precision(values):
tensor = _parse_metadata_array("grid", values, {"grid"})
assert tensor.tolist() == values
assert tensor.dtype == (
torch.float64 if isinstance(values[0], float) else torch.int64
)
@pytest.mark.parametrize(
"key,values",
[
("image_embeds", [1, 2]),
("grid", [True]),
("grid", [float("nan")]),
("grid", [float("inf")]),
("grid", [[1, 2]]),
("grid", [2**64]),
],
)
def test_json_arrays_are_only_valid_numeric_metadata(key, values):
with pytest.raises(VLLMValidationError):
_parse_metadata_array(key, values, {"grid"})
@pytest.mark.asyncio
async def test_json_metadata_bypasses_tensor_deserialization():
from unittest.mock import AsyncMock
tensor = torch.ones(2, 3)
fetch = AsyncMock(return_value=tensor)
result = await _load_embeds_dict(
{"image_embeds": "legacy-base64", "image_grid_thw": [1, 2, 2]}, fetch
)
fetch.assert_awaited_once_with("legacy-base64")
assert result["image_embeds"] is tensor
assert result["image_grid_thw"] == [1, 2, 2]
@pytest.fixture(scope="function")
def kimi_k2_5_model_config():
return ModelConfig(
@@ -1355,8 +1396,10 @@ def test_parse_chat_messages_empty_dict_image_embeds(
_assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])
@pytest.mark.parametrize("json_metadata", [False, True])
def test_parse_chat_messages_multiple_dict_image_embeds(
qwen25omni_model_config_image_embeds,
json_metadata,
):
"""Test that multiple dictionaries for image_embeds is handled without errors."""
# Create two sample image embedding tensors
@@ -1374,7 +1417,9 @@ def test_parse_chat_messages_multiple_dict_image_embeds(
"type": "image_embeds",
"image_embeds": {
"image_embeds": tensor2base64(embeds),
"image_grid_thw": tensor2base64(grid_thw),
"image_grid_thw": grid_thw.tolist()
if json_metadata
else tensor2base64(grid_thw),
},
}
for embeds, grid_thw in zip(
+25 -1
View File
@@ -2,7 +2,31 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.utils.collection_utils import common_prefix, swap_dict_values
from vllm.utils.collection_utils import (
common_prefix,
is_list_of_numbers,
swap_dict_values,
)
@pytest.mark.parametrize(
"value,expected",
[
([], True),
([1, -2, 2**1024], True),
([1, 0.5], True),
([1, True], False),
([1, float("nan")], False),
([1, float("inf")], False),
([1, -float("inf")], False),
([1, "2"], False),
([[1]], False),
((1, 2), False),
(None, False),
],
)
def test_is_list_of_numbers_checks_all_finite_non_boolean_items(value, expected):
assert is_list_of_numbers(value) is expected
@pytest.mark.parametrize(
@@ -123,6 +123,9 @@ def test_decode_rewrite_preserves_engine_reported_ec_hash(proxy):
)
assert rewritten["messages"][0]["content"][0]["uuid"] == "proxy-uuid"
assert rewritten["messages"][0]["content"][0]["image_embeds"] == {
"image_grid_thw": [1, 2, 3]
}
assert rewritten["ec_transfer_params"]["ec_items"] == [
{"mm_hash": "engine-derived-hash", "transfer_id": "transfer"}
]
+76 -44
View File
@@ -73,7 +73,7 @@ from vllm.renderers.embed_utils import (
)
from vllm.transformers_utils.processor import get_video_processor_cls_name
from vllm.utils import random_uuid
from vllm.utils.collection_utils import is_list_of
from vllm.utils.collection_utils import is_list_of, is_list_of_numbers
from vllm.utils.import_utils import LazyLoader
if TYPE_CHECKING:
@@ -146,12 +146,15 @@ class ChatCompletionContentPartAudioParam(TypedDict, total=False):
"""The type of the content part."""
MultiModalEmbedsPayload: TypeAlias = str | dict[str, str | list[int | float]]
class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False):
image_embeds: str | dict[str, str] | None
image_embeds: MultiModalEmbedsPayload | None
"""
The image embeddings. It can be either:
- A single base64 string.
- A dictionary where each value is a base64 string.
- A dictionary of base64 tensors or numeric JSON metadata arrays.
"""
type: Required[Literal["image_embeds"]]
"""The type of the content part."""
@@ -163,11 +166,11 @@ class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False):
class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False):
audio_embeds: str | dict[str, str] | None
audio_embeds: MultiModalEmbedsPayload | None
"""
The audio embeddings. It can be either:
- A single base64 string representing a serialized torch tensor.
- A dictionary where each value is a base64 string.
- A dictionary of base64 tensors or numeric JSON metadata arrays.
"""
type: Required[Literal["audio_embeds"]]
"""The type of the content part."""
@@ -179,11 +182,11 @@ class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False):
class ChatCompletionContentPartVideoEmbedsParam(TypedDict, total=False):
video_embeds: str | dict[str, str] | None
video_embeds: MultiModalEmbedsPayload | None
"""
The video embeddings. It can be either:
- A single base64 string representing a serialized torch tensor.
- A dictionary where each value is a base64 string.
- A dictionary of base64 tensors or numeric JSON metadata arrays.
"""
type: Required[Literal["video_embeds"]]
"""The type of the content part."""
@@ -527,6 +530,27 @@ def _merge_embeds(
return data_merged
async def _load_embeds_dict(
data: dict[str, str | list[int | float]],
fetch: Callable[[str], Awaitable["torch.Tensor"]],
) -> dict[str, Any]:
encoded = {key: value for key, value in data.items() if isinstance(value, str)}
tensors = await asyncio.gather(*(fetch(value) for value in encoded.values()))
return {**data, **dict(zip(encoded, tensors))}
def _parse_metadata_array(key: str, value: list, metadata_fields: set[str]):
if key not in metadata_fields:
raise VLLMValidationError(f"JSON arrays are only supported for metadata: {key}")
if not is_list_of_numbers(value):
raise VLLMValidationError(f"Metadata {key} must be a finite numeric array.")
dtype = torch.float64 if any(isinstance(v, float) for v in value) else torch.int64
try:
return torch.tensor(value, dtype=dtype)
except (ValueError, TypeError, OverflowError, RuntimeError) as error:
raise VLLMValidationError(f"Invalid metadata array: {key}") from error
def _get_embeds_data(
modality: str,
data_items: list[Any],
@@ -544,6 +568,18 @@ def _get_embeds_data(
return _merge_embeds(dict_items, mm_processor)[embeds_key]
if is_list_of(data_items, dict):
metadata_fields = mm_processor.info.data_parser.placeholder_metadata_fields(
modality
)
data_items = [
{
key: _parse_metadata_array(key, value, metadata_fields)
if isinstance(value, list)
else value
for key, value in item.items()
}
for item in data_items
]
return _merge_embeds(data_items, mm_processor)
raise NotImplementedError(type(data_items))
@@ -946,7 +982,7 @@ class BaseMultiModalContentParser(ABC):
@abstractmethod
def parse_image_embeds(
self,
image_embeds: str | dict[str, str] | None,
image_embeds: MultiModalEmbedsPayload | None,
uuid: str | None = None,
) -> None:
raise NotImplementedError
@@ -970,7 +1006,7 @@ class BaseMultiModalContentParser(ABC):
@abstractmethod
def parse_audio_embeds(
self,
audio_embeds: str | dict[str, str] | None,
audio_embeds: MultiModalEmbedsPayload | None,
uuid: str | None = None,
) -> None:
raise NotImplementedError
@@ -986,7 +1022,7 @@ class BaseMultiModalContentParser(ABC):
@abstractmethod
def parse_video_embeds(
self,
video_embeds: str | dict[str, str] | None,
video_embeds: MultiModalEmbedsPayload | None,
uuid: str | None = None,
) -> None:
raise NotImplementedError
@@ -1043,7 +1079,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
def parse_image_embeds(
self,
image_embeds: str | dict[str, str] | None,
image_embeds: MultiModalEmbedsPayload | None,
uuid: str | None = None,
) -> None:
mm_config = self.model_config.get_multimodal_config()
@@ -1055,7 +1091,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
if isinstance(image_embeds, dict):
embeds = {
k: self._connector.fetch_image_embedding(v)
k: self._connector.fetch_image_embedding(v) if isinstance(v, str) else v
for k, v in image_embeds.items()
}
placeholder = self._tracker.add("image_embeds", (embeds, uuid))
@@ -1071,7 +1107,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
def parse_audio_embeds(
self,
audio_embeds: str | dict[str, str] | None,
audio_embeds: MultiModalEmbedsPayload | None,
uuid: str | None = None,
) -> None:
mm_config = self.model_config.get_multimodal_config()
@@ -1083,7 +1119,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
if isinstance(audio_embeds, dict):
embeds = {
k: self._connector.fetch_audio_embedding(v)
k: self._connector.fetch_audio_embedding(v) if isinstance(v, str) else v
for k, v in audio_embeds.items()
}
placeholder = self._tracker.add("audio_embeds", (embeds, uuid))
@@ -1148,7 +1184,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
def parse_video_embeds(
self,
video_embeds: str | dict[str, str] | None,
video_embeds: MultiModalEmbedsPayload | None,
uuid: str | None = None,
) -> None:
mm_config = self.model_config.get_multimodal_config()
@@ -1160,7 +1196,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
if isinstance(video_embeds, dict):
embeds = {
k: self._connector.fetch_video_embedding(v)
k: self._connector.fetch_video_embedding(v) if isinstance(v, str) else v
for k, v in video_embeds.items()
}
placeholder = self._tracker.add("video_embeds", (embeds, uuid))
@@ -1242,7 +1278,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
def parse_image_embeds(
self,
image_embeds: str | dict[str, str] | None,
image_embeds: MultiModalEmbedsPayload | None,
uuid: str | None = None,
) -> None:
mm_config = self.model_config.get_multimodal_config()
@@ -1260,17 +1296,13 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
async def _image_embeds_with_uuid_async(
self,
image_embeds: str | dict[str, str] | None,
image_embeds: MultiModalEmbedsPayload | None,
uuid: str | None,
):
if isinstance(image_embeds, dict):
tensors = await asyncio.gather(
*(
self._connector.fetch_image_embedding_async(v)
for v in image_embeds.values()
)
embeds = await _load_embeds_dict(
image_embeds, self._connector.fetch_image_embedding_async
)
embeds = dict(zip(image_embeds, tensors))
elif isinstance(image_embeds, str):
embeds = await self._connector.fetch_image_embedding_async(image_embeds)
else:
@@ -1279,7 +1311,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
def parse_audio_embeds(
self,
audio_embeds: str | dict[str, str] | None,
audio_embeds: MultiModalEmbedsPayload | None,
uuid: str | None = None,
) -> None:
mm_config = self.model_config.get_multimodal_config()
@@ -1297,17 +1329,13 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
async def _audio_embeds_with_uuid_async(
self,
audio_embeds: str | dict[str, str] | None,
audio_embeds: MultiModalEmbedsPayload | None,
uuid: str | None,
):
if isinstance(audio_embeds, dict):
tensors = await asyncio.gather(
*(
self._connector.fetch_audio_embedding_async(v)
for v in audio_embeds.values()
)
embeds = await _load_embeds_dict(
audio_embeds, self._connector.fetch_audio_embedding_async
)
embeds = dict(zip(audio_embeds, tensors))
elif isinstance(audio_embeds, str):
embeds = await self._connector.fetch_audio_embedding_async(audio_embeds)
else:
@@ -1382,7 +1410,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
def parse_video_embeds(
self,
video_embeds: str | dict[str, str] | None,
video_embeds: MultiModalEmbedsPayload | None,
uuid: str | None = None,
) -> None:
mm_config = self.model_config.get_multimodal_config()
@@ -1400,17 +1428,13 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
async def _video_embeds_with_uuid_async(
self,
video_embeds: str | dict[str, str] | None,
video_embeds: MultiModalEmbedsPayload | None,
uuid: str | None,
):
if isinstance(video_embeds, dict):
tensors = await asyncio.gather(
*(
self._connector.fetch_video_embedding_async(v)
for v in video_embeds.values()
)
embeds = await _load_embeds_dict(
video_embeds, self._connector.fetch_video_embedding_async
)
embeds = dict(zip(video_embeds, tensors))
elif isinstance(video_embeds, str):
embeds = await self._connector.fetch_video_embedding_async(video_embeds)
else:
@@ -1611,7 +1635,9 @@ _AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam).validate_python
_VideoParser = TypeAdapter(ChatCompletionContentPartVideoParam).validate_python
_ResponsesInputImageParser = TypeAdapter(ResponseInputImageParam).validate_python
_ContentPart: TypeAlias = str | dict[str, str] | InputAudio | PILImage
_ContentPart: TypeAlias = (
MultiModalEmbedsPayload | dict[str, str] | InputAudio | PILImage
)
# Define a mapping from part types to their corresponding parsing functions.
MM_PARSER_MAP: dict[
@@ -1901,15 +1927,21 @@ def _parse_chat_message_content_part(
mm_parser.parse_image(str_content, uuid)
modality = "image"
elif part_type == "image_embeds":
content = cast(str | dict[str, str], content) if content is not None else None
content = (
cast(MultiModalEmbedsPayload, content) if content is not None else None
)
mm_parser.parse_image_embeds(content, uuid)
modality = "image"
elif part_type == "audio_embeds":
content = cast(str | dict[str, str], content) if content is not None else None
content = (
cast(MultiModalEmbedsPayload, content) if content is not None else None
)
mm_parser.parse_audio_embeds(content, uuid)
modality = "audio"
elif part_type == "video_embeds":
content = cast(str | dict[str, str], content) if content is not None else None
content = (
cast(MultiModalEmbedsPayload, content) if content is not None else None
)
mm_parser.parse_video_embeds(content, uuid)
modality = "video"
elif part_type == "prompt_embeds":
+8
View File
@@ -6,6 +6,7 @@ Contains helpers that are applied to collections.
This is similar in concept to the `collections` module.
"""
import math
from collections import defaultdict
from collections.abc import Callable, Generator, Hashable, Iterable, Mapping, Sequence
from typing import Generic, Literal, TypeVar
@@ -68,6 +69,13 @@ def is_list_of(
assert_never(check)
def is_list_of_numbers(value: object) -> bool:
"""Check every item is an int or finite float, excluding booleans."""
return isinstance(value, list) and all(
type(v) is int or (type(v) is float and math.isfinite(v)) for v in value
)
@overload
def common_prefix(items: Sequence[str]) -> str: ...