[Frontend][EPD] Use JSON arrays for multimodal metadata (#56090)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -22,7 +22,6 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
@@ -34,7 +33,6 @@ from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
import pybase64 as base64
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
@@ -125,19 +123,6 @@ def content_uuid(item: dict) -> str:
|
||||
return hashlib.sha256(payload.encode()).hexdigest()
|
||||
|
||||
|
||||
def _b64_tensor(values: list) -> str:
|
||||
import torch
|
||||
|
||||
buf = io.BytesIO()
|
||||
flat = [v for item in values for v in (item if isinstance(item, list) else [item])]
|
||||
# Floats stay float64 so timestamp strings format exactly as the
|
||||
# encoder computed them.
|
||||
dtype = torch.float64 if any(isinstance(v, float) for v in flat) else None
|
||||
# Downstream stacks per item, so hand over a flat vector.
|
||||
torch.save(torch.tensor(flat, dtype=dtype), buf)
|
||||
return base64.b64encode(buf.getvalue()).decode()
|
||||
|
||||
|
||||
def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
|
||||
"""Replace each media item with a metadata-only reference for the decoder.
|
||||
|
||||
@@ -172,7 +157,15 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
|
||||
# Whatever keys the encoder reported are the metadata its model
|
||||
# declared as needed to size the placeholder range; the proxy does
|
||||
# not need to know their names.
|
||||
metadata = {k: _b64_tensor(v) for k, v in meta.items()}
|
||||
# Downstream stacks per item; keep the existing per-item vector shape.
|
||||
metadata = {
|
||||
k: [
|
||||
x
|
||||
for item in v
|
||||
for x in (item if isinstance(item, list) else [item])
|
||||
]
|
||||
for k, v in meta.items()
|
||||
}
|
||||
if not metadata or not item_uuid:
|
||||
# Nothing to size the placeholder range with. A processor cache
|
||||
# hit is not a cause on its own: with the default `lru` type the
|
||||
|
||||
@@ -18,6 +18,8 @@ from vllm.entrypoints.chat_utils import (
|
||||
MEDIA_CONNECTOR_REGISTRY,
|
||||
AsyncMultiModalItemTracker,
|
||||
ConversationMessage,
|
||||
_load_embeds_dict,
|
||||
_parse_metadata_array,
|
||||
_postprocess_messages,
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
@@ -39,6 +41,45 @@ QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B"
|
||||
MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("values", [[1, 32, 48], [0.0, 0.12345678912345678]])
|
||||
def test_json_metadata_preserves_numeric_precision(values):
|
||||
tensor = _parse_metadata_array("grid", values, {"grid"})
|
||||
assert tensor.tolist() == values
|
||||
assert tensor.dtype == (
|
||||
torch.float64 if isinstance(values[0], float) else torch.int64
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key,values",
|
||||
[
|
||||
("image_embeds", [1, 2]),
|
||||
("grid", [True]),
|
||||
("grid", [float("nan")]),
|
||||
("grid", [float("inf")]),
|
||||
("grid", [[1, 2]]),
|
||||
("grid", [2**64]),
|
||||
],
|
||||
)
|
||||
def test_json_arrays_are_only_valid_numeric_metadata(key, values):
|
||||
with pytest.raises(VLLMValidationError):
|
||||
_parse_metadata_array(key, values, {"grid"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_json_metadata_bypasses_tensor_deserialization():
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
tensor = torch.ones(2, 3)
|
||||
fetch = AsyncMock(return_value=tensor)
|
||||
result = await _load_embeds_dict(
|
||||
{"image_embeds": "legacy-base64", "image_grid_thw": [1, 2, 2]}, fetch
|
||||
)
|
||||
fetch.assert_awaited_once_with("legacy-base64")
|
||||
assert result["image_embeds"] is tensor
|
||||
assert result["image_grid_thw"] == [1, 2, 2]
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def kimi_k2_5_model_config():
|
||||
return ModelConfig(
|
||||
@@ -1355,8 +1396,10 @@ def test_parse_chat_messages_empty_dict_image_embeds(
|
||||
_assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("json_metadata", [False, True])
|
||||
def test_parse_chat_messages_multiple_dict_image_embeds(
|
||||
qwen25omni_model_config_image_embeds,
|
||||
json_metadata,
|
||||
):
|
||||
"""Test that multiple dictionaries for image_embeds is handled without errors."""
|
||||
# Create two sample image embedding tensors
|
||||
@@ -1374,7 +1417,9 @@ def test_parse_chat_messages_multiple_dict_image_embeds(
|
||||
"type": "image_embeds",
|
||||
"image_embeds": {
|
||||
"image_embeds": tensor2base64(embeds),
|
||||
"image_grid_thw": tensor2base64(grid_thw),
|
||||
"image_grid_thw": grid_thw.tolist()
|
||||
if json_metadata
|
||||
else tensor2base64(grid_thw),
|
||||
},
|
||||
}
|
||||
for embeds, grid_thw in zip(
|
||||
|
||||
@@ -2,7 +2,31 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm.utils.collection_utils import common_prefix, swap_dict_values
|
||||
from vllm.utils.collection_utils import (
|
||||
common_prefix,
|
||||
is_list_of_numbers,
|
||||
swap_dict_values,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value,expected",
|
||||
[
|
||||
([], True),
|
||||
([1, -2, 2**1024], True),
|
||||
([1, 0.5], True),
|
||||
([1, True], False),
|
||||
([1, float("nan")], False),
|
||||
([1, float("inf")], False),
|
||||
([1, -float("inf")], False),
|
||||
([1, "2"], False),
|
||||
([[1]], False),
|
||||
((1, 2), False),
|
||||
(None, False),
|
||||
],
|
||||
)
|
||||
def test_is_list_of_numbers_checks_all_finite_non_boolean_items(value, expected):
|
||||
assert is_list_of_numbers(value) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -123,6 +123,9 @@ def test_decode_rewrite_preserves_engine_reported_ec_hash(proxy):
|
||||
)
|
||||
|
||||
assert rewritten["messages"][0]["content"][0]["uuid"] == "proxy-uuid"
|
||||
assert rewritten["messages"][0]["content"][0]["image_embeds"] == {
|
||||
"image_grid_thw": [1, 2, 3]
|
||||
}
|
||||
assert rewritten["ec_transfer_params"]["ec_items"] == [
|
||||
{"mm_hash": "engine-derived-hash", "transfer_id": "transfer"}
|
||||
]
|
||||
|
||||
@@ -73,7 +73,7 @@ from vllm.renderers.embed_utils import (
|
||||
)
|
||||
from vllm.transformers_utils.processor import get_video_processor_cls_name
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.utils.collection_utils import is_list_of, is_list_of_numbers
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -146,12 +146,15 @@ class ChatCompletionContentPartAudioParam(TypedDict, total=False):
|
||||
"""The type of the content part."""
|
||||
|
||||
|
||||
MultiModalEmbedsPayload: TypeAlias = str | dict[str, str | list[int | float]]
|
||||
|
||||
|
||||
class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False):
|
||||
image_embeds: str | dict[str, str] | None
|
||||
image_embeds: MultiModalEmbedsPayload | None
|
||||
"""
|
||||
The image embeddings. It can be either:
|
||||
- A single base64 string.
|
||||
- A dictionary where each value is a base64 string.
|
||||
- A dictionary of base64 tensors or numeric JSON metadata arrays.
|
||||
"""
|
||||
type: Required[Literal["image_embeds"]]
|
||||
"""The type of the content part."""
|
||||
@@ -163,11 +166,11 @@ class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False):
|
||||
|
||||
|
||||
class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False):
|
||||
audio_embeds: str | dict[str, str] | None
|
||||
audio_embeds: MultiModalEmbedsPayload | None
|
||||
"""
|
||||
The audio embeddings. It can be either:
|
||||
- A single base64 string representing a serialized torch tensor.
|
||||
- A dictionary where each value is a base64 string.
|
||||
- A dictionary of base64 tensors or numeric JSON metadata arrays.
|
||||
"""
|
||||
type: Required[Literal["audio_embeds"]]
|
||||
"""The type of the content part."""
|
||||
@@ -179,11 +182,11 @@ class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False):
|
||||
|
||||
|
||||
class ChatCompletionContentPartVideoEmbedsParam(TypedDict, total=False):
|
||||
video_embeds: str | dict[str, str] | None
|
||||
video_embeds: MultiModalEmbedsPayload | None
|
||||
"""
|
||||
The video embeddings. It can be either:
|
||||
- A single base64 string representing a serialized torch tensor.
|
||||
- A dictionary where each value is a base64 string.
|
||||
- A dictionary of base64 tensors or numeric JSON metadata arrays.
|
||||
"""
|
||||
type: Required[Literal["video_embeds"]]
|
||||
"""The type of the content part."""
|
||||
@@ -527,6 +530,27 @@ def _merge_embeds(
|
||||
return data_merged
|
||||
|
||||
|
||||
async def _load_embeds_dict(
|
||||
data: dict[str, str | list[int | float]],
|
||||
fetch: Callable[[str], Awaitable["torch.Tensor"]],
|
||||
) -> dict[str, Any]:
|
||||
encoded = {key: value for key, value in data.items() if isinstance(value, str)}
|
||||
tensors = await asyncio.gather(*(fetch(value) for value in encoded.values()))
|
||||
return {**data, **dict(zip(encoded, tensors))}
|
||||
|
||||
|
||||
def _parse_metadata_array(key: str, value: list, metadata_fields: set[str]):
|
||||
if key not in metadata_fields:
|
||||
raise VLLMValidationError(f"JSON arrays are only supported for metadata: {key}")
|
||||
if not is_list_of_numbers(value):
|
||||
raise VLLMValidationError(f"Metadata {key} must be a finite numeric array.")
|
||||
dtype = torch.float64 if any(isinstance(v, float) for v in value) else torch.int64
|
||||
try:
|
||||
return torch.tensor(value, dtype=dtype)
|
||||
except (ValueError, TypeError, OverflowError, RuntimeError) as error:
|
||||
raise VLLMValidationError(f"Invalid metadata array: {key}") from error
|
||||
|
||||
|
||||
def _get_embeds_data(
|
||||
modality: str,
|
||||
data_items: list[Any],
|
||||
@@ -544,6 +568,18 @@ def _get_embeds_data(
|
||||
return _merge_embeds(dict_items, mm_processor)[embeds_key]
|
||||
|
||||
if is_list_of(data_items, dict):
|
||||
metadata_fields = mm_processor.info.data_parser.placeholder_metadata_fields(
|
||||
modality
|
||||
)
|
||||
data_items = [
|
||||
{
|
||||
key: _parse_metadata_array(key, value, metadata_fields)
|
||||
if isinstance(value, list)
|
||||
else value
|
||||
for key, value in item.items()
|
||||
}
|
||||
for item in data_items
|
||||
]
|
||||
return _merge_embeds(data_items, mm_processor)
|
||||
|
||||
raise NotImplementedError(type(data_items))
|
||||
@@ -946,7 +982,7 @@ class BaseMultiModalContentParser(ABC):
|
||||
@abstractmethod
|
||||
def parse_image_embeds(
|
||||
self,
|
||||
image_embeds: str | dict[str, str] | None,
|
||||
image_embeds: MultiModalEmbedsPayload | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
@@ -970,7 +1006,7 @@ class BaseMultiModalContentParser(ABC):
|
||||
@abstractmethod
|
||||
def parse_audio_embeds(
|
||||
self,
|
||||
audio_embeds: str | dict[str, str] | None,
|
||||
audio_embeds: MultiModalEmbedsPayload | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
@@ -986,7 +1022,7 @@ class BaseMultiModalContentParser(ABC):
|
||||
@abstractmethod
|
||||
def parse_video_embeds(
|
||||
self,
|
||||
video_embeds: str | dict[str, str] | None,
|
||||
video_embeds: MultiModalEmbedsPayload | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
@@ -1043,7 +1079,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
def parse_image_embeds(
|
||||
self,
|
||||
image_embeds: str | dict[str, str] | None,
|
||||
image_embeds: MultiModalEmbedsPayload | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
mm_config = self.model_config.get_multimodal_config()
|
||||
@@ -1055,7 +1091,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
if isinstance(image_embeds, dict):
|
||||
embeds = {
|
||||
k: self._connector.fetch_image_embedding(v)
|
||||
k: self._connector.fetch_image_embedding(v) if isinstance(v, str) else v
|
||||
for k, v in image_embeds.items()
|
||||
}
|
||||
placeholder = self._tracker.add("image_embeds", (embeds, uuid))
|
||||
@@ -1071,7 +1107,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
def parse_audio_embeds(
|
||||
self,
|
||||
audio_embeds: str | dict[str, str] | None,
|
||||
audio_embeds: MultiModalEmbedsPayload | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
mm_config = self.model_config.get_multimodal_config()
|
||||
@@ -1083,7 +1119,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
if isinstance(audio_embeds, dict):
|
||||
embeds = {
|
||||
k: self._connector.fetch_audio_embedding(v)
|
||||
k: self._connector.fetch_audio_embedding(v) if isinstance(v, str) else v
|
||||
for k, v in audio_embeds.items()
|
||||
}
|
||||
placeholder = self._tracker.add("audio_embeds", (embeds, uuid))
|
||||
@@ -1148,7 +1184,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
def parse_video_embeds(
|
||||
self,
|
||||
video_embeds: str | dict[str, str] | None,
|
||||
video_embeds: MultiModalEmbedsPayload | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
mm_config = self.model_config.get_multimodal_config()
|
||||
@@ -1160,7 +1196,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
if isinstance(video_embeds, dict):
|
||||
embeds = {
|
||||
k: self._connector.fetch_video_embedding(v)
|
||||
k: self._connector.fetch_video_embedding(v) if isinstance(v, str) else v
|
||||
for k, v in video_embeds.items()
|
||||
}
|
||||
placeholder = self._tracker.add("video_embeds", (embeds, uuid))
|
||||
@@ -1242,7 +1278,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
def parse_image_embeds(
|
||||
self,
|
||||
image_embeds: str | dict[str, str] | None,
|
||||
image_embeds: MultiModalEmbedsPayload | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
mm_config = self.model_config.get_multimodal_config()
|
||||
@@ -1260,17 +1296,13 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
async def _image_embeds_with_uuid_async(
|
||||
self,
|
||||
image_embeds: str | dict[str, str] | None,
|
||||
image_embeds: MultiModalEmbedsPayload | None,
|
||||
uuid: str | None,
|
||||
):
|
||||
if isinstance(image_embeds, dict):
|
||||
tensors = await asyncio.gather(
|
||||
*(
|
||||
self._connector.fetch_image_embedding_async(v)
|
||||
for v in image_embeds.values()
|
||||
)
|
||||
embeds = await _load_embeds_dict(
|
||||
image_embeds, self._connector.fetch_image_embedding_async
|
||||
)
|
||||
embeds = dict(zip(image_embeds, tensors))
|
||||
elif isinstance(image_embeds, str):
|
||||
embeds = await self._connector.fetch_image_embedding_async(image_embeds)
|
||||
else:
|
||||
@@ -1279,7 +1311,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
def parse_audio_embeds(
|
||||
self,
|
||||
audio_embeds: str | dict[str, str] | None,
|
||||
audio_embeds: MultiModalEmbedsPayload | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
mm_config = self.model_config.get_multimodal_config()
|
||||
@@ -1297,17 +1329,13 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
async def _audio_embeds_with_uuid_async(
|
||||
self,
|
||||
audio_embeds: str | dict[str, str] | None,
|
||||
audio_embeds: MultiModalEmbedsPayload | None,
|
||||
uuid: str | None,
|
||||
):
|
||||
if isinstance(audio_embeds, dict):
|
||||
tensors = await asyncio.gather(
|
||||
*(
|
||||
self._connector.fetch_audio_embedding_async(v)
|
||||
for v in audio_embeds.values()
|
||||
)
|
||||
embeds = await _load_embeds_dict(
|
||||
audio_embeds, self._connector.fetch_audio_embedding_async
|
||||
)
|
||||
embeds = dict(zip(audio_embeds, tensors))
|
||||
elif isinstance(audio_embeds, str):
|
||||
embeds = await self._connector.fetch_audio_embedding_async(audio_embeds)
|
||||
else:
|
||||
@@ -1382,7 +1410,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
def parse_video_embeds(
|
||||
self,
|
||||
video_embeds: str | dict[str, str] | None,
|
||||
video_embeds: MultiModalEmbedsPayload | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
mm_config = self.model_config.get_multimodal_config()
|
||||
@@ -1400,17 +1428,13 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
async def _video_embeds_with_uuid_async(
|
||||
self,
|
||||
video_embeds: str | dict[str, str] | None,
|
||||
video_embeds: MultiModalEmbedsPayload | None,
|
||||
uuid: str | None,
|
||||
):
|
||||
if isinstance(video_embeds, dict):
|
||||
tensors = await asyncio.gather(
|
||||
*(
|
||||
self._connector.fetch_video_embedding_async(v)
|
||||
for v in video_embeds.values()
|
||||
)
|
||||
embeds = await _load_embeds_dict(
|
||||
video_embeds, self._connector.fetch_video_embedding_async
|
||||
)
|
||||
embeds = dict(zip(video_embeds, tensors))
|
||||
elif isinstance(video_embeds, str):
|
||||
embeds = await self._connector.fetch_video_embedding_async(video_embeds)
|
||||
else:
|
||||
@@ -1611,7 +1635,9 @@ _AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam).validate_python
|
||||
_VideoParser = TypeAdapter(ChatCompletionContentPartVideoParam).validate_python
|
||||
|
||||
_ResponsesInputImageParser = TypeAdapter(ResponseInputImageParam).validate_python
|
||||
_ContentPart: TypeAlias = str | dict[str, str] | InputAudio | PILImage
|
||||
_ContentPart: TypeAlias = (
|
||||
MultiModalEmbedsPayload | dict[str, str] | InputAudio | PILImage
|
||||
)
|
||||
|
||||
# Define a mapping from part types to their corresponding parsing functions.
|
||||
MM_PARSER_MAP: dict[
|
||||
@@ -1901,15 +1927,21 @@ def _parse_chat_message_content_part(
|
||||
mm_parser.parse_image(str_content, uuid)
|
||||
modality = "image"
|
||||
elif part_type == "image_embeds":
|
||||
content = cast(str | dict[str, str], content) if content is not None else None
|
||||
content = (
|
||||
cast(MultiModalEmbedsPayload, content) if content is not None else None
|
||||
)
|
||||
mm_parser.parse_image_embeds(content, uuid)
|
||||
modality = "image"
|
||||
elif part_type == "audio_embeds":
|
||||
content = cast(str | dict[str, str], content) if content is not None else None
|
||||
content = (
|
||||
cast(MultiModalEmbedsPayload, content) if content is not None else None
|
||||
)
|
||||
mm_parser.parse_audio_embeds(content, uuid)
|
||||
modality = "audio"
|
||||
elif part_type == "video_embeds":
|
||||
content = cast(str | dict[str, str], content) if content is not None else None
|
||||
content = (
|
||||
cast(MultiModalEmbedsPayload, content) if content is not None else None
|
||||
)
|
||||
mm_parser.parse_video_embeds(content, uuid)
|
||||
modality = "video"
|
||||
elif part_type == "prompt_embeds":
|
||||
|
||||
@@ -6,6 +6,7 @@ Contains helpers that are applied to collections.
|
||||
This is similar in concept to the `collections` module.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Generator, Hashable, Iterable, Mapping, Sequence
|
||||
from typing import Generic, Literal, TypeVar
|
||||
@@ -68,6 +69,13 @@ def is_list_of(
|
||||
assert_never(check)
|
||||
|
||||
|
||||
def is_list_of_numbers(value: object) -> bool:
|
||||
"""Check every item is an int or finite float, excluding booleans."""
|
||||
return isinstance(value, list) and all(
|
||||
type(v) is int or (type(v) is float and math.isfinite(v)) for v in value
|
||||
)
|
||||
|
||||
|
||||
@overload
|
||||
def common_prefix(items: Sequence[str]) -> str: ...
|
||||
|
||||
|
||||
Reference in New Issue
Block a user