From 479611c0c6118f95d6028a337e69b4713c529843 Mon Sep 17 00:00:00 2001 From: Tianyu Guo Date: Thu, 10 Sep 2026 13:04:14 +0800 Subject: [PATCH] [Frontend][EPD] Use JSON arrays for multimodal metadata (#56090) Signed-off-by: Tianyu Guo --- .../disaggregated_encoder/disagg_epd_proxy.py | 25 ++-- .../entrypoints/unit_tests/test_chat_utils.py | 47 ++++++- tests/utils_/test_collection_utils.py | 26 +++- .../unit/test_epd_proxy_round_robin.py | 3 + vllm/entrypoints/chat_utils.py | 120 +++++++++++------- vllm/utils/collection_utils.py | 8 ++ 6 files changed, 167 insertions(+), 62 deletions(-) diff --git a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py index 1581e21748..ace4455828 100644 --- a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py +++ b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py @@ -22,7 +22,6 @@ from __future__ import annotations import argparse import asyncio import hashlib -import io import itertools import json import logging @@ -34,7 +33,6 @@ from collections.abc import AsyncIterator from typing import Any import aiohttp -import pybase64 as base64 import uvicorn from fastapi import FastAPI, HTTPException, Request from fastapi.responses import JSONResponse, StreamingResponse @@ -125,19 +123,6 @@ def content_uuid(item: dict) -> str: return hashlib.sha256(payload.encode()).hexdigest() -def _b64_tensor(values: list) -> str: - import torch - - buf = io.BytesIO() - flat = [v for item in values for v in (item if isinstance(item, list) else [item])] - # Floats stay float64 so timestamp strings format exactly as the - # encoder computed them. - dtype = torch.float64 if any(isinstance(v, float) for v in flat) else None - # Downstream stacks per item, so hand over a flat vector. - torch.save(torch.tensor(flat, dtype=dtype), buf) - return base64.b64encode(buf.getvalue()).decode() - - def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict: """Replace each media item with a metadata-only reference for the decoder. @@ -172,7 +157,15 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict: # Whatever keys the encoder reported are the metadata its model # declared as needed to size the placeholder range; the proxy does # not need to know their names. - metadata = {k: _b64_tensor(v) for k, v in meta.items()} + # Downstream stacks per item; keep the existing per-item vector shape. + metadata = { + k: [ + x + for item in v + for x in (item if isinstance(item, list) else [item]) + ] + for k, v in meta.items() + } if not metadata or not item_uuid: # Nothing to size the placeholder range with. A processor cache # hit is not a cause on its own: with the default `lru` type the diff --git a/tests/entrypoints/unit_tests/test_chat_utils.py b/tests/entrypoints/unit_tests/test_chat_utils.py index 8570a1b703..575bb904d6 100644 --- a/tests/entrypoints/unit_tests/test_chat_utils.py +++ b/tests/entrypoints/unit_tests/test_chat_utils.py @@ -18,6 +18,8 @@ from vllm.entrypoints.chat_utils import ( MEDIA_CONNECTOR_REGISTRY, AsyncMultiModalItemTracker, ConversationMessage, + _load_embeds_dict, + _parse_metadata_array, _postprocess_messages, parse_chat_messages, parse_chat_messages_async, @@ -39,6 +41,45 @@ QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B" MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503" +@pytest.mark.parametrize("values", [[1, 32, 48], [0.0, 0.12345678912345678]]) +def test_json_metadata_preserves_numeric_precision(values): + tensor = _parse_metadata_array("grid", values, {"grid"}) + assert tensor.tolist() == values + assert tensor.dtype == ( + torch.float64 if isinstance(values[0], float) else torch.int64 + ) + + +@pytest.mark.parametrize( + "key,values", + [ + ("image_embeds", [1, 2]), + ("grid", [True]), + ("grid", [float("nan")]), + ("grid", [float("inf")]), + ("grid", [[1, 2]]), + ("grid", [2**64]), + ], +) +def test_json_arrays_are_only_valid_numeric_metadata(key, values): + with pytest.raises(VLLMValidationError): + _parse_metadata_array(key, values, {"grid"}) + + +@pytest.mark.asyncio +async def test_json_metadata_bypasses_tensor_deserialization(): + from unittest.mock import AsyncMock + + tensor = torch.ones(2, 3) + fetch = AsyncMock(return_value=tensor) + result = await _load_embeds_dict( + {"image_embeds": "legacy-base64", "image_grid_thw": [1, 2, 2]}, fetch + ) + fetch.assert_awaited_once_with("legacy-base64") + assert result["image_embeds"] is tensor + assert result["image_grid_thw"] == [1, 2, 2] + + @pytest.fixture(scope="function") def kimi_k2_5_model_config(): return ModelConfig( @@ -1355,8 +1396,10 @@ def test_parse_chat_messages_empty_dict_image_embeds( _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None]) +@pytest.mark.parametrize("json_metadata", [False, True]) def test_parse_chat_messages_multiple_dict_image_embeds( qwen25omni_model_config_image_embeds, + json_metadata, ): """Test that multiple dictionaries for image_embeds is handled without errors.""" # Create two sample image embedding tensors @@ -1374,7 +1417,9 @@ def test_parse_chat_messages_multiple_dict_image_embeds( "type": "image_embeds", "image_embeds": { "image_embeds": tensor2base64(embeds), - "image_grid_thw": tensor2base64(grid_thw), + "image_grid_thw": grid_thw.tolist() + if json_metadata + else tensor2base64(grid_thw), }, } for embeds, grid_thw in zip( diff --git a/tests/utils_/test_collection_utils.py b/tests/utils_/test_collection_utils.py index 5789412602..0cc43d2f13 100644 --- a/tests/utils_/test_collection_utils.py +++ b/tests/utils_/test_collection_utils.py @@ -2,7 +2,31 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest -from vllm.utils.collection_utils import common_prefix, swap_dict_values +from vllm.utils.collection_utils import ( + common_prefix, + is_list_of_numbers, + swap_dict_values, +) + + +@pytest.mark.parametrize( + "value,expected", + [ + ([], True), + ([1, -2, 2**1024], True), + ([1, 0.5], True), + ([1, True], False), + ([1, float("nan")], False), + ([1, float("inf")], False), + ([1, -float("inf")], False), + ([1, "2"], False), + ([[1]], False), + ((1, 2), False), + (None, False), + ], +) +def test_is_list_of_numbers_checks_all_finite_non_boolean_items(value, expected): + assert is_list_of_numbers(value) is expected @pytest.mark.parametrize( diff --git a/tests/v1/ec_connector/unit/test_epd_proxy_round_robin.py b/tests/v1/ec_connector/unit/test_epd_proxy_round_robin.py index c7278ad55b..558a80f9cf 100644 --- a/tests/v1/ec_connector/unit/test_epd_proxy_round_robin.py +++ b/tests/v1/ec_connector/unit/test_epd_proxy_round_robin.py @@ -123,6 +123,9 @@ def test_decode_rewrite_preserves_engine_reported_ec_hash(proxy): ) assert rewritten["messages"][0]["content"][0]["uuid"] == "proxy-uuid" + assert rewritten["messages"][0]["content"][0]["image_embeds"] == { + "image_grid_thw": [1, 2, 3] + } assert rewritten["ec_transfer_params"]["ec_items"] == [ {"mm_hash": "engine-derived-hash", "transfer_id": "transfer"} ] diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 4410b5981b..327f9c4b24 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -73,7 +73,7 @@ from vllm.renderers.embed_utils import ( ) from vllm.transformers_utils.processor import get_video_processor_cls_name from vllm.utils import random_uuid -from vllm.utils.collection_utils import is_list_of +from vllm.utils.collection_utils import is_list_of, is_list_of_numbers from vllm.utils.import_utils import LazyLoader if TYPE_CHECKING: @@ -146,12 +146,15 @@ class ChatCompletionContentPartAudioParam(TypedDict, total=False): """The type of the content part.""" +MultiModalEmbedsPayload: TypeAlias = str | dict[str, str | list[int | float]] + + class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False): - image_embeds: str | dict[str, str] | None + image_embeds: MultiModalEmbedsPayload | None """ The image embeddings. It can be either: - A single base64 string. - - A dictionary where each value is a base64 string. + - A dictionary of base64 tensors or numeric JSON metadata arrays. """ type: Required[Literal["image_embeds"]] """The type of the content part.""" @@ -163,11 +166,11 @@ class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False): class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False): - audio_embeds: str | dict[str, str] | None + audio_embeds: MultiModalEmbedsPayload | None """ The audio embeddings. It can be either: - A single base64 string representing a serialized torch tensor. - - A dictionary where each value is a base64 string. + - A dictionary of base64 tensors or numeric JSON metadata arrays. """ type: Required[Literal["audio_embeds"]] """The type of the content part.""" @@ -179,11 +182,11 @@ class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False): class ChatCompletionContentPartVideoEmbedsParam(TypedDict, total=False): - video_embeds: str | dict[str, str] | None + video_embeds: MultiModalEmbedsPayload | None """ The video embeddings. It can be either: - A single base64 string representing a serialized torch tensor. - - A dictionary where each value is a base64 string. + - A dictionary of base64 tensors or numeric JSON metadata arrays. """ type: Required[Literal["video_embeds"]] """The type of the content part.""" @@ -527,6 +530,27 @@ def _merge_embeds( return data_merged +async def _load_embeds_dict( + data: dict[str, str | list[int | float]], + fetch: Callable[[str], Awaitable["torch.Tensor"]], +) -> dict[str, Any]: + encoded = {key: value for key, value in data.items() if isinstance(value, str)} + tensors = await asyncio.gather(*(fetch(value) for value in encoded.values())) + return {**data, **dict(zip(encoded, tensors))} + + +def _parse_metadata_array(key: str, value: list, metadata_fields: set[str]): + if key not in metadata_fields: + raise VLLMValidationError(f"JSON arrays are only supported for metadata: {key}") + if not is_list_of_numbers(value): + raise VLLMValidationError(f"Metadata {key} must be a finite numeric array.") + dtype = torch.float64 if any(isinstance(v, float) for v in value) else torch.int64 + try: + return torch.tensor(value, dtype=dtype) + except (ValueError, TypeError, OverflowError, RuntimeError) as error: + raise VLLMValidationError(f"Invalid metadata array: {key}") from error + + def _get_embeds_data( modality: str, data_items: list[Any], @@ -544,6 +568,18 @@ def _get_embeds_data( return _merge_embeds(dict_items, mm_processor)[embeds_key] if is_list_of(data_items, dict): + metadata_fields = mm_processor.info.data_parser.placeholder_metadata_fields( + modality + ) + data_items = [ + { + key: _parse_metadata_array(key, value, metadata_fields) + if isinstance(value, list) + else value + for key, value in item.items() + } + for item in data_items + ] return _merge_embeds(data_items, mm_processor) raise NotImplementedError(type(data_items)) @@ -946,7 +982,7 @@ class BaseMultiModalContentParser(ABC): @abstractmethod def parse_image_embeds( self, - image_embeds: str | dict[str, str] | None, + image_embeds: MultiModalEmbedsPayload | None, uuid: str | None = None, ) -> None: raise NotImplementedError @@ -970,7 +1006,7 @@ class BaseMultiModalContentParser(ABC): @abstractmethod def parse_audio_embeds( self, - audio_embeds: str | dict[str, str] | None, + audio_embeds: MultiModalEmbedsPayload | None, uuid: str | None = None, ) -> None: raise NotImplementedError @@ -986,7 +1022,7 @@ class BaseMultiModalContentParser(ABC): @abstractmethod def parse_video_embeds( self, - video_embeds: str | dict[str, str] | None, + video_embeds: MultiModalEmbedsPayload | None, uuid: str | None = None, ) -> None: raise NotImplementedError @@ -1043,7 +1079,7 @@ class MultiModalContentParser(BaseMultiModalContentParser): def parse_image_embeds( self, - image_embeds: str | dict[str, str] | None, + image_embeds: MultiModalEmbedsPayload | None, uuid: str | None = None, ) -> None: mm_config = self.model_config.get_multimodal_config() @@ -1055,7 +1091,7 @@ class MultiModalContentParser(BaseMultiModalContentParser): if isinstance(image_embeds, dict): embeds = { - k: self._connector.fetch_image_embedding(v) + k: self._connector.fetch_image_embedding(v) if isinstance(v, str) else v for k, v in image_embeds.items() } placeholder = self._tracker.add("image_embeds", (embeds, uuid)) @@ -1071,7 +1107,7 @@ class MultiModalContentParser(BaseMultiModalContentParser): def parse_audio_embeds( self, - audio_embeds: str | dict[str, str] | None, + audio_embeds: MultiModalEmbedsPayload | None, uuid: str | None = None, ) -> None: mm_config = self.model_config.get_multimodal_config() @@ -1083,7 +1119,7 @@ class MultiModalContentParser(BaseMultiModalContentParser): if isinstance(audio_embeds, dict): embeds = { - k: self._connector.fetch_audio_embedding(v) + k: self._connector.fetch_audio_embedding(v) if isinstance(v, str) else v for k, v in audio_embeds.items() } placeholder = self._tracker.add("audio_embeds", (embeds, uuid)) @@ -1148,7 +1184,7 @@ class MultiModalContentParser(BaseMultiModalContentParser): def parse_video_embeds( self, - video_embeds: str | dict[str, str] | None, + video_embeds: MultiModalEmbedsPayload | None, uuid: str | None = None, ) -> None: mm_config = self.model_config.get_multimodal_config() @@ -1160,7 +1196,7 @@ class MultiModalContentParser(BaseMultiModalContentParser): if isinstance(video_embeds, dict): embeds = { - k: self._connector.fetch_video_embedding(v) + k: self._connector.fetch_video_embedding(v) if isinstance(v, str) else v for k, v in video_embeds.items() } placeholder = self._tracker.add("video_embeds", (embeds, uuid)) @@ -1242,7 +1278,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser): def parse_image_embeds( self, - image_embeds: str | dict[str, str] | None, + image_embeds: MultiModalEmbedsPayload | None, uuid: str | None = None, ) -> None: mm_config = self.model_config.get_multimodal_config() @@ -1260,17 +1296,13 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser): async def _image_embeds_with_uuid_async( self, - image_embeds: str | dict[str, str] | None, + image_embeds: MultiModalEmbedsPayload | None, uuid: str | None, ): if isinstance(image_embeds, dict): - tensors = await asyncio.gather( - *( - self._connector.fetch_image_embedding_async(v) - for v in image_embeds.values() - ) + embeds = await _load_embeds_dict( + image_embeds, self._connector.fetch_image_embedding_async ) - embeds = dict(zip(image_embeds, tensors)) elif isinstance(image_embeds, str): embeds = await self._connector.fetch_image_embedding_async(image_embeds) else: @@ -1279,7 +1311,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser): def parse_audio_embeds( self, - audio_embeds: str | dict[str, str] | None, + audio_embeds: MultiModalEmbedsPayload | None, uuid: str | None = None, ) -> None: mm_config = self.model_config.get_multimodal_config() @@ -1297,17 +1329,13 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser): async def _audio_embeds_with_uuid_async( self, - audio_embeds: str | dict[str, str] | None, + audio_embeds: MultiModalEmbedsPayload | None, uuid: str | None, ): if isinstance(audio_embeds, dict): - tensors = await asyncio.gather( - *( - self._connector.fetch_audio_embedding_async(v) - for v in audio_embeds.values() - ) + embeds = await _load_embeds_dict( + audio_embeds, self._connector.fetch_audio_embedding_async ) - embeds = dict(zip(audio_embeds, tensors)) elif isinstance(audio_embeds, str): embeds = await self._connector.fetch_audio_embedding_async(audio_embeds) else: @@ -1382,7 +1410,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser): def parse_video_embeds( self, - video_embeds: str | dict[str, str] | None, + video_embeds: MultiModalEmbedsPayload | None, uuid: str | None = None, ) -> None: mm_config = self.model_config.get_multimodal_config() @@ -1400,17 +1428,13 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser): async def _video_embeds_with_uuid_async( self, - video_embeds: str | dict[str, str] | None, + video_embeds: MultiModalEmbedsPayload | None, uuid: str | None, ): if isinstance(video_embeds, dict): - tensors = await asyncio.gather( - *( - self._connector.fetch_video_embedding_async(v) - for v in video_embeds.values() - ) + embeds = await _load_embeds_dict( + video_embeds, self._connector.fetch_video_embedding_async ) - embeds = dict(zip(video_embeds, tensors)) elif isinstance(video_embeds, str): embeds = await self._connector.fetch_video_embedding_async(video_embeds) else: @@ -1611,7 +1635,9 @@ _AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam).validate_python _VideoParser = TypeAdapter(ChatCompletionContentPartVideoParam).validate_python _ResponsesInputImageParser = TypeAdapter(ResponseInputImageParam).validate_python -_ContentPart: TypeAlias = str | dict[str, str] | InputAudio | PILImage +_ContentPart: TypeAlias = ( + MultiModalEmbedsPayload | dict[str, str] | InputAudio | PILImage +) # Define a mapping from part types to their corresponding parsing functions. MM_PARSER_MAP: dict[ @@ -1901,15 +1927,21 @@ def _parse_chat_message_content_part( mm_parser.parse_image(str_content, uuid) modality = "image" elif part_type == "image_embeds": - content = cast(str | dict[str, str], content) if content is not None else None + content = ( + cast(MultiModalEmbedsPayload, content) if content is not None else None + ) mm_parser.parse_image_embeds(content, uuid) modality = "image" elif part_type == "audio_embeds": - content = cast(str | dict[str, str], content) if content is not None else None + content = ( + cast(MultiModalEmbedsPayload, content) if content is not None else None + ) mm_parser.parse_audio_embeds(content, uuid) modality = "audio" elif part_type == "video_embeds": - content = cast(str | dict[str, str], content) if content is not None else None + content = ( + cast(MultiModalEmbedsPayload, content) if content is not None else None + ) mm_parser.parse_video_embeds(content, uuid) modality = "video" elif part_type == "prompt_embeds": diff --git a/vllm/utils/collection_utils.py b/vllm/utils/collection_utils.py index e0bd2045f7..5852aff549 100644 --- a/vllm/utils/collection_utils.py +++ b/vllm/utils/collection_utils.py @@ -6,6 +6,7 @@ Contains helpers that are applied to collections. This is similar in concept to the `collections` module. """ +import math from collections import defaultdict from collections.abc import Callable, Generator, Hashable, Iterable, Mapping, Sequence from typing import Generic, Literal, TypeVar @@ -68,6 +69,13 @@ def is_list_of( assert_never(check) +def is_list_of_numbers(value: object) -> bool: + """Check every item is an int or finite float, excluding booleans.""" + return isinstance(value, list) and all( + type(v) is int or (type(v) is float and math.isfinite(v)) for v in value + ) + + @overload def common_prefix(items: Sequence[str]) -> str: ...