[Frontend][EPD] Use JSON arrays for multimodal metadata (#56090)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-09-10 05:04:14 +00:00
committed by GitHub
parent 2a02f6efe3
commit 479611c0c6
6 changed files with 167 additions and 62 deletions
@@ -18,6 +18,8 @@ from vllm.entrypoints.chat_utils import (
MEDIA_CONNECTOR_REGISTRY,
AsyncMultiModalItemTracker,
ConversationMessage,
_load_embeds_dict,
_parse_metadata_array,
_postprocess_messages,
parse_chat_messages,
parse_chat_messages_async,
@@ -39,6 +41,45 @@ QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B"
MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
@pytest.mark.parametrize("values", [[1, 32, 48], [0.0, 0.12345678912345678]])
def test_json_metadata_preserves_numeric_precision(values):
tensor = _parse_metadata_array("grid", values, {"grid"})
assert tensor.tolist() == values
assert tensor.dtype == (
torch.float64 if isinstance(values[0], float) else torch.int64
)
@pytest.mark.parametrize(
"key,values",
[
("image_embeds", [1, 2]),
("grid", [True]),
("grid", [float("nan")]),
("grid", [float("inf")]),
("grid", [[1, 2]]),
("grid", [2**64]),
],
)
def test_json_arrays_are_only_valid_numeric_metadata(key, values):
with pytest.raises(VLLMValidationError):
_parse_metadata_array(key, values, {"grid"})
@pytest.mark.asyncio
async def test_json_metadata_bypasses_tensor_deserialization():
from unittest.mock import AsyncMock
tensor = torch.ones(2, 3)
fetch = AsyncMock(return_value=tensor)
result = await _load_embeds_dict(
{"image_embeds": "legacy-base64", "image_grid_thw": [1, 2, 2]}, fetch
)
fetch.assert_awaited_once_with("legacy-base64")
assert result["image_embeds"] is tensor
assert result["image_grid_thw"] == [1, 2, 2]
@pytest.fixture(scope="function")
def kimi_k2_5_model_config():
return ModelConfig(
@@ -1355,8 +1396,10 @@ def test_parse_chat_messages_empty_dict_image_embeds(
_assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])
@pytest.mark.parametrize("json_metadata", [False, True])
def test_parse_chat_messages_multiple_dict_image_embeds(
qwen25omni_model_config_image_embeds,
json_metadata,
):
"""Test that multiple dictionaries for image_embeds is handled without errors."""
# Create two sample image embedding tensors
@@ -1374,7 +1417,9 @@ def test_parse_chat_messages_multiple_dict_image_embeds(
"type": "image_embeds",
"image_embeds": {
"image_embeds": tensor2base64(embeds),
"image_grid_thw": tensor2base64(grid_thw),
"image_grid_thw": grid_thw.tolist()
if json_metadata
else tensor2base64(grid_thw),
},
}
for embeds, grid_thw in zip(