[Frontend] Add video embeds input support (#54242)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-08-31 03:20:47 +00:00
committed by GitHub
parent 5707355209
commit 555ea65e8c
8 changed files with 461 additions and 24 deletions
@@ -98,6 +98,16 @@ def qwen25omni_model_config_image_embeds():
)
@pytest.fixture(scope="function")
def qwen25omni_model_config_video_embeds():
return ModelConfig(
QWEN25OMNI_MODEL_ID,
runner="generate",
limit_mm_per_prompt={"video": 2},
enable_mm_embeds=True,
)
@pytest.fixture(scope="function")
def qwen2_audio_model_config():
return ModelConfig(
@@ -969,6 +979,133 @@ async def test_parse_chat_messages_audio_embeds_async(
_assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
def test_parse_chat_messages_empty_video_embeds_with_uuid(
qwen25omni_model_config_video_embeds,
):
"""Test video_embeds with UUID (no actual embeds data)."""
uuid = "test-video-uuid-123"
conversation, mm_data, mm_uuids = parse_chat_messages(
[
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this video"},
{"type": "video_embeds", "video_embeds": None, "uuid": uuid},
],
}
],
qwen25omni_model_config_video_embeds,
content_format="string",
)
# Should have video in mm_data as None (UUID provided)
assert mm_data is not None
assert "video" in mm_data
assert isinstance(mm_data["video"], list)
assert len(mm_data["video"]) == 1
assert mm_data["video"][0] is None
# UUID should be recorded
_assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[uuid])
def test_parse_chat_messages_video_embeds_with_string(
qwen25omni_model_config_video_embeds,
):
"""Test video_embeds with base64 string embedding data."""
hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size()
video_embedding = torch.randn(1, 128, hidden_size)
base64_video_embedding = tensor2base64(video_embedding)
conversation, mm_data, mm_uuids = parse_chat_messages(
[
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this video"},
{
"type": "video_embeds",
"video_embeds": base64_video_embedding,
},
],
}
],
qwen25omni_model_config_video_embeds,
content_format="string",
)
# Should have video embedding in mm_data (single tensor, not a list)
assert mm_data is not None
assert "video" in mm_data
assert isinstance(mm_data["video"], torch.Tensor)
assert mm_data["video"].shape == video_embedding.shape
# No UUID provided
assert mm_uuids is not None
assert "video" in mm_uuids
_assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
@pytest.mark.asyncio
async def test_parse_chat_messages_video_embeds_async(
qwen25omni_model_config_video_embeds,
):
"""Test video_embeds with async futures."""
hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size()
video_embedding = torch.randn(1, 128, hidden_size)
base64_video_embedding = tensor2base64(video_embedding)
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
[
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this video"},
{
"type": "video_embeds",
"video_embeds": base64_video_embedding,
},
],
}
],
qwen25omni_model_config_video_embeds,
content_format="string",
)
# Should have video embedding in mm_data (single tensor, not a list)
assert mm_data is not None
assert "video" in mm_data
assert isinstance(mm_data["video"], torch.Tensor)
assert mm_data["video"].shape == video_embedding.shape
# No UUID provided
assert mm_uuids is not None
assert "video" in mm_uuids
_assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
def test_parse_chat_messages_video_and_video_embeds_mixed(
qwen25omni_model_config_video_embeds,
video_url,
):
"""Test that mixing raw video and video_embeds is rejected."""
with pytest.raises(VLLMValidationError, match="Mixing raw video"):
parse_chat_messages(
[
{
"role": "user",
"content": [
{"type": "video_url", "video_url": {"url": video_url}},
{"type": "video_embeds", "video_embeds": None, "uuid": "x"},
],
}
],
qwen25omni_model_config_video_embeds,
content_format="string",
)
def test_parse_chat_messages_multiple_image_embeds(
phi3v_model_config_image_embeds,
):