[Frontend] Add video embeds input support (#54242)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -98,6 +98,16 @@ def qwen25omni_model_config_image_embeds():
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def qwen25omni_model_config_video_embeds():
|
||||
return ModelConfig(
|
||||
QWEN25OMNI_MODEL_ID,
|
||||
runner="generate",
|
||||
limit_mm_per_prompt={"video": 2},
|
||||
enable_mm_embeds=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def qwen2_audio_model_config():
|
||||
return ModelConfig(
|
||||
@@ -969,6 +979,133 @@ async def test_parse_chat_messages_audio_embeds_async(
|
||||
_assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
|
||||
|
||||
|
||||
def test_parse_chat_messages_empty_video_embeds_with_uuid(
|
||||
qwen25omni_model_config_video_embeds,
|
||||
):
|
||||
"""Test video_embeds with UUID (no actual embeds data)."""
|
||||
uuid = "test-video-uuid-123"
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this video"},
|
||||
{"type": "video_embeds", "video_embeds": None, "uuid": uuid},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen25omni_model_config_video_embeds,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
# Should have video in mm_data as None (UUID provided)
|
||||
assert mm_data is not None
|
||||
assert "video" in mm_data
|
||||
assert isinstance(mm_data["video"], list)
|
||||
assert len(mm_data["video"]) == 1
|
||||
assert mm_data["video"][0] is None
|
||||
|
||||
# UUID should be recorded
|
||||
_assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[uuid])
|
||||
|
||||
|
||||
def test_parse_chat_messages_video_embeds_with_string(
|
||||
qwen25omni_model_config_video_embeds,
|
||||
):
|
||||
"""Test video_embeds with base64 string embedding data."""
|
||||
hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size()
|
||||
video_embedding = torch.randn(1, 128, hidden_size)
|
||||
|
||||
base64_video_embedding = tensor2base64(video_embedding)
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this video"},
|
||||
{
|
||||
"type": "video_embeds",
|
||||
"video_embeds": base64_video_embedding,
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen25omni_model_config_video_embeds,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
# Should have video embedding in mm_data (single tensor, not a list)
|
||||
assert mm_data is not None
|
||||
assert "video" in mm_data
|
||||
assert isinstance(mm_data["video"], torch.Tensor)
|
||||
assert mm_data["video"].shape == video_embedding.shape
|
||||
# No UUID provided
|
||||
assert mm_uuids is not None
|
||||
assert "video" in mm_uuids
|
||||
_assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_chat_messages_video_embeds_async(
|
||||
qwen25omni_model_config_video_embeds,
|
||||
):
|
||||
"""Test video_embeds with async futures."""
|
||||
hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size()
|
||||
video_embedding = torch.randn(1, 128, hidden_size)
|
||||
|
||||
base64_video_embedding = tensor2base64(video_embedding)
|
||||
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe this video"},
|
||||
{
|
||||
"type": "video_embeds",
|
||||
"video_embeds": base64_video_embedding,
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen25omni_model_config_video_embeds,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
# Should have video embedding in mm_data (single tensor, not a list)
|
||||
assert mm_data is not None
|
||||
assert "video" in mm_data
|
||||
assert isinstance(mm_data["video"], torch.Tensor)
|
||||
assert mm_data["video"].shape == video_embedding.shape
|
||||
# No UUID provided
|
||||
assert mm_uuids is not None
|
||||
assert "video" in mm_uuids
|
||||
_assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
|
||||
|
||||
|
||||
def test_parse_chat_messages_video_and_video_embeds_mixed(
|
||||
qwen25omni_model_config_video_embeds,
|
||||
video_url,
|
||||
):
|
||||
"""Test that mixing raw video and video_embeds is rejected."""
|
||||
with pytest.raises(VLLMValidationError, match="Mixing raw video"):
|
||||
parse_chat_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "video_url", "video_url": {"url": video_url}},
|
||||
{"type": "video_embeds", "video_embeds": None, "uuid": "x"},
|
||||
],
|
||||
}
|
||||
],
|
||||
qwen25omni_model_config_video_embeds,
|
||||
content_format="string",
|
||||
)
|
||||
|
||||
|
||||
def test_parse_chat_messages_multiple_image_embeds(
|
||||
phi3v_model_config_image_embeds,
|
||||
):
|
||||
|
||||
@@ -11,8 +11,10 @@ from typing import Any
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
from ...utils import build_model_context
|
||||
|
||||
MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
|
||||
@@ -186,6 +188,89 @@ def test_processor_multi_video_list_kwargs(
|
||||
)
|
||||
|
||||
|
||||
def _build_video_embeds_mm_data(
|
||||
hidden_size: int,
|
||||
grid_thw: tuple[int, int, int] = (2, 4, 4),
|
||||
) -> dict[str, Any]:
|
||||
"""Create an embeds-only video item as an EPD consumer receives it:
|
||||
pre-computed embeddings plus the metadata published by the encoder."""
|
||||
import torch
|
||||
|
||||
t, h, w = grid_thw
|
||||
num_tokens = t * h * w // 4 # spatial_merge_size ** 2
|
||||
return {
|
||||
"video": {
|
||||
"video_embeds": torch.zeros(num_tokens, hidden_size),
|
||||
"video_grid_thw": torch.tensor([grid_thw]),
|
||||
"timestamps": torch.tensor([[float(i) for i in range(t)]]),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_video_embeds_with_timestamps(model_id: str) -> None:
|
||||
"""Embeds-only video input must size the placeholder range from the
|
||||
grid and the real timestamps published by the encoder (EC consumer
|
||||
path); synthesized or missing timestamps would change the token count
|
||||
and break embedding merging downstream."""
|
||||
# `build_model_context` forces enable_mm_embeds off, so build the
|
||||
# config directly; keep the same online-availability skip behavior.
|
||||
HF_EXAMPLE_MODELS.find_hf_info(model_id).check_available_online(on_fail="skip")
|
||||
model_config = ModelConfig(
|
||||
model_id,
|
||||
runner="generate",
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
enable_mm_embeds=True,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
|
||||
grid_thw = (2, 4, 4)
|
||||
mm_data = _build_video_embeds_mm_data(
|
||||
model_config.get_inputs_embeds_size(), grid_thw
|
||||
)
|
||||
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
video_phs = processed["mm_placeholders"].get("video", [])
|
||||
assert len(video_phs) == 1, (
|
||||
f"Expected exactly 1 video placeholder, got {len(video_phs)}"
|
||||
)
|
||||
|
||||
t, h, w = grid_thw
|
||||
tokens_per_frame = h * w // 4
|
||||
timestamp_tokens = sum(
|
||||
len(tokenizer.encode(f"<{float(i):.1f} seconds>", add_special_tokens=False))
|
||||
for i in range(t)
|
||||
)
|
||||
# per frame: timestamp tokens + vision_start + video tokens + vision_end
|
||||
expected_len = timestamp_tokens + t * (tokens_per_frame + 2)
|
||||
assert video_phs[0].length == expected_len
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_video_embeds_missing_timestamps(model_id: str) -> None:
|
||||
"""Timestamps are required metadata for video embeds: they size the
|
||||
placeholder range, so omitting them must fail loudly at parse time
|
||||
instead of silently producing a wrong prompt."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
mm_data = _build_video_embeds_mm_data(ctx.model_config.get_inputs_embeds_size())
|
||||
del mm_data["video"]["timestamps"]
|
||||
|
||||
with pytest.raises(ValueError, match="timestamps"):
|
||||
processor.info.parse_mm_data(mm_data)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> None:
|
||||
"""Regression test for memory profiling with ``cap_pixels_per_frame``.
|
||||
|
||||
Reference in New Issue
Block a user