[Frontend] Add video embeds input support (#54242)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-08-31 03:20:47 +00:00
committed by GitHub
parent 5707355209
commit 555ea65e8c
8 changed files with 461 additions and 24 deletions
@@ -11,8 +11,10 @@ from typing import Any
import numpy as np
import pytest
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from ...registry import HF_EXAMPLE_MODELS
from ...utils import build_model_context
MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
@@ -186,6 +188,89 @@ def test_processor_multi_video_list_kwargs(
)
def _build_video_embeds_mm_data(
hidden_size: int,
grid_thw: tuple[int, int, int] = (2, 4, 4),
) -> dict[str, Any]:
"""Create an embeds-only video item as an EPD consumer receives it:
pre-computed embeddings plus the metadata published by the encoder."""
import torch
t, h, w = grid_thw
num_tokens = t * h * w // 4 # spatial_merge_size ** 2
return {
"video": {
"video_embeds": torch.zeros(num_tokens, hidden_size),
"video_grid_thw": torch.tensor([grid_thw]),
"timestamps": torch.tensor([[float(i) for i in range(t)]]),
}
}
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_video_embeds_with_timestamps(model_id: str) -> None:
"""Embeds-only video input must size the placeholder range from the
grid and the real timestamps published by the encoder (EC consumer
path); synthesized or missing timestamps would change the token count
and break embedding merging downstream."""
# `build_model_context` forces enable_mm_embeds off, so build the
# config directly; keep the same online-availability skip behavior.
HF_EXAMPLE_MODELS.find_hf_info(model_id).check_available_online(on_fail="skip")
model_config = ModelConfig(
model_id,
runner="generate",
limit_mm_per_prompt={"image": 0, "video": 1},
enable_mm_embeds=True,
)
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
tokenizer = processor.info.get_tokenizer()
grid_thw = (2, 4, 4)
mm_data = _build_video_embeds_mm_data(
model_config.get_inputs_embeds_size(), grid_thw
)
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == 1, (
f"Expected exactly 1 video placeholder, got {len(video_phs)}"
)
t, h, w = grid_thw
tokens_per_frame = h * w // 4
timestamp_tokens = sum(
len(tokenizer.encode(f"<{float(i):.1f} seconds>", add_special_tokens=False))
for i in range(t)
)
# per frame: timestamp tokens + vision_start + video tokens + vision_end
expected_len = timestamp_tokens + t * (tokens_per_frame + 2)
assert video_phs[0].length == expected_len
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_video_embeds_missing_timestamps(model_id: str) -> None:
"""Timestamps are required metadata for video embeds: they size the
placeholder range, so omitting them must fail loudly at parse time
instead of silently producing a wrong prompt."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
mm_data = _build_video_embeds_mm_data(ctx.model_config.get_inputs_embeds_size())
del mm_data["video"]["timestamps"]
with pytest.raises(ValueError, match="timestamps"):
processor.info.parse_mm_data(mm_data)
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> None:
"""Regression test for memory profiling with ``cap_pixels_per_frame``.