[Bugfix][Qwen2.5-VL] Honor video fps for temporal M-RoPE (#47736)

Signed-off-by: Ting Sun <suntcrick@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Ting SUN
2026-09-22 05:56:12 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5abf85d27a
commit a33b3bac5d
7 changed files with 117 additions and 8 deletions
@@ -151,10 +151,12 @@ VLM_TEST_SETTINGS = {
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>",
enforce_eager=False,
needs_video_metadata=True,
max_model_len=4096,
max_num_seqs=2,
auto_cls=AutoModelForImageTextToText,
vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
patch_hf_runner=model_utils.qwen3_vl_patch_hf_runner,
image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
),
@@ -4,10 +4,10 @@
import pytest
from vllm.assets.image import ImageAsset
from vllm.multimodal.video import sample_frames_from_video
from vllm.platforms import current_platform
from ....conftest import VIDEO_ASSETS
from .vlm_utils.builders import sample_frames_with_video_metadata
models = ["Qwen/Qwen2.5-VL-3B-Instruct"]
target_dtype = "bfloat16"
@@ -81,7 +81,9 @@ def test_qwen2_5_vl_evs_functionality(
# Sample frames from video assets
sampled_vids = [
sample_frames_from_video(asset.np_ndarrays, num_frames)
sample_frames_with_video_metadata(
(asset.np_ndarrays, asset.metadata), num_frames
)
for asset in video_assets
]
@@ -145,7 +147,9 @@ def test_qwen2_5_vl_evs_batched_videos(
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1" if use_bytecode_hook else "0")
# Sample frames from video assets
sampled_vids = [
sample_frames_from_video(asset.np_ndarrays, num_frames)
sample_frames_with_video_metadata(
(asset.np_ndarrays, asset.metadata), num_frames
)
for asset in video_assets
]
@@ -144,7 +144,7 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
"<|vision_start|><|video_pad|><|vision_end|>"
"Describe this video in one sentence."
),
needs_video_metadata=False,
needs_video_metadata=True,
marks=[pytest.mark.core_model],
),
"kimi_vl": VitCudagraphTestConfig(
@@ -1,6 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import numpy as np
import pytest
import torch
from PIL import Image
@@ -208,6 +211,35 @@ def test_get_image_size_with_most_features(
assert tokens < max_tokens
def _build_qwen2_5_vl_video_mm_data(num_frames: int, fps: float) -> dict[str, Any]:
video = np.zeros((num_frames, 56, 56, 3), dtype=np.uint8)
metadata = {
"fps": fps,
"duration": num_frames / fps,
"total_num_frames": num_frames,
"frames_indices": list(range(num_frames)),
"video_backend": "opencv",
"do_sample_frames": False,
}
return {"video": [(video, metadata)]}
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2.5-VL-3B-Instruct"])
def test_qwen2_5_vl_video_fps_to_second_per_grid_ts(model_id: str) -> None:
ctx = build_model_context(model_id, limit_mm_per_prompt={"image": 0, "video": 1})
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
temporal_patch_size = (
processor.info.get_hf_processor().video_processor.temporal_patch_size
)
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
for fps in (1.0, 30.0):
mm_data = _build_qwen2_5_vl_video_mm_data(num_frames=32, fps=fps)
processed = processor(prompt, mm_items=processor.info.parse_mm_data(mm_data))
spg = processed["mm_kwargs"]["video"][0].get_data()["second_per_grid_ts"]
assert float(spg) == pytest.approx(temporal_patch_size / fps, rel=2e-2)
@pytest.mark.parametrize(
"model_id", ["Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"]
)
@@ -15,7 +15,11 @@ from vllm.tokenizers.registry import cached_tokenizer_from_config
cherry_pil_image = ImageAsset("cherry_blossom").pil_image
stop_pil_image = ImageAsset("stop_sign").pil_image
baby_reading_np_ndarrays = VideoAsset("baby_reading").np_ndarrays
baby_reading_video_asset = VideoAsset("baby_reading")
baby_reading_video = (
baby_reading_video_asset.np_ndarrays,
baby_reading_video_asset.metadata,
)
def _build_renderer(
@@ -127,7 +131,7 @@ def test_multi_modal_uuids_accepts_none_and_passes_through(
mm_data = {
"image": [cherry_pil_image, stop_pil_image],
"video": baby_reading_np_ndarrays,
"video": baby_reading_video,
}
# Use a consistent two-image scenario across all configurations
@@ -207,7 +211,7 @@ def test_multi_modal_uuids_preserved_when_caching_disabled(mm_uuids, expected):
request_id = "req-42"
mm_data = {
"image": [cherry_pil_image, stop_pil_image],
"video": baby_reading_np_ndarrays,
"video": baby_reading_video,
}
mm_processor = renderer.get_mm_processor()