[Bugfix][Qwen2.5-VL] Honor video fps for temporal M-RoPE (#47736)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5abf85d27a
commit
a33b3bac5d
@@ -151,10 +151,12 @@ VLM_TEST_SETTINGS = {
|
||||
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
|
||||
video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>",
|
||||
enforce_eager=False,
|
||||
needs_video_metadata=True,
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
|
||||
patch_hf_runner=model_utils.qwen3_vl_patch_hf_runner,
|
||||
image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
),
|
||||
|
||||
@@ -4,10 +4,10 @@
|
||||
import pytest
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.multimodal.video import sample_frames_from_video
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import VIDEO_ASSETS
|
||||
from .vlm_utils.builders import sample_frames_with_video_metadata
|
||||
|
||||
models = ["Qwen/Qwen2.5-VL-3B-Instruct"]
|
||||
target_dtype = "bfloat16"
|
||||
@@ -81,7 +81,9 @@ def test_qwen2_5_vl_evs_functionality(
|
||||
|
||||
# Sample frames from video assets
|
||||
sampled_vids = [
|
||||
sample_frames_from_video(asset.np_ndarrays, num_frames)
|
||||
sample_frames_with_video_metadata(
|
||||
(asset.np_ndarrays, asset.metadata), num_frames
|
||||
)
|
||||
for asset in video_assets
|
||||
]
|
||||
|
||||
@@ -145,7 +147,9 @@ def test_qwen2_5_vl_evs_batched_videos(
|
||||
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1" if use_bytecode_hook else "0")
|
||||
# Sample frames from video assets
|
||||
sampled_vids = [
|
||||
sample_frames_from_video(asset.np_ndarrays, num_frames)
|
||||
sample_frames_with_video_metadata(
|
||||
(asset.np_ndarrays, asset.metadata), num_frames
|
||||
)
|
||||
for asset in video_assets
|
||||
]
|
||||
|
||||
|
||||
@@ -144,7 +144,7 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"Describe this video in one sentence."
|
||||
),
|
||||
needs_video_metadata=False,
|
||||
needs_video_metadata=True,
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"kimi_vl": VitCudagraphTestConfig(
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
@@ -208,6 +211,35 @@ def test_get_image_size_with_most_features(
|
||||
assert tokens < max_tokens
|
||||
|
||||
|
||||
def _build_qwen2_5_vl_video_mm_data(num_frames: int, fps: float) -> dict[str, Any]:
|
||||
video = np.zeros((num_frames, 56, 56, 3), dtype=np.uint8)
|
||||
metadata = {
|
||||
"fps": fps,
|
||||
"duration": num_frames / fps,
|
||||
"total_num_frames": num_frames,
|
||||
"frames_indices": list(range(num_frames)),
|
||||
"video_backend": "opencv",
|
||||
"do_sample_frames": False,
|
||||
}
|
||||
return {"video": [(video, metadata)]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2.5-VL-3B-Instruct"])
|
||||
def test_qwen2_5_vl_video_fps_to_second_per_grid_ts(model_id: str) -> None:
|
||||
ctx = build_model_context(model_id, limit_mm_per_prompt={"image": 0, "video": 1})
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
temporal_patch_size = (
|
||||
processor.info.get_hf_processor().video_processor.temporal_patch_size
|
||||
)
|
||||
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
for fps in (1.0, 30.0):
|
||||
mm_data = _build_qwen2_5_vl_video_mm_data(num_frames=32, fps=fps)
|
||||
processed = processor(prompt, mm_items=processor.info.parse_mm_data(mm_data))
|
||||
spg = processed["mm_kwargs"]["video"][0].get_data()["second_per_grid_ts"]
|
||||
assert float(spg) == pytest.approx(temporal_patch_size / fps, rel=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id", ["Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"]
|
||||
)
|
||||
|
||||
@@ -15,7 +15,11 @@ from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
|
||||
cherry_pil_image = ImageAsset("cherry_blossom").pil_image
|
||||
stop_pil_image = ImageAsset("stop_sign").pil_image
|
||||
baby_reading_np_ndarrays = VideoAsset("baby_reading").np_ndarrays
|
||||
baby_reading_video_asset = VideoAsset("baby_reading")
|
||||
baby_reading_video = (
|
||||
baby_reading_video_asset.np_ndarrays,
|
||||
baby_reading_video_asset.metadata,
|
||||
)
|
||||
|
||||
|
||||
def _build_renderer(
|
||||
@@ -127,7 +131,7 @@ def test_multi_modal_uuids_accepts_none_and_passes_through(
|
||||
|
||||
mm_data = {
|
||||
"image": [cherry_pil_image, stop_pil_image],
|
||||
"video": baby_reading_np_ndarrays,
|
||||
"video": baby_reading_video,
|
||||
}
|
||||
|
||||
# Use a consistent two-image scenario across all configurations
|
||||
@@ -207,7 +211,7 @@ def test_multi_modal_uuids_preserved_when_caching_disabled(mm_uuids, expected):
|
||||
request_id = "req-42"
|
||||
mm_data = {
|
||||
"image": [cherry_pil_image, stop_pil_image],
|
||||
"video": baby_reading_np_ndarrays,
|
||||
"video": baby_reading_video,
|
||||
}
|
||||
|
||||
mm_processor = renderer.get_mm_processor()
|
||||
|
||||
Reference in New Issue
Block a user