[Multimodal] Add Qwen3-VL video loader (#44412)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
2026-06-16 03:45:34 +00:00
committed by GitHub
parent f99260d2aa
commit e3cfea2e1b
3 changed files with 100 additions and 3 deletions
+50 -2
View File
@@ -1,11 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
from pathlib import Path
import numpy as np
import numpy.typing as npt
import pytest
from transformers import AutoVideoProcessor
from transformers.video_utils import VideoMetadata
from vllm.assets.base import get_vllm_public_assets
from vllm.multimodal.video import (
@@ -13,6 +15,7 @@ from vllm.multimodal.video import (
DynamicVideoBackend,
GLM46VVideoBackend,
Molmo2VideoBackend,
Qwen3VLVideoBackend,
VideoLoader,
VideoSourceMetadata,
VideoTargetMetadata,
@@ -72,6 +75,9 @@ def test_video_loader_type_doesnt_exist():
pytest.param(
"allenai/Molmo2-4B",
Molmo2VideoBackend,
marks=pytest.mark.skip(
reason="Video processor not aligned, investigate later.",
),
id="molmo2",
),
pytest.param(
@@ -84,6 +90,11 @@ def test_video_loader_type_doesnt_exist():
GLM46VVideoBackend,
id="glm46v",
),
pytest.param(
"Qwen/Qwen3-VL-4B-Instruct",
Qwen3VLVideoBackend,
id="qwen3vl",
),
],
)
def test_video_processor_from_model_repo(
@@ -94,7 +105,9 @@ def test_video_processor_from_model_repo(
The test downloads the preprocessor config from HuggingFace Hub,
extracts the ``video_processor_type`` field, and verifies it maps
to the expected backend and loader class.
to the expected backend and loader class. When a corresponding HF
``VideoProcessor.sample_frames`` implementation exists, the test
also verifies that the vLLM backend produces identical frame indices.
"""
video_processor = get_video_processor_cls_name_from_config(model_repo)
assert video_processor is not None, (
@@ -109,6 +122,41 @@ def test_video_processor_from_model_repo(
f"{type(loader)}, expected {expected_loader_cls}"
)
# --- Alignment check with HF VideoProcessor.sample_frames ---
processor = AutoVideoProcessor.from_pretrained(model_repo, trust_remote_code=True)
fps_list = [1, 2, 30, 60]
duration_list = [10, 60, 600]
for fps, duration_secs in itertools.product(fps_list, duration_list):
num_frames = fps * duration_secs
video_bytes = create_long_gop_video(
num_frames=num_frames,
fps=fps,
width=8,
height=8,
)
_, vllm_meta = loader.load_bytes(video_bytes) # type: ignore[attr-defined]
hf_metadata = VideoMetadata(
total_num_frames=vllm_meta["total_num_frames"],
fps=vllm_meta["fps"],
duration=vllm_meta["duration"],
)
hf_indices = processor.sample_frames(hf_metadata)
vllm_indices = np.array(vllm_meta["frames_indices"])
np.testing.assert_array_equal(
hf_indices,
vllm_indices,
err_msg=(
f"{model_repo!r} fps={fps} duration={duration_secs}s: "
f"HF has {len(hf_indices)} indices "
f"{hf_indices[:5].tolist()}..{hf_indices[-5:].tolist()}, "
f"vLLM has {len(vllm_indices)} indices "
f"{vllm_indices[:5].tolist()}..{vllm_indices[-5:].tolist()}"
),
)
def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
"""
+1 -1
View File
@@ -94,7 +94,7 @@ def create_long_gop_video(
}
for i in range(num_frames):
img = np.zeros((height, width, 3), dtype=np.uint8)
img[:, :, 1] = i
img[:, :, 1] = i % 256
frame = av.VideoFrame.from_ndarray(img, format="rgb24")
for packet in stream.encode(frame):
container.mux(packet)
+49
View File
@@ -604,6 +604,55 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
)
@VIDEO_LOADER_REGISTRY.register(
"qwen3_vl",
video_processor="Qwen3VLVideoProcessor",
)
class Qwen3VLVideoBackend(VideoBackend):
@classmethod
def compute_frames_index_to_sample(
cls,
source: VideoSourceMetadata,
target: VideoTargetMetadata,
**kwargs,
) -> list[int]:
total_frames_num = source.total_frames_num
original_fps = source.original_fps
fps = target.fps
max_frame_idx = source.total_frames_num - 1
min_frames = kwargs.get("min_frames", 4)
max_frames = kwargs.get("max_frames", 768)
# Refer to:
# https://github.com/huggingface/transformers/blob/v5.9.0/src/transformers/models/qwen3_vl/video_processing_qwen3_vl.py#L119-L125
num_frames = int(total_frames_num / original_fps * fps)
num_frames = min(max(num_frames, min_frames), max_frames, total_frames_num)
indices = np.linspace(0, max_frame_idx, num_frames).round().astype(int).tolist()
return indices
@classmethod
def load_bytes(
cls,
data: bytes,
num_frames: int = -1,
fps: int = 2,
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal["opencv", "pyav"] = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
data,
num_frames=num_frames,
fps=fps,
max_duration=max_duration,
frame_recovery=frame_recovery,
backend=backend,
**kwargs,
)
@VIDEO_LOADER_REGISTRY.register(
"opencv_dynamic",
video_processor="Glm4vVideoProcessor",