[Multimodal] Add Qwen3-VL video loader (#44412)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -1,11 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import itertools
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
import pytest
|
||||
from transformers import AutoVideoProcessor
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.assets.base import get_vllm_public_assets
|
||||
from vllm.multimodal.video import (
|
||||
@@ -13,6 +15,7 @@ from vllm.multimodal.video import (
|
||||
DynamicVideoBackend,
|
||||
GLM46VVideoBackend,
|
||||
Molmo2VideoBackend,
|
||||
Qwen3VLVideoBackend,
|
||||
VideoLoader,
|
||||
VideoSourceMetadata,
|
||||
VideoTargetMetadata,
|
||||
@@ -72,6 +75,9 @@ def test_video_loader_type_doesnt_exist():
|
||||
pytest.param(
|
||||
"allenai/Molmo2-4B",
|
||||
Molmo2VideoBackend,
|
||||
marks=pytest.mark.skip(
|
||||
reason="Video processor not aligned, investigate later.",
|
||||
),
|
||||
id="molmo2",
|
||||
),
|
||||
pytest.param(
|
||||
@@ -84,6 +90,11 @@ def test_video_loader_type_doesnt_exist():
|
||||
GLM46VVideoBackend,
|
||||
id="glm46v",
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen/Qwen3-VL-4B-Instruct",
|
||||
Qwen3VLVideoBackend,
|
||||
id="qwen3vl",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_video_processor_from_model_repo(
|
||||
@@ -94,7 +105,9 @@ def test_video_processor_from_model_repo(
|
||||
|
||||
The test downloads the preprocessor config from HuggingFace Hub,
|
||||
extracts the ``video_processor_type`` field, and verifies it maps
|
||||
to the expected backend and loader class.
|
||||
to the expected backend and loader class. When a corresponding HF
|
||||
``VideoProcessor.sample_frames`` implementation exists, the test
|
||||
also verifies that the vLLM backend produces identical frame indices.
|
||||
"""
|
||||
video_processor = get_video_processor_cls_name_from_config(model_repo)
|
||||
assert video_processor is not None, (
|
||||
@@ -109,6 +122,41 @@ def test_video_processor_from_model_repo(
|
||||
f"{type(loader)}, expected {expected_loader_cls}"
|
||||
)
|
||||
|
||||
# --- Alignment check with HF VideoProcessor.sample_frames ---
|
||||
processor = AutoVideoProcessor.from_pretrained(model_repo, trust_remote_code=True)
|
||||
|
||||
fps_list = [1, 2, 30, 60]
|
||||
duration_list = [10, 60, 600]
|
||||
for fps, duration_secs in itertools.product(fps_list, duration_list):
|
||||
num_frames = fps * duration_secs
|
||||
video_bytes = create_long_gop_video(
|
||||
num_frames=num_frames,
|
||||
fps=fps,
|
||||
width=8,
|
||||
height=8,
|
||||
)
|
||||
|
||||
_, vllm_meta = loader.load_bytes(video_bytes) # type: ignore[attr-defined]
|
||||
|
||||
hf_metadata = VideoMetadata(
|
||||
total_num_frames=vllm_meta["total_num_frames"],
|
||||
fps=vllm_meta["fps"],
|
||||
duration=vllm_meta["duration"],
|
||||
)
|
||||
hf_indices = processor.sample_frames(hf_metadata)
|
||||
vllm_indices = np.array(vllm_meta["frames_indices"])
|
||||
np.testing.assert_array_equal(
|
||||
hf_indices,
|
||||
vllm_indices,
|
||||
err_msg=(
|
||||
f"{model_repo!r} fps={fps} duration={duration_secs}s: "
|
||||
f"HF has {len(hf_indices)} indices "
|
||||
f"{hf_indices[:5].tolist()}..{hf_indices[-5:].tolist()}, "
|
||||
f"vLLM has {len(vllm_indices)} indices "
|
||||
f"{vllm_indices[:5].tolist()}..{vllm_indices[-5:].tolist()}"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
|
||||
@@ -94,7 +94,7 @@ def create_long_gop_video(
|
||||
}
|
||||
for i in range(num_frames):
|
||||
img = np.zeros((height, width, 3), dtype=np.uint8)
|
||||
img[:, :, 1] = i
|
||||
img[:, :, 1] = i % 256
|
||||
frame = av.VideoFrame.from_ndarray(img, format="rgb24")
|
||||
for packet in stream.encode(frame):
|
||||
container.mux(packet)
|
||||
|
||||
@@ -604,6 +604,55 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
|
||||
)
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register(
|
||||
"qwen3_vl",
|
||||
video_processor="Qwen3VLVideoProcessor",
|
||||
)
|
||||
class Qwen3VLVideoBackend(VideoBackend):
|
||||
@classmethod
|
||||
def compute_frames_index_to_sample(
|
||||
cls,
|
||||
source: VideoSourceMetadata,
|
||||
target: VideoTargetMetadata,
|
||||
**kwargs,
|
||||
) -> list[int]:
|
||||
total_frames_num = source.total_frames_num
|
||||
original_fps = source.original_fps
|
||||
fps = target.fps
|
||||
max_frame_idx = source.total_frames_num - 1
|
||||
min_frames = kwargs.get("min_frames", 4)
|
||||
max_frames = kwargs.get("max_frames", 768)
|
||||
|
||||
# Refer to:
|
||||
# https://github.com/huggingface/transformers/blob/v5.9.0/src/transformers/models/qwen3_vl/video_processing_qwen3_vl.py#L119-L125
|
||||
num_frames = int(total_frames_num / original_fps * fps)
|
||||
num_frames = min(max(num_frames, min_frames), max_frames, total_frames_num)
|
||||
indices = np.linspace(0, max_frame_idx, num_frames).round().astype(int).tolist()
|
||||
return indices
|
||||
|
||||
@classmethod
|
||||
def load_bytes(
|
||||
cls,
|
||||
data: bytes,
|
||||
num_frames: int = -1,
|
||||
fps: int = 2,
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal["opencv", "pyav"] = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
data,
|
||||
num_frames=num_frames,
|
||||
fps=fps,
|
||||
max_duration=max_duration,
|
||||
frame_recovery=frame_recovery,
|
||||
backend=backend,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register(
|
||||
"opencv_dynamic",
|
||||
video_processor="Glm4vVideoProcessor",
|
||||
|
||||
Reference in New Issue
Block a user