[Multimodal] Add Qwen3-VL video loader (#44412)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -1,11 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import itertools
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
import pytest
|
||||
from transformers import AutoVideoProcessor
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.assets.base import get_vllm_public_assets
|
||||
from vllm.multimodal.video import (
|
||||
@@ -13,6 +15,7 @@ from vllm.multimodal.video import (
|
||||
DynamicVideoBackend,
|
||||
GLM46VVideoBackend,
|
||||
Molmo2VideoBackend,
|
||||
Qwen3VLVideoBackend,
|
||||
VideoLoader,
|
||||
VideoSourceMetadata,
|
||||
VideoTargetMetadata,
|
||||
@@ -72,6 +75,9 @@ def test_video_loader_type_doesnt_exist():
|
||||
pytest.param(
|
||||
"allenai/Molmo2-4B",
|
||||
Molmo2VideoBackend,
|
||||
marks=pytest.mark.skip(
|
||||
reason="Video processor not aligned, investigate later.",
|
||||
),
|
||||
id="molmo2",
|
||||
),
|
||||
pytest.param(
|
||||
@@ -84,6 +90,11 @@ def test_video_loader_type_doesnt_exist():
|
||||
GLM46VVideoBackend,
|
||||
id="glm46v",
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen/Qwen3-VL-4B-Instruct",
|
||||
Qwen3VLVideoBackend,
|
||||
id="qwen3vl",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_video_processor_from_model_repo(
|
||||
@@ -94,7 +105,9 @@ def test_video_processor_from_model_repo(
|
||||
|
||||
The test downloads the preprocessor config from HuggingFace Hub,
|
||||
extracts the ``video_processor_type`` field, and verifies it maps
|
||||
to the expected backend and loader class.
|
||||
to the expected backend and loader class. When a corresponding HF
|
||||
``VideoProcessor.sample_frames`` implementation exists, the test
|
||||
also verifies that the vLLM backend produces identical frame indices.
|
||||
"""
|
||||
video_processor = get_video_processor_cls_name_from_config(model_repo)
|
||||
assert video_processor is not None, (
|
||||
@@ -109,6 +122,41 @@ def test_video_processor_from_model_repo(
|
||||
f"{type(loader)}, expected {expected_loader_cls}"
|
||||
)
|
||||
|
||||
# --- Alignment check with HF VideoProcessor.sample_frames ---
|
||||
processor = AutoVideoProcessor.from_pretrained(model_repo, trust_remote_code=True)
|
||||
|
||||
fps_list = [1, 2, 30, 60]
|
||||
duration_list = [10, 60, 600]
|
||||
for fps, duration_secs in itertools.product(fps_list, duration_list):
|
||||
num_frames = fps * duration_secs
|
||||
video_bytes = create_long_gop_video(
|
||||
num_frames=num_frames,
|
||||
fps=fps,
|
||||
width=8,
|
||||
height=8,
|
||||
)
|
||||
|
||||
_, vllm_meta = loader.load_bytes(video_bytes) # type: ignore[attr-defined]
|
||||
|
||||
hf_metadata = VideoMetadata(
|
||||
total_num_frames=vllm_meta["total_num_frames"],
|
||||
fps=vllm_meta["fps"],
|
||||
duration=vllm_meta["duration"],
|
||||
)
|
||||
hf_indices = processor.sample_frames(hf_metadata)
|
||||
vllm_indices = np.array(vllm_meta["frames_indices"])
|
||||
np.testing.assert_array_equal(
|
||||
hf_indices,
|
||||
vllm_indices,
|
||||
err_msg=(
|
||||
f"{model_repo!r} fps={fps} duration={duration_secs}s: "
|
||||
f"HF has {len(hf_indices)} indices "
|
||||
f"{hf_indices[:5].tolist()}..{hf_indices[-5:].tolist()}, "
|
||||
f"vLLM has {len(vllm_indices)} indices "
|
||||
f"{vllm_indices[:5].tolist()}..{vllm_indices[-5:].tolist()}"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
|
||||
@@ -94,7 +94,7 @@ def create_long_gop_video(
|
||||
}
|
||||
for i in range(num_frames):
|
||||
img = np.zeros((height, width, 3), dtype=np.uint8)
|
||||
img[:, :, 1] = i
|
||||
img[:, :, 1] = i % 256
|
||||
frame = av.VideoFrame.from_ndarray(img, format="rgb24")
|
||||
for packet in stream.encode(frame):
|
||||
container.mux(packet)
|
||||
|
||||
Reference in New Issue
Block a user