[Multimodal] Add Qwen2-VL/Qwen2.5-VL processor-mapped video loader (#45555)
Signed-off-by: Ranran <hzz5361@psu.edu> Signed-off-by: Ranran Haoran Zhang <ranzhang@redhat.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Isotr0py
parent
c441ad1c07
commit
635c38338a
@@ -25,7 +25,7 @@ def server():
|
||||
"--runner",
|
||||
"pooling",
|
||||
"--max-model-len",
|
||||
"5000",
|
||||
"16384",
|
||||
"--enforce-eager",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"video": MAXIMUM_VIDEOS}),
|
||||
@@ -143,4 +143,4 @@ def test_chat_video_url_request(server: RemoteOpenAIServer, model_name: str):
|
||||
assert output.model == model_name
|
||||
assert len(output.data) == 1
|
||||
assert len(output.data[0].probs) == 2
|
||||
assert output.usage.prompt_tokens == 4807
|
||||
assert output.usage.prompt_tokens == 8993
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.multimodal.video import (
|
||||
DynamicVideoBackend,
|
||||
GLM46VVideoBackend,
|
||||
Molmo2VideoBackend,
|
||||
Qwen2VLVideoBackend,
|
||||
Qwen3VLVideoBackend,
|
||||
VideoLoader,
|
||||
VideoSourceMetadata,
|
||||
@@ -70,11 +71,12 @@ def test_video_loader_type_doesnt_exist():
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_repo, expected_loader_cls",
|
||||
"model_repo, expected_loader_cls, hf_sample_kwargs",
|
||||
[
|
||||
pytest.param(
|
||||
"allenai/Molmo2-4B",
|
||||
Molmo2VideoBackend,
|
||||
None,
|
||||
marks=pytest.mark.skip(
|
||||
reason="Video processor not aligned, investigate later.",
|
||||
),
|
||||
@@ -83,23 +85,44 @@ def test_video_loader_type_doesnt_exist():
|
||||
pytest.param(
|
||||
"zai-org/GLM-4.1V-9B-Thinking",
|
||||
DynamicVideoBackend,
|
||||
None,
|
||||
id="glm4v",
|
||||
),
|
||||
pytest.param(
|
||||
"zai-org/GLM-4.6V-Flash",
|
||||
GLM46VVideoBackend,
|
||||
None,
|
||||
id="glm46v",
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen/Qwen3-VL-4B-Instruct",
|
||||
Qwen3VLVideoBackend,
|
||||
None,
|
||||
id="qwen3vl",
|
||||
),
|
||||
# Qwen2-VL/Qwen2.5-VL ship no ``video_processor_type`` in their
|
||||
# preprocessor config, so resolution relies on the model_type ->
|
||||
# video processor fallback in get_video_processor_cls_name_from_config.
|
||||
# They also ship no default fps/num_frames, so the HF sampler needs an
|
||||
# explicit target rate; pass fps=2 to match the loader default.
|
||||
pytest.param(
|
||||
"Qwen/Qwen2-VL-7B-Instruct",
|
||||
Qwen2VLVideoBackend,
|
||||
{"fps": 2},
|
||||
id="qwen2vl",
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen/Qwen2.5-VL-7B-Instruct",
|
||||
Qwen2VLVideoBackend,
|
||||
{"fps": 2},
|
||||
id="qwen2_5_vl",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_video_processor_from_model_repo(
|
||||
model_repo: str,
|
||||
expected_loader_cls: type,
|
||||
hf_sample_kwargs: dict[str, int | float] | None,
|
||||
):
|
||||
"""Test that a model repo resolves to the correct video loader backend.
|
||||
|
||||
@@ -143,7 +166,7 @@ def test_video_processor_from_model_repo(
|
||||
fps=vllm_meta["fps"],
|
||||
duration=vllm_meta["duration"],
|
||||
)
|
||||
hf_indices = processor.sample_frames(hf_metadata)
|
||||
hf_indices = processor.sample_frames(hf_metadata, **(hf_sample_kwargs or {}))
|
||||
vllm_indices = np.array(vllm_meta["frames_indices"])
|
||||
np.testing.assert_array_equal(
|
||||
hf_indices,
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any, ClassVar, Literal, NamedTuple, cast
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
@@ -653,6 +654,85 @@ class Qwen3VLVideoBackend(VideoBackend):
|
||||
)
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register(
|
||||
"qwen2_vl",
|
||||
video_processor="Qwen2VLVideoProcessor",
|
||||
)
|
||||
class Qwen2VLVideoBackend(VideoBackend):
|
||||
"""Qwen2-VL / Qwen2.5-VL fps-based video backend.
|
||||
|
||||
Ports transformers' ``Qwen2VLVideoProcessor.sample_frames`` (fps mode),
|
||||
shared by Qwen2-VL and Qwen2.5-VL (the latter has no video processor of its
|
||||
own): sample ``total / original_fps * fps`` frames, clamp to
|
||||
``[min_frames, max_frames]`` (4 and 768), floor to a multiple of
|
||||
``temporal_patch_size`` (2), and take indices with the exact
|
||||
``torch.arange(0, total, total / n)`` call so they match HF byte-for-byte.
|
||||
|
||||
``num_frames`` is ignored (fps-driven, like the Qwen3-VL loader). The
|
||||
float32 step can emit an out-of-range tail index (e.g. 451 for a 451-frame
|
||||
clip); it is clamped to the last valid frame.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def compute_frames_index_to_sample(
|
||||
cls,
|
||||
source: VideoSourceMetadata,
|
||||
target: VideoTargetMetadata,
|
||||
**kwargs,
|
||||
) -> list[int]:
|
||||
# Refer to:
|
||||
# https://github.com/huggingface/transformers/blob/v5.7.0/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py#L122-L190
|
||||
total_frames_num = source.total_frames_num
|
||||
original_fps = source.original_fps
|
||||
temporal_patch_size = kwargs.get("temporal_patch_size", 2)
|
||||
min_frames = kwargs.get("min_frames", 4)
|
||||
max_frames = kwargs.get("max_frames", 768)
|
||||
|
||||
# vLLM reports original_fps == 0 for clips with unknown/variable fps
|
||||
# (VFR, malformed, streaming); fail loudly instead of dividing by zero.
|
||||
if original_fps <= 0:
|
||||
raise ValueError(
|
||||
"Qwen2-VL video sampling needs a known source fps, but the "
|
||||
"container reported 0 (variable or unknown frame rate)."
|
||||
)
|
||||
|
||||
max_frames = (
|
||||
math.floor(min(max_frames, total_frames_num) / temporal_patch_size)
|
||||
* temporal_patch_size
|
||||
)
|
||||
n = total_frames_num / original_fps * target.fps
|
||||
n = min(max(n, min_frames), max_frames, total_frames_num)
|
||||
n = math.floor(n / temporal_patch_size) * temporal_patch_size
|
||||
|
||||
# ``torch.arange`` matches transformers' float32 index math exactly
|
||||
# (numpy's float64 diverges by a frame on some inputs); clamp the tail
|
||||
# because that step can emit an index == total_frames_num.
|
||||
indices = torch.arange(0, total_frames_num, total_frames_num / n).int()
|
||||
return torch.clamp(indices, max=total_frames_num - 1).tolist()
|
||||
|
||||
@classmethod
|
||||
def load_bytes(
|
||||
cls,
|
||||
data: bytes,
|
||||
num_frames: int = -1,
|
||||
fps: int = 2,
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal["opencv", "pyav"] = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
data,
|
||||
num_frames=num_frames,
|
||||
fps=fps,
|
||||
max_duration=max_duration,
|
||||
frame_recovery=frame_recovery,
|
||||
backend=backend,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register(
|
||||
"opencv_dynamic",
|
||||
video_processor="Glm4vVideoProcessor",
|
||||
|
||||
@@ -18,6 +18,7 @@ from transformers.audio_utils import AudioInput
|
||||
from transformers.feature_extraction_utils import FeatureExtractionMixin
|
||||
from transformers.image_processing_utils import BaseImageProcessor
|
||||
from transformers.image_utils import ImageInput
|
||||
from transformers.models.auto.video_processing_auto import VIDEO_PROCESSOR_MAPPING_NAMES
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
from transformers.video_processing_utils import BaseVideoProcessor
|
||||
from transformers.video_utils import VideoInput
|
||||
@@ -169,6 +170,15 @@ def get_video_processor_cls_name_from_config(
|
||||
config = get_hf_file_to_dict(file, processor_name, revision=revision)
|
||||
if config and "video_processor_type" in config:
|
||||
return config["video_processor_type"]
|
||||
|
||||
# Some models ship no explicit ``video_processor_type`` in their
|
||||
# preprocessor config. Fall back to transformers' ``model_type`` -> video
|
||||
# processor mapping so these still resolve to their registered loader
|
||||
# instead of the generic opencv fallback. The mapping is ``None`` for a
|
||||
# given type when torchvision is unavailable; callers then use opencv.
|
||||
model_config = get_hf_file_to_dict("config.json", processor_name, revision=revision)
|
||||
if model_config and "model_type" in model_config:
|
||||
return VIDEO_PROCESSOR_MAPPING_NAMES.get(model_config["model_type"])
|
||||
return None
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user