diff --git a/tests/entrypoints/pooling/classify/test_online_vision.py b/tests/entrypoints/pooling/classify/test_online_vision.py index 2776dc8d80..ce60e01ebe 100644 --- a/tests/entrypoints/pooling/classify/test_online_vision.py +++ b/tests/entrypoints/pooling/classify/test_online_vision.py @@ -25,7 +25,7 @@ def server(): "--runner", "pooling", "--max-model-len", - "5000", + "16384", "--enforce-eager", "--limit-mm-per-prompt", json.dumps({"video": MAXIMUM_VIDEOS}), @@ -143,4 +143,4 @@ def test_chat_video_url_request(server: RemoteOpenAIServer, model_name: str): assert output.model == model_name assert len(output.data) == 1 assert len(output.data[0].probs) == 2 - assert output.usage.prompt_tokens == 4807 + assert output.usage.prompt_tokens == 8993 diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index d9f5413b63..694eb392c4 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -15,6 +15,7 @@ from vllm.multimodal.video import ( DynamicVideoBackend, GLM46VVideoBackend, Molmo2VideoBackend, + Qwen2VLVideoBackend, Qwen3VLVideoBackend, VideoLoader, VideoSourceMetadata, @@ -70,11 +71,12 @@ def test_video_loader_type_doesnt_exist(): @pytest.mark.parametrize( - "model_repo, expected_loader_cls", + "model_repo, expected_loader_cls, hf_sample_kwargs", [ pytest.param( "allenai/Molmo2-4B", Molmo2VideoBackend, + None, marks=pytest.mark.skip( reason="Video processor not aligned, investigate later.", ), @@ -83,23 +85,44 @@ def test_video_loader_type_doesnt_exist(): pytest.param( "zai-org/GLM-4.1V-9B-Thinking", DynamicVideoBackend, + None, id="glm4v", ), pytest.param( "zai-org/GLM-4.6V-Flash", GLM46VVideoBackend, + None, id="glm46v", ), pytest.param( "Qwen/Qwen3-VL-4B-Instruct", Qwen3VLVideoBackend, + None, id="qwen3vl", ), + # Qwen2-VL/Qwen2.5-VL ship no ``video_processor_type`` in their + # preprocessor config, so resolution relies on the model_type -> + # video processor fallback in get_video_processor_cls_name_from_config. + # They also ship no default fps/num_frames, so the HF sampler needs an + # explicit target rate; pass fps=2 to match the loader default. + pytest.param( + "Qwen/Qwen2-VL-7B-Instruct", + Qwen2VLVideoBackend, + {"fps": 2}, + id="qwen2vl", + ), + pytest.param( + "Qwen/Qwen2.5-VL-7B-Instruct", + Qwen2VLVideoBackend, + {"fps": 2}, + id="qwen2_5_vl", + ), ], ) def test_video_processor_from_model_repo( model_repo: str, expected_loader_cls: type, + hf_sample_kwargs: dict[str, int | float] | None, ): """Test that a model repo resolves to the correct video loader backend. @@ -143,7 +166,7 @@ def test_video_processor_from_model_repo( fps=vllm_meta["fps"], duration=vllm_meta["duration"], ) - hf_indices = processor.sample_frames(hf_metadata) + hf_indices = processor.sample_frames(hf_metadata, **(hf_sample_kwargs or {})) vllm_indices = np.array(vllm_meta["frames_indices"]) np.testing.assert_array_equal( hf_indices, diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index bb74f073fb..4a82dd24e7 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -7,6 +7,7 @@ from typing import Any, ClassVar, Literal, NamedTuple, cast import numpy as np import numpy.typing as npt +import torch from vllm.logger import init_logger from vllm.utils.import_utils import PlaceholderModule @@ -653,6 +654,85 @@ class Qwen3VLVideoBackend(VideoBackend): ) +@VIDEO_LOADER_REGISTRY.register( + "qwen2_vl", + video_processor="Qwen2VLVideoProcessor", +) +class Qwen2VLVideoBackend(VideoBackend): + """Qwen2-VL / Qwen2.5-VL fps-based video backend. + + Ports transformers' ``Qwen2VLVideoProcessor.sample_frames`` (fps mode), + shared by Qwen2-VL and Qwen2.5-VL (the latter has no video processor of its + own): sample ``total / original_fps * fps`` frames, clamp to + ``[min_frames, max_frames]`` (4 and 768), floor to a multiple of + ``temporal_patch_size`` (2), and take indices with the exact + ``torch.arange(0, total, total / n)`` call so they match HF byte-for-byte. + + ``num_frames`` is ignored (fps-driven, like the Qwen3-VL loader). The + float32 step can emit an out-of-range tail index (e.g. 451 for a 451-frame + clip); it is clamped to the last valid frame. + """ + + @classmethod + def compute_frames_index_to_sample( + cls, + source: VideoSourceMetadata, + target: VideoTargetMetadata, + **kwargs, + ) -> list[int]: + # Refer to: + # https://github.com/huggingface/transformers/blob/v5.7.0/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py#L122-L190 + total_frames_num = source.total_frames_num + original_fps = source.original_fps + temporal_patch_size = kwargs.get("temporal_patch_size", 2) + min_frames = kwargs.get("min_frames", 4) + max_frames = kwargs.get("max_frames", 768) + + # vLLM reports original_fps == 0 for clips with unknown/variable fps + # (VFR, malformed, streaming); fail loudly instead of dividing by zero. + if original_fps <= 0: + raise ValueError( + "Qwen2-VL video sampling needs a known source fps, but the " + "container reported 0 (variable or unknown frame rate)." + ) + + max_frames = ( + math.floor(min(max_frames, total_frames_num) / temporal_patch_size) + * temporal_patch_size + ) + n = total_frames_num / original_fps * target.fps + n = min(max(n, min_frames), max_frames, total_frames_num) + n = math.floor(n / temporal_patch_size) * temporal_patch_size + + # ``torch.arange`` matches transformers' float32 index math exactly + # (numpy's float64 diverges by a frame on some inputs); clamp the tail + # because that step can emit an index == total_frames_num. + indices = torch.arange(0, total_frames_num, total_frames_num / n).int() + return torch.clamp(indices, max=total_frames_num - 1).tolist() + + @classmethod + def load_bytes( + cls, + data: bytes, + num_frames: int = -1, + fps: int = 2, + max_duration: int = 300, + frame_recovery: bool = False, + *, + backend: Literal["opencv", "pyav"] = "opencv", + **kwargs, + ) -> tuple[npt.NDArray, dict[str, Any]]: + return super().load_bytes( + data, + num_frames=num_frames, + fps=fps, + max_duration=max_duration, + frame_recovery=frame_recovery, + backend=backend, + **kwargs, + ) + + @VIDEO_LOADER_REGISTRY.register( "opencv_dynamic", video_processor="Glm4vVideoProcessor", diff --git a/vllm/transformers_utils/processor.py b/vllm/transformers_utils/processor.py index 462a6582ed..fa4c558a73 100644 --- a/vllm/transformers_utils/processor.py +++ b/vllm/transformers_utils/processor.py @@ -18,6 +18,7 @@ from transformers.audio_utils import AudioInput from transformers.feature_extraction_utils import FeatureExtractionMixin from transformers.image_processing_utils import BaseImageProcessor from transformers.image_utils import ImageInput +from transformers.models.auto.video_processing_auto import VIDEO_PROCESSOR_MAPPING_NAMES from transformers.processing_utils import ProcessorMixin from transformers.video_processing_utils import BaseVideoProcessor from transformers.video_utils import VideoInput @@ -169,6 +170,15 @@ def get_video_processor_cls_name_from_config( config = get_hf_file_to_dict(file, processor_name, revision=revision) if config and "video_processor_type" in config: return config["video_processor_type"] + + # Some models ship no explicit ``video_processor_type`` in their + # preprocessor config. Fall back to transformers' ``model_type`` -> video + # processor mapping so these still resolve to their registered loader + # instead of the generic opencv fallback. The mapping is ``None`` for a + # given type when torchvision is unavailable; callers then use opencv. + model_config = get_hf_file_to_dict("config.json", processor_name, revision=revision) + if model_config and "model_type" in model_config: + return VIDEO_PROCESSOR_MAPPING_NAMES.get(model_config["model_type"]) return None