From f8e9c56d1587b2eb5599800aa907e091ec9a8182 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Tue, 2 Jun 2026 17:09:47 +0800 Subject: [PATCH] [Multimodal] Automatically select registered video loader for VLM (#44126) Signed-off-by: Isotr0py --- tests/multimodal/test_video.py | 48 ++++++++++++++++++ vllm/entrypoints/chat_utils.py | 21 +++++++- vllm/multimodal/media/connector.py | 17 ++++++- vllm/multimodal/video.py | 73 ++++++++++++++++++++++++++-- vllm/transformers_utils/processor.py | 38 +++++++++++++++ 5 files changed, 188 insertions(+), 9 deletions(-) diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index 7c024052a4..b82bcec9dc 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -10,8 +10,12 @@ import pytest from vllm.assets.base import get_vllm_public_assets from vllm.multimodal.video import ( VIDEO_LOADER_REGISTRY, + DynamicVideoBackend, + Molmo2VideoBackend, VideoLoader, + get_video_loader_backend_for_processor, ) +from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config from .utils import create_long_gop_video, create_video_from_image @@ -54,6 +58,50 @@ def test_video_loader_type_doesnt_exist(): VIDEO_LOADER_REGISTRY.load("non_existing_video_loader") +# ============================================================================ +# Video Processor → Video Loader Tests (via model repo) +# ============================================================================ + + +@pytest.mark.parametrize( + "model_repo, expected_loader_cls", + [ + pytest.param( + "allenai/Molmo2-4B", + Molmo2VideoBackend, + id="molmo2", + ), + pytest.param( + "zai-org/GLM-4.1V-9B-Thinking", + DynamicVideoBackend, + id="glm4v", + ), + ], +) +def test_video_processor_from_model_repo( + model_repo: str, + expected_loader_cls: type, +): + """Test that a model repo resolves to the correct video loader backend. + + The test downloads the preprocessor config from HuggingFace Hub, + extracts the ``video_processor_type`` field, and verifies it maps + to the expected backend and loader class. + """ + video_processor = get_video_processor_cls_name_from_config(model_repo) + assert video_processor is not None, ( + f"Model repo {model_repo!r} did not contain a video_processor_type " + f"in its preprocessor config" + ) + + backend = get_video_loader_backend_for_processor(video_processor) + loader = VIDEO_LOADER_REGISTRY.load(backend) + assert isinstance(loader, expected_loader_cls), ( + f"{model_repo!r}: backend={backend!r} loaded " + f"{type(loader)}, expected {expected_loader_cls}" + ) + + def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch): """ Regression test for handling videos with broken frames. diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 35256bc647..240f26e6c5 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -58,6 +58,7 @@ from vllm.renderers.embed_utils import ( safe_load_prompt_embeds, safe_load_prompt_embeds_async, ) +from vllm.transformers_utils.processor import get_video_processor_cls_name from vllm.utils import random_uuid from vllm.utils.collection_utils import is_list_of from vllm.utils.import_utils import LazyLoader @@ -577,6 +578,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]): def mm_processor(self): return self.mm_registry.create_processor(self.model_config) + @property + def video_processor_name(self) -> str | None: + return get_video_processor_cls_name(self.model_config) + def add(self, modality: ModalityStr, item: _T) -> str | None: """ Add a multi-modal item to the current prompt and returns the @@ -1025,7 +1030,14 @@ class MultiModalContentParser(BaseMultiModalContentParser): return self.parse_audio(audio_url, uuid) def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: - video = self._connector.fetch_video(video_url=video_url) if video_url else None + video = ( + self._connector.fetch_video( + video_url=video_url, + video_processor=self._tracker.video_processor_name, + ) + if video_url + else None + ) placeholder = self._tracker.add("video", (video, uuid)) self._add_placeholder("video", placeholder) @@ -1205,7 +1217,12 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser): async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None): video = ( - await self._connector.fetch_video_async(video_url) if video_url else None + await self._connector.fetch_video_async( + video_url, + video_processor=self._tracker.video_processor_name, + ) + if video_url + else None ) return video, uuid diff --git a/vllm/multimodal/media/connector.py b/vllm/multimodal/media/connector.py index babc4c742a..312239ad3f 100644 --- a/vllm/multimodal/media/connector.py +++ b/vllm/multimodal/media/connector.py @@ -22,6 +22,7 @@ from urllib3.util import Url, parse_url import vllm.envs as envs from vllm.connections import HTTPConnection, global_http_connection from vllm.logger import init_logger +from vllm.multimodal.video import get_video_loader_backend_for_processor from vllm.utils.registry import ExtensionManager from .audio import AudioEmbeddingMediaIO, AudioMediaIO @@ -452,6 +453,7 @@ class MediaConnector: video_url: str, *, image_mode: str = "RGB", + video_processor: str | None = None, ) -> tuple[npt.NDArray, dict[str, Any]]: """ Load video from an HTTP or base64 data URL. @@ -459,7 +461,12 @@ class MediaConnector: image_io = ImageMediaIO( image_mode=image_mode, **self.media_io_kwargs.get("image", {}) ) - video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {})) + video_io_kwargs = dict(self.media_io_kwargs.get("video", {})) + if "video_backend" not in video_io_kwargs and ( + video_backend := get_video_loader_backend_for_processor(video_processor) + ): + video_io_kwargs["video_backend"] = video_backend + video_io = VideoMediaIO(image_io, **video_io_kwargs) return self.load_from_url( video_url, @@ -472,6 +479,7 @@ class MediaConnector: video_url: str, *, image_mode: str = "RGB", + video_processor: str | None = None, ) -> tuple[npt.NDArray, dict[str, Any]]: """ Asynchronously load video from an HTTP or base64 data URL. @@ -481,7 +489,12 @@ class MediaConnector: image_io = ImageMediaIO( image_mode=image_mode, **self.media_io_kwargs.get("image", {}) ) - video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {})) + video_io_kwargs = dict(self.media_io_kwargs.get("video", {})) + if "video_backend" not in video_io_kwargs and ( + video_backend := get_video_loader_backend_for_processor(video_processor) + ): + video_io_kwargs["video_backend"] = video_backend + video_io = VideoMediaIO(image_io, **video_io_kwargs) return await self.load_from_url_async( video_url, diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index df296437fe..1324e79c5c 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -28,6 +28,60 @@ except ImportError: logger = init_logger(__name__) +class VideoLoaderRegistry(ExtensionManager): + def __init__(self) -> None: + super().__init__() + self.processor2backend: dict[str, str] = {} + + @staticmethod + def _normalize_registered_video_processors( + video_processor: str | tuple[str, ...] | None, + ) -> tuple[str, ...]: + if video_processor is None: + return () + + if isinstance(video_processor, str): + return (video_processor,) + + if all(isinstance(processor, str) for processor in video_processor): + return video_processor + + raise TypeError( + "video_processor must be a class name or a tuple of class names" + ) + + def register( + self, + name: str, + *, + video_processor: str | tuple[str, ...] | None = None, + ): + processors = self._normalize_registered_video_processors(video_processor) + + def wrap(cls_to_register): + self.name2class[name] = cls_to_register + for processor_name in processors: + self.processor2backend[processor_name] = name + return cls_to_register + + return wrap + + def get_backend_for_video_processor( + self, + video_processor: str | None, + ) -> str | None: + if video_processor is None: + return None + + return self.processor2backend.get(video_processor) + + +def get_video_loader_backend_for_processor( + video_processor: str | None, +) -> str | None: + return VIDEO_LOADER_REGISTRY.get_backend_for_video_processor(video_processor) + + def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray: num_frames, _, _, channels = frames.shape new_height, new_width = size @@ -113,7 +167,7 @@ class VideoLoader: } -VIDEO_LOADER_REGISTRY = ExtensionManager() +VIDEO_LOADER_REGISTRY = VideoLoaderRegistry() class OpenCVVideoBackendMixin: @@ -550,7 +604,10 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin): ) -@VIDEO_LOADER_REGISTRY.register("opencv_dynamic") +@VIDEO_LOADER_REGISTRY.register( + "opencv_dynamic", + video_processor="Glm4vVideoProcessor", +) class DynamicVideoBackend(VideoBackend): """Duration-aware dynamic-sampling video backend. @@ -639,8 +696,11 @@ class DynamicVideoBackend(VideoBackend): ) -@VIDEO_LOADER_REGISTRY.register("glm4_6v") -class GLM4_6VVideoBackend(VideoBackend): +@VIDEO_LOADER_REGISTRY.register( + "glmga", + video_processor="GlmgaVideoProcessor", +) +class GLMGAVideoBackend(VideoBackend): @classmethod def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata: # Estimate duration from frame count and fps when the container @@ -740,7 +800,10 @@ class GLM4_6VVideoBackend(VideoBackend): return frames, metadata -@VIDEO_LOADER_REGISTRY.register("molmo2") +@VIDEO_LOADER_REGISTRY.register( + "molmo2", + video_processor="Molmo2VideoProcessor", +) class Molmo2VideoBackend(VideoLoader, OpenCVVideoBackendMixin): @classmethod def get_candidate_target_fps( diff --git a/vllm/transformers_utils/processor.py b/vllm/transformers_utils/processor.py index 0e241f6abf..ec01f65d77 100644 --- a/vllm/transformers_utils/processor.py +++ b/vllm/transformers_utils/processor.py @@ -161,6 +161,44 @@ def get_processor_cls_name_from_config( return None +def get_video_processor_cls_name_from_config( + processor_name: str, + revision: str | None = "main", +) -> str | None: + processor_name = convert_model_repo_to_path(processor_name) + config_file = [ + "video_preprocessor_config.json", + "preprocessor_config.json", + ] + for file in config_file: + config = get_hf_file_to_dict(file, processor_name, revision=revision) + if config and "video_processor_type" in config: + return config["video_processor_type"] + return None + + +_cached_get_video_processor_cls_name = lru_cache( + get_video_processor_cls_name_from_config +) + + +def get_video_processor_cls_name( + model_config: "ModelConfig", +) -> str | None: + if is_gguf(model_config.model): + assert not is_gguf(model_config.tokenizer), ( + "For multimodal GGUF models, the original tokenizer " + "should be used to correctly load video processor metadata." + ) + model = model_config.tokenizer + revision = model_config.tokenizer_revision + else: + model = model_config.model + revision = model_config.revision + + return _cached_get_video_processor_cls_name(model, revision=revision) + + def get_processor( processor_name: str, *args: Any,