[Multimodal] Automatically select registered video loader for VLM (#44126)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
2026-06-02 09:09:47 +00:00
committed by GitHub
parent e30313220c
commit f8e9c56d15
5 changed files with 188 additions and 9 deletions
+48
View File
@@ -10,8 +10,12 @@ import pytest
from vllm.assets.base import get_vllm_public_assets
from vllm.multimodal.video import (
VIDEO_LOADER_REGISTRY,
DynamicVideoBackend,
Molmo2VideoBackend,
VideoLoader,
get_video_loader_backend_for_processor,
)
from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
from .utils import create_long_gop_video, create_video_from_image
@@ -54,6 +58,50 @@ def test_video_loader_type_doesnt_exist():
VIDEO_LOADER_REGISTRY.load("non_existing_video_loader")
# ============================================================================
# Video Processor → Video Loader Tests (via model repo)
# ============================================================================
@pytest.mark.parametrize(
"model_repo, expected_loader_cls",
[
pytest.param(
"allenai/Molmo2-4B",
Molmo2VideoBackend,
id="molmo2",
),
pytest.param(
"zai-org/GLM-4.1V-9B-Thinking",
DynamicVideoBackend,
id="glm4v",
),
],
)
def test_video_processor_from_model_repo(
model_repo: str,
expected_loader_cls: type,
):
"""Test that a model repo resolves to the correct video loader backend.
The test downloads the preprocessor config from HuggingFace Hub,
extracts the ``video_processor_type`` field, and verifies it maps
to the expected backend and loader class.
"""
video_processor = get_video_processor_cls_name_from_config(model_repo)
assert video_processor is not None, (
f"Model repo {model_repo!r} did not contain a video_processor_type "
f"in its preprocessor config"
)
backend = get_video_loader_backend_for_processor(video_processor)
loader = VIDEO_LOADER_REGISTRY.load(backend)
assert isinstance(loader, expected_loader_cls), (
f"{model_repo!r}: backend={backend!r} loaded "
f"{type(loader)}, expected {expected_loader_cls}"
)
def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
"""
Regression test for handling videos with broken frames.
+19 -2
View File
@@ -58,6 +58,7 @@ from vllm.renderers.embed_utils import (
safe_load_prompt_embeds,
safe_load_prompt_embeds_async,
)
from vllm.transformers_utils.processor import get_video_processor_cls_name
from vllm.utils import random_uuid
from vllm.utils.collection_utils import is_list_of
from vllm.utils.import_utils import LazyLoader
@@ -577,6 +578,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
def mm_processor(self):
return self.mm_registry.create_processor(self.model_config)
@property
def video_processor_name(self) -> str | None:
return get_video_processor_cls_name(self.model_config)
def add(self, modality: ModalityStr, item: _T) -> str | None:
"""
Add a multi-modal item to the current prompt and returns the
@@ -1025,7 +1030,14 @@ class MultiModalContentParser(BaseMultiModalContentParser):
return self.parse_audio(audio_url, uuid)
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
video = self._connector.fetch_video(video_url=video_url) if video_url else None
video = (
self._connector.fetch_video(
video_url=video_url,
video_processor=self._tracker.video_processor_name,
)
if video_url
else None
)
placeholder = self._tracker.add("video", (video, uuid))
self._add_placeholder("video", placeholder)
@@ -1205,7 +1217,12 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None):
video = (
await self._connector.fetch_video_async(video_url) if video_url else None
await self._connector.fetch_video_async(
video_url,
video_processor=self._tracker.video_processor_name,
)
if video_url
else None
)
return video, uuid
+15 -2
View File
@@ -22,6 +22,7 @@ from urllib3.util import Url, parse_url
import vllm.envs as envs
from vllm.connections import HTTPConnection, global_http_connection
from vllm.logger import init_logger
from vllm.multimodal.video import get_video_loader_backend_for_processor
from vllm.utils.registry import ExtensionManager
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
@@ -452,6 +453,7 @@ class MediaConnector:
video_url: str,
*,
image_mode: str = "RGB",
video_processor: str | None = None,
) -> tuple[npt.NDArray, dict[str, Any]]:
"""
Load video from an HTTP or base64 data URL.
@@ -459,7 +461,12 @@ class MediaConnector:
image_io = ImageMediaIO(
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
)
video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
if "video_backend" not in video_io_kwargs and (
video_backend := get_video_loader_backend_for_processor(video_processor)
):
video_io_kwargs["video_backend"] = video_backend
video_io = VideoMediaIO(image_io, **video_io_kwargs)
return self.load_from_url(
video_url,
@@ -472,6 +479,7 @@ class MediaConnector:
video_url: str,
*,
image_mode: str = "RGB",
video_processor: str | None = None,
) -> tuple[npt.NDArray, dict[str, Any]]:
"""
Asynchronously load video from an HTTP or base64 data URL.
@@ -481,7 +489,12 @@ class MediaConnector:
image_io = ImageMediaIO(
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
)
video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
if "video_backend" not in video_io_kwargs and (
video_backend := get_video_loader_backend_for_processor(video_processor)
):
video_io_kwargs["video_backend"] = video_backend
video_io = VideoMediaIO(image_io, **video_io_kwargs)
return await self.load_from_url_async(
video_url,
+68 -5
View File
@@ -28,6 +28,60 @@ except ImportError:
logger = init_logger(__name__)
class VideoLoaderRegistry(ExtensionManager):
def __init__(self) -> None:
super().__init__()
self.processor2backend: dict[str, str] = {}
@staticmethod
def _normalize_registered_video_processors(
video_processor: str | tuple[str, ...] | None,
) -> tuple[str, ...]:
if video_processor is None:
return ()
if isinstance(video_processor, str):
return (video_processor,)
if all(isinstance(processor, str) for processor in video_processor):
return video_processor
raise TypeError(
"video_processor must be a class name or a tuple of class names"
)
def register(
self,
name: str,
*,
video_processor: str | tuple[str, ...] | None = None,
):
processors = self._normalize_registered_video_processors(video_processor)
def wrap(cls_to_register):
self.name2class[name] = cls_to_register
for processor_name in processors:
self.processor2backend[processor_name] = name
return cls_to_register
return wrap
def get_backend_for_video_processor(
self,
video_processor: str | None,
) -> str | None:
if video_processor is None:
return None
return self.processor2backend.get(video_processor)
def get_video_loader_backend_for_processor(
video_processor: str | None,
) -> str | None:
return VIDEO_LOADER_REGISTRY.get_backend_for_video_processor(video_processor)
def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:
num_frames, _, _, channels = frames.shape
new_height, new_width = size
@@ -113,7 +167,7 @@ class VideoLoader:
}
VIDEO_LOADER_REGISTRY = ExtensionManager()
VIDEO_LOADER_REGISTRY = VideoLoaderRegistry()
class OpenCVVideoBackendMixin:
@@ -550,7 +604,10 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
)
@VIDEO_LOADER_REGISTRY.register("opencv_dynamic")
@VIDEO_LOADER_REGISTRY.register(
"opencv_dynamic",
video_processor="Glm4vVideoProcessor",
)
class DynamicVideoBackend(VideoBackend):
"""Duration-aware dynamic-sampling video backend.
@@ -639,8 +696,11 @@ class DynamicVideoBackend(VideoBackend):
)
@VIDEO_LOADER_REGISTRY.register("glm4_6v")
class GLM4_6VVideoBackend(VideoBackend):
@VIDEO_LOADER_REGISTRY.register(
"glmga",
video_processor="GlmgaVideoProcessor",
)
class GLMGAVideoBackend(VideoBackend):
@classmethod
def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata:
# Estimate duration from frame count and fps when the container
@@ -740,7 +800,10 @@ class GLM4_6VVideoBackend(VideoBackend):
return frames, metadata
@VIDEO_LOADER_REGISTRY.register("molmo2")
@VIDEO_LOADER_REGISTRY.register(
"molmo2",
video_processor="Molmo2VideoProcessor",
)
class Molmo2VideoBackend(VideoLoader, OpenCVVideoBackendMixin):
@classmethod
def get_candidate_target_fps(
+38
View File
@@ -161,6 +161,44 @@ def get_processor_cls_name_from_config(
return None
def get_video_processor_cls_name_from_config(
processor_name: str,
revision: str | None = "main",
) -> str | None:
processor_name = convert_model_repo_to_path(processor_name)
config_file = [
"video_preprocessor_config.json",
"preprocessor_config.json",
]
for file in config_file:
config = get_hf_file_to_dict(file, processor_name, revision=revision)
if config and "video_processor_type" in config:
return config["video_processor_type"]
return None
_cached_get_video_processor_cls_name = lru_cache(
get_video_processor_cls_name_from_config
)
def get_video_processor_cls_name(
model_config: "ModelConfig",
) -> str | None:
if is_gguf(model_config.model):
assert not is_gguf(model_config.tokenizer), (
"For multimodal GGUF models, the original tokenizer "
"should be used to correctly load video processor metadata."
)
model = model_config.tokenizer
revision = model_config.tokenizer_revision
else:
model = model_config.model
revision = model_config.revision
return _cached_get_video_processor_cls_name(model, revision=revision)
def get_processor(
processor_name: str,
*args: Any,