[Multimodal] Automatically select registered video loader for VLM (#44126)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
2026-06-02 09:09:47 +00:00
committed by GitHub
parent e30313220c
commit f8e9c56d15
5 changed files with 188 additions and 9 deletions
+19 -2
View File
@@ -58,6 +58,7 @@ from vllm.renderers.embed_utils import (
safe_load_prompt_embeds,
safe_load_prompt_embeds_async,
)
from vllm.transformers_utils.processor import get_video_processor_cls_name
from vllm.utils import random_uuid
from vllm.utils.collection_utils import is_list_of
from vllm.utils.import_utils import LazyLoader
@@ -577,6 +578,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
def mm_processor(self):
return self.mm_registry.create_processor(self.model_config)
@property
def video_processor_name(self) -> str | None:
return get_video_processor_cls_name(self.model_config)
def add(self, modality: ModalityStr, item: _T) -> str | None:
"""
Add a multi-modal item to the current prompt and returns the
@@ -1025,7 +1030,14 @@ class MultiModalContentParser(BaseMultiModalContentParser):
return self.parse_audio(audio_url, uuid)
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
video = self._connector.fetch_video(video_url=video_url) if video_url else None
video = (
self._connector.fetch_video(
video_url=video_url,
video_processor=self._tracker.video_processor_name,
)
if video_url
else None
)
placeholder = self._tracker.add("video", (video, uuid))
self._add_placeholder("video", placeholder)
@@ -1205,7 +1217,12 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None):
video = (
await self._connector.fetch_video_async(video_url) if video_url else None
await self._connector.fetch_video_async(
video_url,
video_processor=self._tracker.video_processor_name,
)
if video_url
else None
)
return video, uuid
+15 -2
View File
@@ -22,6 +22,7 @@ from urllib3.util import Url, parse_url
import vllm.envs as envs
from vllm.connections import HTTPConnection, global_http_connection
from vllm.logger import init_logger
from vllm.multimodal.video import get_video_loader_backend_for_processor
from vllm.utils.registry import ExtensionManager
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
@@ -452,6 +453,7 @@ class MediaConnector:
video_url: str,
*,
image_mode: str = "RGB",
video_processor: str | None = None,
) -> tuple[npt.NDArray, dict[str, Any]]:
"""
Load video from an HTTP or base64 data URL.
@@ -459,7 +461,12 @@ class MediaConnector:
image_io = ImageMediaIO(
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
)
video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
if "video_backend" not in video_io_kwargs and (
video_backend := get_video_loader_backend_for_processor(video_processor)
):
video_io_kwargs["video_backend"] = video_backend
video_io = VideoMediaIO(image_io, **video_io_kwargs)
return self.load_from_url(
video_url,
@@ -472,6 +479,7 @@ class MediaConnector:
video_url: str,
*,
image_mode: str = "RGB",
video_processor: str | None = None,
) -> tuple[npt.NDArray, dict[str, Any]]:
"""
Asynchronously load video from an HTTP or base64 data URL.
@@ -481,7 +489,12 @@ class MediaConnector:
image_io = ImageMediaIO(
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
)
video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
if "video_backend" not in video_io_kwargs and (
video_backend := get_video_loader_backend_for_processor(video_processor)
):
video_io_kwargs["video_backend"] = video_backend
video_io = VideoMediaIO(image_io, **video_io_kwargs)
return await self.load_from_url_async(
video_url,
+68 -5
View File
@@ -28,6 +28,60 @@ except ImportError:
logger = init_logger(__name__)
class VideoLoaderRegistry(ExtensionManager):
def __init__(self) -> None:
super().__init__()
self.processor2backend: dict[str, str] = {}
@staticmethod
def _normalize_registered_video_processors(
video_processor: str | tuple[str, ...] | None,
) -> tuple[str, ...]:
if video_processor is None:
return ()
if isinstance(video_processor, str):
return (video_processor,)
if all(isinstance(processor, str) for processor in video_processor):
return video_processor
raise TypeError(
"video_processor must be a class name or a tuple of class names"
)
def register(
self,
name: str,
*,
video_processor: str | tuple[str, ...] | None = None,
):
processors = self._normalize_registered_video_processors(video_processor)
def wrap(cls_to_register):
self.name2class[name] = cls_to_register
for processor_name in processors:
self.processor2backend[processor_name] = name
return cls_to_register
return wrap
def get_backend_for_video_processor(
self,
video_processor: str | None,
) -> str | None:
if video_processor is None:
return None
return self.processor2backend.get(video_processor)
def get_video_loader_backend_for_processor(
video_processor: str | None,
) -> str | None:
return VIDEO_LOADER_REGISTRY.get_backend_for_video_processor(video_processor)
def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:
num_frames, _, _, channels = frames.shape
new_height, new_width = size
@@ -113,7 +167,7 @@ class VideoLoader:
}
VIDEO_LOADER_REGISTRY = ExtensionManager()
VIDEO_LOADER_REGISTRY = VideoLoaderRegistry()
class OpenCVVideoBackendMixin:
@@ -550,7 +604,10 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
)
@VIDEO_LOADER_REGISTRY.register("opencv_dynamic")
@VIDEO_LOADER_REGISTRY.register(
"opencv_dynamic",
video_processor="Glm4vVideoProcessor",
)
class DynamicVideoBackend(VideoBackend):
"""Duration-aware dynamic-sampling video backend.
@@ -639,8 +696,11 @@ class DynamicVideoBackend(VideoBackend):
)
@VIDEO_LOADER_REGISTRY.register("glm4_6v")
class GLM4_6VVideoBackend(VideoBackend):
@VIDEO_LOADER_REGISTRY.register(
"glmga",
video_processor="GlmgaVideoProcessor",
)
class GLMGAVideoBackend(VideoBackend):
@classmethod
def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata:
# Estimate duration from frame count and fps when the container
@@ -740,7 +800,10 @@ class GLM4_6VVideoBackend(VideoBackend):
return frames, metadata
@VIDEO_LOADER_REGISTRY.register("molmo2")
@VIDEO_LOADER_REGISTRY.register(
"molmo2",
video_processor="Molmo2VideoProcessor",
)
class Molmo2VideoBackend(VideoLoader, OpenCVVideoBackendMixin):
@classmethod
def get_candidate_target_fps(
+38
View File
@@ -161,6 +161,44 @@ def get_processor_cls_name_from_config(
return None
def get_video_processor_cls_name_from_config(
processor_name: str,
revision: str | None = "main",
) -> str | None:
processor_name = convert_model_repo_to_path(processor_name)
config_file = [
"video_preprocessor_config.json",
"preprocessor_config.json",
]
for file in config_file:
config = get_hf_file_to_dict(file, processor_name, revision=revision)
if config and "video_processor_type" in config:
return config["video_processor_type"]
return None
_cached_get_video_processor_cls_name = lru_cache(
get_video_processor_cls_name_from_config
)
def get_video_processor_cls_name(
model_config: "ModelConfig",
) -> str | None:
if is_gguf(model_config.model):
assert not is_gguf(model_config.tokenizer), (
"For multimodal GGUF models, the original tokenizer "
"should be used to correctly load video processor metadata."
)
model = model_config.tokenizer
revision = model_config.tokenizer_revision
else:
model = model_config.model
revision = model_config.revision
return _cached_get_video_processor_cls_name(model, revision=revision)
def get_processor(
processor_name: str,
*args: Any,