[Multimodal] Automatically select registered video loader for VLM (#44126)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -10,8 +10,12 @@ import pytest
|
||||
from vllm.assets.base import get_vllm_public_assets
|
||||
from vllm.multimodal.video import (
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
DynamicVideoBackend,
|
||||
Molmo2VideoBackend,
|
||||
VideoLoader,
|
||||
get_video_loader_backend_for_processor,
|
||||
)
|
||||
from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
|
||||
|
||||
from .utils import create_long_gop_video, create_video_from_image
|
||||
|
||||
@@ -54,6 +58,50 @@ def test_video_loader_type_doesnt_exist():
|
||||
VIDEO_LOADER_REGISTRY.load("non_existing_video_loader")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Video Processor → Video Loader Tests (via model repo)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_repo, expected_loader_cls",
|
||||
[
|
||||
pytest.param(
|
||||
"allenai/Molmo2-4B",
|
||||
Molmo2VideoBackend,
|
||||
id="molmo2",
|
||||
),
|
||||
pytest.param(
|
||||
"zai-org/GLM-4.1V-9B-Thinking",
|
||||
DynamicVideoBackend,
|
||||
id="glm4v",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_video_processor_from_model_repo(
|
||||
model_repo: str,
|
||||
expected_loader_cls: type,
|
||||
):
|
||||
"""Test that a model repo resolves to the correct video loader backend.
|
||||
|
||||
The test downloads the preprocessor config from HuggingFace Hub,
|
||||
extracts the ``video_processor_type`` field, and verifies it maps
|
||||
to the expected backend and loader class.
|
||||
"""
|
||||
video_processor = get_video_processor_cls_name_from_config(model_repo)
|
||||
assert video_processor is not None, (
|
||||
f"Model repo {model_repo!r} did not contain a video_processor_type "
|
||||
f"in its preprocessor config"
|
||||
)
|
||||
|
||||
backend = get_video_loader_backend_for_processor(video_processor)
|
||||
loader = VIDEO_LOADER_REGISTRY.load(backend)
|
||||
assert isinstance(loader, expected_loader_cls), (
|
||||
f"{model_repo!r}: backend={backend!r} loaded "
|
||||
f"{type(loader)}, expected {expected_loader_cls}"
|
||||
)
|
||||
|
||||
|
||||
def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
Regression test for handling videos with broken frames.
|
||||
|
||||
@@ -58,6 +58,7 @@ from vllm.renderers.embed_utils import (
|
||||
safe_load_prompt_embeds,
|
||||
safe_load_prompt_embeds_async,
|
||||
)
|
||||
from vllm.transformers_utils.processor import get_video_processor_cls_name
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
@@ -577,6 +578,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
def mm_processor(self):
|
||||
return self.mm_registry.create_processor(self.model_config)
|
||||
|
||||
@property
|
||||
def video_processor_name(self) -> str | None:
|
||||
return get_video_processor_cls_name(self.model_config)
|
||||
|
||||
def add(self, modality: ModalityStr, item: _T) -> str | None:
|
||||
"""
|
||||
Add a multi-modal item to the current prompt and returns the
|
||||
@@ -1025,7 +1030,14 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
return self.parse_audio(audio_url, uuid)
|
||||
|
||||
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
|
||||
video = self._connector.fetch_video(video_url=video_url) if video_url else None
|
||||
video = (
|
||||
self._connector.fetch_video(
|
||||
video_url=video_url,
|
||||
video_processor=self._tracker.video_processor_name,
|
||||
)
|
||||
if video_url
|
||||
else None
|
||||
)
|
||||
|
||||
placeholder = self._tracker.add("video", (video, uuid))
|
||||
self._add_placeholder("video", placeholder)
|
||||
@@ -1205,7 +1217,12 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None):
|
||||
video = (
|
||||
await self._connector.fetch_video_async(video_url) if video_url else None
|
||||
await self._connector.fetch_video_async(
|
||||
video_url,
|
||||
video_processor=self._tracker.video_processor_name,
|
||||
)
|
||||
if video_url
|
||||
else None
|
||||
)
|
||||
return video, uuid
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ from urllib3.util import Url, parse_url
|
||||
import vllm.envs as envs
|
||||
from vllm.connections import HTTPConnection, global_http_connection
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal.video import get_video_loader_backend_for_processor
|
||||
from vllm.utils.registry import ExtensionManager
|
||||
|
||||
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
|
||||
@@ -452,6 +453,7 @@ class MediaConnector:
|
||||
video_url: str,
|
||||
*,
|
||||
image_mode: str = "RGB",
|
||||
video_processor: str | None = None,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
"""
|
||||
Load video from an HTTP or base64 data URL.
|
||||
@@ -459,7 +461,12 @@ class MediaConnector:
|
||||
image_io = ImageMediaIO(
|
||||
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
|
||||
)
|
||||
video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
|
||||
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
|
||||
if "video_backend" not in video_io_kwargs and (
|
||||
video_backend := get_video_loader_backend_for_processor(video_processor)
|
||||
):
|
||||
video_io_kwargs["video_backend"] = video_backend
|
||||
video_io = VideoMediaIO(image_io, **video_io_kwargs)
|
||||
|
||||
return self.load_from_url(
|
||||
video_url,
|
||||
@@ -472,6 +479,7 @@ class MediaConnector:
|
||||
video_url: str,
|
||||
*,
|
||||
image_mode: str = "RGB",
|
||||
video_processor: str | None = None,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
"""
|
||||
Asynchronously load video from an HTTP or base64 data URL.
|
||||
@@ -481,7 +489,12 @@ class MediaConnector:
|
||||
image_io = ImageMediaIO(
|
||||
image_mode=image_mode, **self.media_io_kwargs.get("image", {})
|
||||
)
|
||||
video_io = VideoMediaIO(image_io, **self.media_io_kwargs.get("video", {}))
|
||||
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
|
||||
if "video_backend" not in video_io_kwargs and (
|
||||
video_backend := get_video_loader_backend_for_processor(video_processor)
|
||||
):
|
||||
video_io_kwargs["video_backend"] = video_backend
|
||||
video_io = VideoMediaIO(image_io, **video_io_kwargs)
|
||||
|
||||
return await self.load_from_url_async(
|
||||
video_url,
|
||||
|
||||
@@ -28,6 +28,60 @@ except ImportError:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class VideoLoaderRegistry(ExtensionManager):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.processor2backend: dict[str, str] = {}
|
||||
|
||||
@staticmethod
|
||||
def _normalize_registered_video_processors(
|
||||
video_processor: str | tuple[str, ...] | None,
|
||||
) -> tuple[str, ...]:
|
||||
if video_processor is None:
|
||||
return ()
|
||||
|
||||
if isinstance(video_processor, str):
|
||||
return (video_processor,)
|
||||
|
||||
if all(isinstance(processor, str) for processor in video_processor):
|
||||
return video_processor
|
||||
|
||||
raise TypeError(
|
||||
"video_processor must be a class name or a tuple of class names"
|
||||
)
|
||||
|
||||
def register(
|
||||
self,
|
||||
name: str,
|
||||
*,
|
||||
video_processor: str | tuple[str, ...] | None = None,
|
||||
):
|
||||
processors = self._normalize_registered_video_processors(video_processor)
|
||||
|
||||
def wrap(cls_to_register):
|
||||
self.name2class[name] = cls_to_register
|
||||
for processor_name in processors:
|
||||
self.processor2backend[processor_name] = name
|
||||
return cls_to_register
|
||||
|
||||
return wrap
|
||||
|
||||
def get_backend_for_video_processor(
|
||||
self,
|
||||
video_processor: str | None,
|
||||
) -> str | None:
|
||||
if video_processor is None:
|
||||
return None
|
||||
|
||||
return self.processor2backend.get(video_processor)
|
||||
|
||||
|
||||
def get_video_loader_backend_for_processor(
|
||||
video_processor: str | None,
|
||||
) -> str | None:
|
||||
return VIDEO_LOADER_REGISTRY.get_backend_for_video_processor(video_processor)
|
||||
|
||||
|
||||
def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:
|
||||
num_frames, _, _, channels = frames.shape
|
||||
new_height, new_width = size
|
||||
@@ -113,7 +167,7 @@ class VideoLoader:
|
||||
}
|
||||
|
||||
|
||||
VIDEO_LOADER_REGISTRY = ExtensionManager()
|
||||
VIDEO_LOADER_REGISTRY = VideoLoaderRegistry()
|
||||
|
||||
|
||||
class OpenCVVideoBackendMixin:
|
||||
@@ -550,7 +604,10 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
|
||||
)
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register("opencv_dynamic")
|
||||
@VIDEO_LOADER_REGISTRY.register(
|
||||
"opencv_dynamic",
|
||||
video_processor="Glm4vVideoProcessor",
|
||||
)
|
||||
class DynamicVideoBackend(VideoBackend):
|
||||
"""Duration-aware dynamic-sampling video backend.
|
||||
|
||||
@@ -639,8 +696,11 @@ class DynamicVideoBackend(VideoBackend):
|
||||
)
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register("glm4_6v")
|
||||
class GLM4_6VVideoBackend(VideoBackend):
|
||||
@VIDEO_LOADER_REGISTRY.register(
|
||||
"glmga",
|
||||
video_processor="GlmgaVideoProcessor",
|
||||
)
|
||||
class GLMGAVideoBackend(VideoBackend):
|
||||
@classmethod
|
||||
def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata:
|
||||
# Estimate duration from frame count and fps when the container
|
||||
@@ -740,7 +800,10 @@ class GLM4_6VVideoBackend(VideoBackend):
|
||||
return frames, metadata
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register("molmo2")
|
||||
@VIDEO_LOADER_REGISTRY.register(
|
||||
"molmo2",
|
||||
video_processor="Molmo2VideoProcessor",
|
||||
)
|
||||
class Molmo2VideoBackend(VideoLoader, OpenCVVideoBackendMixin):
|
||||
@classmethod
|
||||
def get_candidate_target_fps(
|
||||
|
||||
@@ -161,6 +161,44 @@ def get_processor_cls_name_from_config(
|
||||
return None
|
||||
|
||||
|
||||
def get_video_processor_cls_name_from_config(
|
||||
processor_name: str,
|
||||
revision: str | None = "main",
|
||||
) -> str | None:
|
||||
processor_name = convert_model_repo_to_path(processor_name)
|
||||
config_file = [
|
||||
"video_preprocessor_config.json",
|
||||
"preprocessor_config.json",
|
||||
]
|
||||
for file in config_file:
|
||||
config = get_hf_file_to_dict(file, processor_name, revision=revision)
|
||||
if config and "video_processor_type" in config:
|
||||
return config["video_processor_type"]
|
||||
return None
|
||||
|
||||
|
||||
_cached_get_video_processor_cls_name = lru_cache(
|
||||
get_video_processor_cls_name_from_config
|
||||
)
|
||||
|
||||
|
||||
def get_video_processor_cls_name(
|
||||
model_config: "ModelConfig",
|
||||
) -> str | None:
|
||||
if is_gguf(model_config.model):
|
||||
assert not is_gguf(model_config.tokenizer), (
|
||||
"For multimodal GGUF models, the original tokenizer "
|
||||
"should be used to correctly load video processor metadata."
|
||||
)
|
||||
model = model_config.tokenizer
|
||||
revision = model_config.tokenizer_revision
|
||||
else:
|
||||
model = model_config.model
|
||||
revision = model_config.revision
|
||||
|
||||
return _cached_get_video_processor_cls_name(model, revision=revision)
|
||||
|
||||
|
||||
def get_processor(
|
||||
processor_name: str,
|
||||
*args: Any,
|
||||
|
||||
Reference in New Issue
Block a user