[Multimodal] Automatically select registered video loader for VLM (#44126)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
2026-06-02 09:09:47 +00:00
committed by GitHub
parent e30313220c
commit f8e9c56d15
5 changed files with 188 additions and 9 deletions
+19 -2
View File
@@ -58,6 +58,7 @@ from vllm.renderers.embed_utils import (
safe_load_prompt_embeds,
safe_load_prompt_embeds_async,
)
from vllm.transformers_utils.processor import get_video_processor_cls_name
from vllm.utils import random_uuid
from vllm.utils.collection_utils import is_list_of
from vllm.utils.import_utils import LazyLoader
@@ -577,6 +578,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
def mm_processor(self):
return self.mm_registry.create_processor(self.model_config)
@property
def video_processor_name(self) -> str | None:
return get_video_processor_cls_name(self.model_config)
def add(self, modality: ModalityStr, item: _T) -> str | None:
"""
Add a multi-modal item to the current prompt and returns the
@@ -1025,7 +1030,14 @@ class MultiModalContentParser(BaseMultiModalContentParser):
return self.parse_audio(audio_url, uuid)
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
video = self._connector.fetch_video(video_url=video_url) if video_url else None
video = (
self._connector.fetch_video(
video_url=video_url,
video_processor=self._tracker.video_processor_name,
)
if video_url
else None
)
placeholder = self._tracker.add("video", (video, uuid))
self._add_placeholder("video", placeholder)
@@ -1205,7 +1217,12 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None):
video = (
await self._connector.fetch_video_async(video_url) if video_url else None
await self._connector.fetch_video_async(
video_url,
video_processor=self._tracker.video_processor_name,
)
if video_url
else None
)
return video, uuid