feat: expose media_io_kwargs at runtime (#34778)

Signed-off-by: Alexandre Milesi <milesial@users.noreply.github.com>
This commit is contained in:
milesial
2026-03-07 04:27:04 +00:00
committed by GitHub
parent 58928475e4
commit 755356b3d1
20 changed files with 298 additions and 16 deletions
+22 -9
View File
@@ -462,10 +462,15 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
maximum per prompt.
"""
def __init__(self, model_config: ModelConfig):
def __init__(
self,
model_config: ModelConfig,
media_io_kwargs: dict[str, dict[str, Any]] | None = None,
):
super().__init__()
self._model_config = model_config
self._media_io_kwargs = media_io_kwargs
self._items_by_modality = defaultdict[str, list[_T]](list)
# Track original modality for each vision_chunk item (image or video)
@@ -487,6 +492,14 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
model_cls = get_model_cls(self.model_config)
return cast(type[SupportsMultiModal], model_cls)
@property
def media_io_kwargs(self) -> dict[str, dict[str, Any]] | None:
return self._media_io_kwargs or (
self._model_config.multimodal_config.media_io_kwargs
if self._model_config.multimodal_config
else None
)
@property
def allowed_local_media_path(self):
return self._model_config.allowed_local_media_path
@@ -769,12 +782,10 @@ class MultiModalContentParser(BaseMultiModalContentParser):
super().__init__()
self._tracker = tracker
multimodal_config = self._tracker.model_config.multimodal_config
media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)
self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
envs.VLLM_MEDIA_CONNECTOR,
media_io_kwargs=media_io_kwargs,
media_io_kwargs=tracker.media_io_kwargs,
allowed_local_media_path=tracker.allowed_local_media_path,
allowed_media_domains=tracker.allowed_media_domains,
)
@@ -881,11 +892,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
super().__init__()
self._tracker = tracker
multimodal_config = self._tracker.model_config.multimodal_config
media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)
self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
envs.VLLM_MEDIA_CONNECTOR,
media_io_kwargs=media_io_kwargs,
media_io_kwargs=tracker.media_io_kwargs,
allowed_local_media_path=tracker.allowed_local_media_path,
allowed_media_domains=tracker.allowed_media_domains,
)
@@ -1530,13 +1539,14 @@ def parse_chat_messages(
messages: list[ChatCompletionMessageParam],
model_config: ModelConfig,
content_format: ChatTemplateContentFormat,
media_io_kwargs: dict[str, dict[str, Any]] | None = None,
) -> tuple[
list[ConversationMessage],
MultiModalDataDict | None,
MultiModalUUIDDict | None,
]:
conversation: list[ConversationMessage] = []
mm_tracker = MultiModalItemTracker(model_config)
mm_tracker = MultiModalItemTracker(model_config, media_io_kwargs=media_io_kwargs)
for msg in messages:
sub_messages = _parse_chat_message_content(
@@ -1563,13 +1573,16 @@ async def parse_chat_messages_async(
messages: list[ChatCompletionMessageParam],
model_config: ModelConfig,
content_format: ChatTemplateContentFormat,
media_io_kwargs: dict[str, dict[str, Any]] | None = None,
) -> tuple[
list[ConversationMessage],
MultiModalDataDict | None,
MultiModalUUIDDict | None,
]:
conversation: list[ConversationMessage] = []
mm_tracker = AsyncMultiModalItemTracker(model_config)
mm_tracker = AsyncMultiModalItemTracker(
model_config, media_io_kwargs=media_io_kwargs
)
for msg in messages:
sub_messages = _parse_chat_message_content(