From e04a30a77cb17d5a9753c53c51ee30bba887053c Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Thu, 30 Jul 2026 01:56:37 -0500 Subject: [PATCH] [Frontend] Lazily initialize chat media connectors (#49914) Signed-off-by: Andreas Karatzas --- .../entrypoints/unit_tests/test_chat_utils.py | 24 +++++++++++++ vllm/entrypoints/chat_utils.py | 36 +++++++++++-------- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/tests/entrypoints/unit_tests/test_chat_utils.py b/tests/entrypoints/unit_tests/test_chat_utils.py index b5a35c2cb2..fe190a2f18 100644 --- a/tests/entrypoints/unit_tests/test_chat_utils.py +++ b/tests/entrypoints/unit_tests/test_chat_utils.py @@ -15,6 +15,7 @@ from vllm.assets.image import ImageAsset from vllm.assets.video import VideoAsset from vllm.config import ModelConfig from vllm.entrypoints.chat_utils import ( + MEDIA_CONNECTOR_REGISTRY, AsyncMultiModalItemTracker, ConversationMessage, _postprocess_messages, @@ -709,6 +710,29 @@ def test_parse_chat_messages_empty_system( ] +@pytest.mark.asyncio +async def test_text_only_chat_does_not_initialize_media_connector( + mistral_model_config, + monkeypatch, +): + load_connector = MagicMock() + monkeypatch.setattr(MEDIA_CONNECTOR_REGISTRY, "load", load_connector) + messages = [{"role": "user", "content": "Who are you?"}] + + parse_chat_messages( + messages, + mistral_model_config, + content_format="string", + ) + await parse_chat_messages_async( + messages, + mistral_model_config, + content_format="string", + ) + + load_connector.assert_not_called() + + @pytest.mark.asyncio async def test_parse_chat_messages_single_image_async( phi3v_model_config, diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 171f0c7e36..f2f8498849 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -945,16 +945,19 @@ class MultiModalContentParser(BaseMultiModalContentParser): super().__init__() self._tracker = tracker - - self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load( - envs.VLLM_MEDIA_CONNECTOR, - media_io_kwargs=tracker.media_io_kwargs, - allowed_local_media_path=tracker.allowed_local_media_path, - allowed_media_domains=tracker.allowed_media_domains, - ) - self._mm_processor_kwargs = mm_processor_kwargs + @cached_property + def _connector(self) -> MediaConnector: + # Connector setup may probe VLLM_MEDIA_CACHE. Defer it until a request + # actually contains media so text-only parsing never blocks on that I/O. + return MEDIA_CONNECTOR_REGISTRY.load( + envs.VLLM_MEDIA_CONNECTOR, + media_io_kwargs=self._tracker.media_io_kwargs, + allowed_local_media_path=self._tracker.allowed_local_media_path, + allowed_media_domains=self._tracker.allowed_media_domains, + ) + @property def model_config(self) -> ModelConfig: return self._tracker.model_config @@ -1097,14 +1100,19 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser): super().__init__() self._tracker = tracker - self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load( - envs.VLLM_MEDIA_CONNECTOR, - media_io_kwargs=tracker.media_io_kwargs, - allowed_local_media_path=tracker.allowed_local_media_path, - allowed_media_domains=tracker.allowed_media_domains, - ) self._mm_processor_kwargs: dict[str, Any] | None = mm_processor_kwargs + @cached_property + def _connector(self) -> MediaConnector: + # Connector setup may probe VLLM_MEDIA_CACHE. Defer it until a request + # actually contains media so text-only parsing never blocks on that I/O. + return MEDIA_CONNECTOR_REGISTRY.load( + envs.VLLM_MEDIA_CONNECTOR, + media_io_kwargs=self._tracker.media_io_kwargs, + allowed_local_media_path=self._tracker.allowed_local_media_path, + allowed_media_domains=self._tracker.allowed_media_domains, + ) + @property def model_config(self) -> ModelConfig: return self._tracker.model_config