feat: expose media_io_kwargs at runtime (#34778)
Signed-off-by: Alexandre Milesi <milesial@users.noreply.github.com>
This commit is contained in:
@@ -462,10 +462,15 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
maximum per prompt.
|
||||
"""
|
||||
|
||||
def __init__(self, model_config: ModelConfig):
|
||||
def __init__(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
media_io_kwargs: dict[str, dict[str, Any]] | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self._model_config = model_config
|
||||
self._media_io_kwargs = media_io_kwargs
|
||||
|
||||
self._items_by_modality = defaultdict[str, list[_T]](list)
|
||||
# Track original modality for each vision_chunk item (image or video)
|
||||
@@ -487,6 +492,14 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
model_cls = get_model_cls(self.model_config)
|
||||
return cast(type[SupportsMultiModal], model_cls)
|
||||
|
||||
@property
|
||||
def media_io_kwargs(self) -> dict[str, dict[str, Any]] | None:
|
||||
return self._media_io_kwargs or (
|
||||
self._model_config.multimodal_config.media_io_kwargs
|
||||
if self._model_config.multimodal_config
|
||||
else None
|
||||
)
|
||||
|
||||
@property
|
||||
def allowed_local_media_path(self):
|
||||
return self._model_config.allowed_local_media_path
|
||||
@@ -769,12 +782,10 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
super().__init__()
|
||||
|
||||
self._tracker = tracker
|
||||
multimodal_config = self._tracker.model_config.multimodal_config
|
||||
media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)
|
||||
|
||||
self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
|
||||
envs.VLLM_MEDIA_CONNECTOR,
|
||||
media_io_kwargs=media_io_kwargs,
|
||||
media_io_kwargs=tracker.media_io_kwargs,
|
||||
allowed_local_media_path=tracker.allowed_local_media_path,
|
||||
allowed_media_domains=tracker.allowed_media_domains,
|
||||
)
|
||||
@@ -881,11 +892,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
super().__init__()
|
||||
|
||||
self._tracker = tracker
|
||||
multimodal_config = self._tracker.model_config.multimodal_config
|
||||
media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)
|
||||
self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
|
||||
envs.VLLM_MEDIA_CONNECTOR,
|
||||
media_io_kwargs=media_io_kwargs,
|
||||
media_io_kwargs=tracker.media_io_kwargs,
|
||||
allowed_local_media_path=tracker.allowed_local_media_path,
|
||||
allowed_media_domains=tracker.allowed_media_domains,
|
||||
)
|
||||
@@ -1530,13 +1539,14 @@ def parse_chat_messages(
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
model_config: ModelConfig,
|
||||
content_format: ChatTemplateContentFormat,
|
||||
media_io_kwargs: dict[str, dict[str, Any]] | None = None,
|
||||
) -> tuple[
|
||||
list[ConversationMessage],
|
||||
MultiModalDataDict | None,
|
||||
MultiModalUUIDDict | None,
|
||||
]:
|
||||
conversation: list[ConversationMessage] = []
|
||||
mm_tracker = MultiModalItemTracker(model_config)
|
||||
mm_tracker = MultiModalItemTracker(model_config, media_io_kwargs=media_io_kwargs)
|
||||
|
||||
for msg in messages:
|
||||
sub_messages = _parse_chat_message_content(
|
||||
@@ -1563,13 +1573,16 @@ async def parse_chat_messages_async(
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
model_config: ModelConfig,
|
||||
content_format: ChatTemplateContentFormat,
|
||||
media_io_kwargs: dict[str, dict[str, Any]] | None = None,
|
||||
) -> tuple[
|
||||
list[ConversationMessage],
|
||||
MultiModalDataDict | None,
|
||||
MultiModalUUIDDict | None,
|
||||
]:
|
||||
conversation: list[ConversationMessage] = []
|
||||
mm_tracker = AsyncMultiModalItemTracker(model_config)
|
||||
mm_tracker = AsyncMultiModalItemTracker(
|
||||
model_config, media_io_kwargs=media_io_kwargs
|
||||
)
|
||||
|
||||
for msg in messages:
|
||||
sub_messages = _parse_chat_message_content(
|
||||
|
||||
@@ -268,6 +268,13 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
"Will be accessible by the chat template."
|
||||
),
|
||||
)
|
||||
media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional kwargs to pass to the media IO connectors, "
|
||||
"keyed by modality. Merged with engine-level media_io_kwargs."
|
||||
),
|
||||
)
|
||||
mm_processor_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=("Additional kwargs to pass to the HF processor."),
|
||||
@@ -366,6 +373,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
reasoning_effort=self.reasoning_effort,
|
||||
),
|
||||
),
|
||||
media_io_kwargs=self.media_io_kwargs,
|
||||
)
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
|
||||
@@ -900,10 +900,15 @@ class OpenAIServing:
|
||||
),
|
||||
)
|
||||
|
||||
mm_config = self.model_config.multimodal_config
|
||||
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
chat_params = request.build_chat_params(
|
||||
default_template, default_template_content_format
|
||||
).with_defaults(default_template_kwargs)
|
||||
).with_defaults(
|
||||
default_template_kwargs,
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
)
|
||||
|
||||
(conversation,), (engine_prompt,) = await renderer.render_chat_async(
|
||||
[messages],
|
||||
|
||||
@@ -197,6 +197,13 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional kwargs to pass to the media IO connectors, "
|
||||
"keyed by modality. Merged with engine-level media_io_kwargs."
|
||||
),
|
||||
)
|
||||
mm_processor_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=("Additional kwargs to pass to the HF processor."),
|
||||
@@ -276,6 +283,7 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
reasoning_effort=None if reasoning is None else reasoning.effort,
|
||||
),
|
||||
),
|
||||
media_io_kwargs=self.media_io_kwargs,
|
||||
)
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
|
||||
@@ -123,10 +123,15 @@ class PoolingIOProcessor:
|
||||
),
|
||||
)
|
||||
|
||||
mm_config = self.model_config.multimodal_config
|
||||
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
chat_params = request.build_chat_params(
|
||||
default_template, default_template_content_format
|
||||
).with_defaults(default_template_kwargs)
|
||||
).with_defaults(
|
||||
default_template_kwargs,
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
)
|
||||
|
||||
(conversation,), (engine_prompt,) = renderer.render_chat(
|
||||
[messages],
|
||||
|
||||
@@ -124,6 +124,13 @@ class ChatRequestMixin(OpenAIBaseModel):
|
||||
"Will be accessible by the chat template."
|
||||
),
|
||||
)
|
||||
media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional kwargs to pass to the media IO connectors, "
|
||||
"keyed by modality. Merged with engine-level media_io_kwargs."
|
||||
),
|
||||
)
|
||||
# --8<-- [end:chat-extra-params]
|
||||
|
||||
@model_validator(mode="before")
|
||||
@@ -151,6 +158,7 @@ class ChatRequestMixin(OpenAIBaseModel):
|
||||
continue_final_message=self.continue_final_message,
|
||||
),
|
||||
),
|
||||
media_io_kwargs=self.media_io_kwargs,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -100,6 +100,13 @@ class TokenizeChatRequest(OpenAIBaseModel):
|
||||
"Will be accessible by the chat template."
|
||||
),
|
||||
)
|
||||
media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional kwargs to pass to the media IO connectors, "
|
||||
"keyed by modality. Merged with engine-level media_io_kwargs."
|
||||
),
|
||||
)
|
||||
mm_processor_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="Additional kwargs to pass to the HF processor.",
|
||||
@@ -134,6 +141,7 @@ class TokenizeChatRequest(OpenAIBaseModel):
|
||||
continue_final_message=self.continue_final_message,
|
||||
),
|
||||
),
|
||||
media_io_kwargs=self.media_io_kwargs,
|
||||
)
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
|
||||
Reference in New Issue
Block a user