feat: expose media_io_kwargs at runtime (#34778)
Signed-off-by: Alexandre Milesi <milesial@users.noreply.github.com>
This commit is contained in:
@@ -268,6 +268,13 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
"Will be accessible by the chat template."
|
||||
),
|
||||
)
|
||||
media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional kwargs to pass to the media IO connectors, "
|
||||
"keyed by modality. Merged with engine-level media_io_kwargs."
|
||||
),
|
||||
)
|
||||
mm_processor_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=("Additional kwargs to pass to the HF processor."),
|
||||
@@ -366,6 +373,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
reasoning_effort=self.reasoning_effort,
|
||||
),
|
||||
),
|
||||
media_io_kwargs=self.media_io_kwargs,
|
||||
)
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
|
||||
@@ -900,10 +900,15 @@ class OpenAIServing:
|
||||
),
|
||||
)
|
||||
|
||||
mm_config = self.model_config.multimodal_config
|
||||
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
chat_params = request.build_chat_params(
|
||||
default_template, default_template_content_format
|
||||
).with_defaults(default_template_kwargs)
|
||||
).with_defaults(
|
||||
default_template_kwargs,
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
)
|
||||
|
||||
(conversation,), (engine_prompt,) = await renderer.render_chat_async(
|
||||
[messages],
|
||||
|
||||
@@ -197,6 +197,13 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional kwargs to pass to the media IO connectors, "
|
||||
"keyed by modality. Merged with engine-level media_io_kwargs."
|
||||
),
|
||||
)
|
||||
mm_processor_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=("Additional kwargs to pass to the HF processor."),
|
||||
@@ -276,6 +283,7 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
reasoning_effort=None if reasoning is None else reasoning.effort,
|
||||
),
|
||||
),
|
||||
media_io_kwargs=self.media_io_kwargs,
|
||||
)
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
|
||||
Reference in New Issue
Block a user