Support online use_audio_in_video (#36319)

Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Tianyu Guo
2026-03-09 07:16:44 -07:00
committed by GitHub
co-authored by Isotr0py
parent 3ec2115015
commit 5578f2a4d3
10 changed files with 152 additions and 10 deletions
+54 -8
View File
@@ -564,7 +564,9 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
return self.model_cls.get_placeholder_str(modality, num_items)
@abstractmethod
def create_parser(self) -> "BaseMultiModalContentParser":
def create_parser(
self, mm_processor_kwargs: dict[str, Any] | None = None
) -> "BaseMultiModalContentParser":
raise NotImplementedError
@@ -690,8 +692,10 @@ class MultiModalItemTracker(BaseMultiModalItemTracker[tuple[object, str | None]]
dict(self._items_by_modality), self.mm_processor, self._modality_order
)
def create_parser(self) -> "BaseMultiModalContentParser":
return MultiModalContentParser(self)
def create_parser(
self, mm_processor_kwargs: dict[str, Any] | None = None
) -> "BaseMultiModalContentParser":
return MultiModalContentParser(self, mm_processor_kwargs=mm_processor_kwargs)
class AsyncMultiModalItemTracker(
@@ -712,8 +716,12 @@ class AsyncMultiModalItemTracker(
resolved_items_by_modality, self.mm_processor, self._modality_order
)
def create_parser(self) -> "BaseMultiModalContentParser":
return AsyncMultiModalContentParser(self)
def create_parser(
self, mm_processor_kwargs: dict[str, Any] | None = None
) -> "BaseMultiModalContentParser":
return AsyncMultiModalContentParser(
self, mm_processor_kwargs=mm_processor_kwargs
)
class BaseMultiModalContentParser(ABC):
@@ -778,7 +786,11 @@ class BaseMultiModalContentParser(ABC):
class MultiModalContentParser(BaseMultiModalContentParser):
def __init__(self, tracker: MultiModalItemTracker) -> None:
def __init__(
self,
tracker: MultiModalItemTracker,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> None:
super().__init__()
self._tracker = tracker
@@ -790,6 +802,8 @@ class MultiModalContentParser(BaseMultiModalContentParser):
allowed_media_domains=tracker.allowed_media_domains,
)
self._mm_processor_kwargs = mm_processor_kwargs
@property
def model_config(self) -> ModelConfig:
return self._tracker.model_config
@@ -886,9 +900,23 @@ class MultiModalContentParser(BaseMultiModalContentParser):
placeholder = self._tracker.add("video", (video, uuid))
self._add_placeholder("video", placeholder)
# Extract audio from video if use_audio_in_video is True
if (
video_url
and self._mm_processor_kwargs
and self._mm_processor_kwargs.get("use_audio_in_video", False)
):
audio = self._connector.fetch_audio(video_url) if video_url else None
audio_placeholder = self._tracker.add("audio", (audio, uuid))
self._add_placeholder("audio", audio_placeholder)
class AsyncMultiModalContentParser(BaseMultiModalContentParser):
def __init__(self, tracker: AsyncMultiModalItemTracker) -> None:
def __init__(
self,
tracker: AsyncMultiModalItemTracker,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> None:
super().__init__()
self._tracker = tracker
@@ -898,6 +926,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
allowed_local_media_path=tracker.allowed_local_media_path,
allowed_media_domains=tracker.allowed_media_domains,
)
self._mm_processor_kwargs: dict[str, Any] | None = mm_processor_kwargs
@property
def model_config(self) -> ModelConfig:
@@ -1033,6 +1062,16 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
placeholder = self._tracker.add("video", coro)
self._add_placeholder("video", placeholder)
# Extract audio from video if use_audio_in_video is True
if (
video_url
and self._mm_processor_kwargs
and self._mm_processor_kwargs.get("use_audio_in_video", False)
):
audio_coro = self._audio_with_uuid_async(video_url, uuid)
audio_placeholder = self._tracker.add("audio", audio_coro)
self._add_placeholder("audio", audio_placeholder)
@dataclass
class ChatTemplateConfig:
@@ -1343,10 +1382,11 @@ def _parse_chat_message_content_parts(
*,
wrap_dicts: bool,
interleave_strings: bool,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[ConversationMessage]:
content = list[_ContentPart]()
mm_parser = mm_tracker.create_parser()
mm_parser = mm_tracker.create_parser(mm_processor_kwargs=mm_processor_kwargs)
for part in parts:
parse_res = _parse_chat_message_content_part(
@@ -1464,6 +1504,7 @@ def _parse_chat_message_content(
mm_tracker: BaseMultiModalItemTracker,
content_format: ChatTemplateContentFormat,
interleave_strings: bool,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[ConversationMessage]:
role = message["role"]
content = message.get("content")
@@ -1479,6 +1520,7 @@ def _parse_chat_message_content(
mm_tracker,
wrap_dicts=(content_format == "openai"),
interleave_strings=interleave_strings,
mm_processor_kwargs=mm_processor_kwargs,
)
for result_msg in result:
@@ -1540,6 +1582,7 @@ def parse_chat_messages(
model_config: ModelConfig,
content_format: ChatTemplateContentFormat,
media_io_kwargs: dict[str, dict[str, Any]] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> tuple[
list[ConversationMessage],
MultiModalDataDict | None,
@@ -1558,6 +1601,7 @@ def parse_chat_messages(
and model_config.multimodal_config is not None
and model_config.multimodal_config.interleave_mm_strings
),
mm_processor_kwargs=mm_processor_kwargs,
)
conversation.extend(sub_messages)
@@ -1574,6 +1618,7 @@ async def parse_chat_messages_async(
model_config: ModelConfig,
content_format: ChatTemplateContentFormat,
media_io_kwargs: dict[str, dict[str, Any]] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> tuple[
list[ConversationMessage],
MultiModalDataDict | None,
@@ -1594,6 +1639,7 @@ async def parse_chat_messages_async(
and model_config.multimodal_config is not None
and model_config.multimodal_config.interleave_mm_strings
),
mm_processor_kwargs=mm_processor_kwargs,
)
conversation.extend(sub_messages)