[V1][Bugfix] Fix data item ordering in mixed-modality inference (#12259)

Signed-off-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
Roger Wang
2025-01-21 13:18:43 +00:00
committed by GitHub
parent c81081fece
commit b197a5ccfd
2 changed files with 61 additions and 13 deletions
+33 -1
View File
@@ -1,4 +1,5 @@
from functools import lru_cache
from itertools import groupby
from pathlib import Path
from typing import TYPE_CHECKING, Optional, TypeVar, Union
from urllib.parse import ParseResult, urlparse
@@ -26,7 +27,7 @@ _M = TypeVar("_M")
if TYPE_CHECKING:
from .hasher import MultiModalHashDict
from .inputs import MultiModalPlaceholderDict
from .inputs import MultiModalKwargs, MultiModalPlaceholderDict
class MediaConnector:
@@ -477,3 +478,34 @@ def merge_and_sort_multimodal_metadata(
merged_hashes = None
return sorted_modalities, merged_placeholders, merged_hashes
def group_mm_inputs_by_modality(
mm_inputs: list["MultiModalKwargs"]) -> list[list["MultiModalKwargs"]]:
"""Group consecutive MultiModalKwargs from mm_inputs with the same modality
together into the same list for batching purpose. For MultiModalKwargs with
multiple modalities, put them into their own list.
Args:
mm_inputs: List of MultiModalKwargs.
Returns:
list[list[MultiModalKwargs]]: List of list of MultiModalKwargs, each
inner list contains consecutive MultiModalKwargs with same modality, or
one with multimodal modalities.
"""
if not mm_inputs:
return []
def modality_group_func(mm_input: "MultiModalKwargs") -> Union[str, int]:
# If the input has multiple modalities, return a id as the unique key
# for the mm_input input.
if len(mm_input.modalities) > 1:
return id(mm_input)
# Otherwise return the modality string
return list(mm_input.modalities)[0]
return [
list(group) for _, group in groupby(mm_inputs, key=modality_group_func)
]