[V1][Bugfix] Fix data item ordering in mixed-modality inference (#12259)
Signed-off-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
from functools import lru_cache
|
||||
from itertools import groupby
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Optional, TypeVar, Union
|
||||
from urllib.parse import ParseResult, urlparse
|
||||
@@ -26,7 +27,7 @@ _M = TypeVar("_M")
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .hasher import MultiModalHashDict
|
||||
from .inputs import MultiModalPlaceholderDict
|
||||
from .inputs import MultiModalKwargs, MultiModalPlaceholderDict
|
||||
|
||||
|
||||
class MediaConnector:
|
||||
@@ -477,3 +478,34 @@ def merge_and_sort_multimodal_metadata(
|
||||
merged_hashes = None
|
||||
|
||||
return sorted_modalities, merged_placeholders, merged_hashes
|
||||
|
||||
|
||||
def group_mm_inputs_by_modality(
|
||||
mm_inputs: list["MultiModalKwargs"]) -> list[list["MultiModalKwargs"]]:
|
||||
"""Group consecutive MultiModalKwargs from mm_inputs with the same modality
|
||||
together into the same list for batching purpose. For MultiModalKwargs with
|
||||
multiple modalities, put them into their own list.
|
||||
|
||||
Args:
|
||||
mm_inputs: List of MultiModalKwargs.
|
||||
|
||||
Returns:
|
||||
list[list[MultiModalKwargs]]: List of list of MultiModalKwargs, each
|
||||
inner list contains consecutive MultiModalKwargs with same modality, or
|
||||
one with multimodal modalities.
|
||||
"""
|
||||
if not mm_inputs:
|
||||
return []
|
||||
|
||||
def modality_group_func(mm_input: "MultiModalKwargs") -> Union[str, int]:
|
||||
# If the input has multiple modalities, return a id as the unique key
|
||||
# for the mm_input input.
|
||||
if len(mm_input.modalities) > 1:
|
||||
return id(mm_input)
|
||||
|
||||
# Otherwise return the modality string
|
||||
return list(mm_input.modalities)[0]
|
||||
|
||||
return [
|
||||
list(group) for _, group in groupby(mm_inputs, key=modality_group_func)
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user