[Frontend][Multimodal] Allow skipping media data when UUIDs are provided. (#23950)
Signed-off-by: Roger Wang <hey@rogerw.io> Signed-off-by: Chenheli Hua <huachenheli@outlook.com> Signed-off-by: Roger Wang <hey@rogerw.me> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roger Wang <hey@rogerw.me>
This commit is contained in:
co-authored by
Roger Wang
Roger Wang
parent
4fdd6f5cbf
commit
7f2ea7074e
@@ -85,9 +85,10 @@ which are treated as audio embeddings;
|
||||
these are directly passed to the model without HF processing.
|
||||
"""
|
||||
|
||||
ModalityData: TypeAlias = Union[_T, list[_T]]
|
||||
ModalityData: TypeAlias = Union[_T, list[Optional[_T]], None]
|
||||
"""
|
||||
Either a single data item, or a list of data items.
|
||||
Either a single data item, or a list of data items. Can only be None if UUID
|
||||
is provided.
|
||||
|
||||
The number of data items allowed per modality is restricted by
|
||||
`--limit-mm-per-prompt`.
|
||||
|
||||
@@ -36,7 +36,7 @@ class ModalityDataItems(ABC, Generic[_T, _I]):
|
||||
def __init__(self, data: _T, modality: str) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.data = data
|
||||
self.data: _T = data
|
||||
self.modality = modality
|
||||
|
||||
def __repr__(self) -> str:
|
||||
@@ -177,7 +177,9 @@ class DictEmbeddingItems(ModalityDataItems[Mapping[str, torch.Tensor],
|
||||
|
||||
class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
|
||||
|
||||
def __init__(self, data: Sequence[HfAudioItem]) -> None:
|
||||
def __init__(self, data: Optional[Sequence[HfAudioItem]]) -> None:
|
||||
if data is None:
|
||||
data = [None]
|
||||
super().__init__(data, "audio")
|
||||
|
||||
def get_audio_length(self, item_idx: int) -> int:
|
||||
@@ -198,7 +200,9 @@ class ImageSize(NamedTuple):
|
||||
|
||||
class ImageProcessorItems(ProcessorBatchItems[HfImageItem]):
|
||||
|
||||
def __init__(self, data: Sequence[HfImageItem]) -> None:
|
||||
def __init__(self, data: Optional[Sequence[HfImageItem]]) -> None:
|
||||
if data is None:
|
||||
data = [None]
|
||||
super().__init__(data, "image")
|
||||
|
||||
def get_image_size(self, item_idx: int) -> ImageSize:
|
||||
@@ -223,10 +227,12 @@ class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
data: Sequence[HfVideoItem],
|
||||
data: Optional[Sequence[HfVideoItem]],
|
||||
metadata: Optional[Union[dict[str, Any],
|
||||
list[Optional[dict[str, Any]]]]] = None,
|
||||
) -> None:
|
||||
if data is None:
|
||||
data = [None]
|
||||
super().__init__(data, "video")
|
||||
self.metadata = metadata
|
||||
|
||||
@@ -385,6 +391,9 @@ class MultiModalDataParser:
|
||||
self,
|
||||
data: ModalityData[AudioItem],
|
||||
) -> Optional[ModalityDataItems[Any, Any]]:
|
||||
if data is None:
|
||||
return AudioProcessorItems(None)
|
||||
|
||||
# also check single audio item with sampling rate
|
||||
if self._is_empty(data) or (isinstance(data, tuple)
|
||||
and self._is_empty(data[0])):
|
||||
@@ -420,6 +429,9 @@ class MultiModalDataParser:
|
||||
self,
|
||||
data: ModalityData[ImageItem],
|
||||
) -> Optional[ModalityDataItems[Any, Any]]:
|
||||
if data is None:
|
||||
return ImageProcessorItems(None)
|
||||
|
||||
if self._is_empty(data):
|
||||
return None
|
||||
|
||||
@@ -441,6 +453,9 @@ class MultiModalDataParser:
|
||||
self,
|
||||
data: ModalityData[VideoItem],
|
||||
) -> Optional[ModalityDataItems[Any, Any]]:
|
||||
if data is None:
|
||||
return VideoProcessorItems(None)
|
||||
|
||||
if self._is_empty(data):
|
||||
return None
|
||||
|
||||
|
||||
@@ -1075,7 +1075,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
|
||||
"""
|
||||
mm_items = self.data_parser.parse_mm_data(mm_data)
|
||||
|
||||
for modality, items in mm_items.items():
|
||||
self.validate_num_items(modality, len(items))
|
||||
|
||||
@@ -1436,10 +1435,18 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
]
|
||||
for modality, items_is_cached in mm_is_cached.items()
|
||||
}
|
||||
mm_missing_data = {
|
||||
modality: [mm_data_items[modality][idx] for idx in idxs]
|
||||
for modality, idxs in mm_missing_idxs.items()
|
||||
}
|
||||
mm_missing_data = {}
|
||||
for modality, idxs in mm_missing_idxs.items():
|
||||
missing_modality_data = []
|
||||
for idx in idxs:
|
||||
data = mm_data_items[modality][idx]
|
||||
if data is None:
|
||||
raise ValueError(
|
||||
f"Cache miss for {modality} at index {idx} "
|
||||
f"but data is not provided.")
|
||||
else:
|
||||
missing_modality_data.append(data)
|
||||
mm_missing_data[modality] = missing_modality_data
|
||||
|
||||
return self._to_mm_items(mm_missing_data)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user