[Renderer] Move MM Hash parsing into Renderer (#34711)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-02-17 19:18:55 -08:00
committed by GitHub
parent df3f537a66
commit a0d8d944e2
16 changed files with 254 additions and 217 deletions
+63 -48
View File
@@ -3,7 +3,7 @@
from abc import ABC, abstractmethod
from collections import UserDict
from collections.abc import Callable, Iterator, Mapping, Sequence
from collections.abc import Callable, Iterator, Mapping, Sequence, Set
from typing import (
TYPE_CHECKING,
Any,
@@ -33,6 +33,7 @@ from .inputs import (
MultiModalDataDict,
MultiModalFieldConfig,
MultiModalKwargsItems,
MultiModalUUIDDict,
VideoItem,
)
from .media import MediaWithBytes
@@ -297,14 +298,15 @@ class DictEmbeddingItems(
return self.data
class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
def __init__(self, data: Sequence[HfAudioItem] | None) -> None:
if data is None:
data = [None]
class AudioProcessorItems(ProcessorBatchItems[HfAudioItem | None]):
def __init__(self, data: Sequence[HfAudioItem | None]) -> None:
super().__init__(data, "audio")
def get_audio_length(self, item_idx: int) -> int:
audio = self.get(item_idx)
if audio is None:
raise ValueError(f"Cannot get length of cached audio at {item_idx}")
return len(audio)
@@ -322,14 +324,14 @@ class ImageSize(NamedTuple):
height: int
class ImageProcessorItems(ProcessorBatchItems[HfImageItem]):
def __init__(self, data: Sequence[HfImageItem] | None) -> None:
if data is None:
data = [None]
class ImageProcessorItems(ProcessorBatchItems[HfImageItem | None]):
def __init__(self, data: Sequence[HfImageItem | None]) -> None:
super().__init__(data, "image")
def get_image_size(self, item_idx: int) -> ImageSize:
image = self.get(item_idx)
if image is None:
raise ValueError(f"Cannot get size of cached image at {item_idx}")
if isinstance(image, PILImage.Image):
return ImageSize(*image.size)
@@ -349,22 +351,31 @@ class ImageEmbeddingItems(EmbeddingItems):
super().__init__(data, "image", expected_hidden_size)
class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
class VideoProcessorItems(ProcessorBatchItems[HfVideoItem | None]):
def __init__(
self,
data: Sequence[HfVideoItem] | None,
data: Sequence[HfVideoItem | None],
metadata: dict[str, Any] | list[dict[str, Any] | None] | None = None,
) -> None:
if data is None:
data = [None]
super().__init__(data, "video")
self.metadata = metadata
def get_num_frames(self, item_idx: int) -> int:
return len(self.get(item_idx))
video = self.get(item_idx)
if video is None:
raise ValueError(f"Cannot get length of cached video at {item_idx}")
return len(video)
def get_frame_size(self, item_idx: int) -> ImageSize:
image = self.get(item_idx)[0] # Assume that the video isn't empty
video = self.get(item_idx)
if video is None:
raise ValueError(f"Cannot get size of cached video at {item_idx}")
if len(video) == 0:
raise ValueError(f"Cannot get size of empty video at {item_idx}")
image = video[0]
if isinstance(image, PILImage.Image):
return ImageSize(*image.size)
@@ -400,6 +411,15 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
normalized such that each entry corresponds to a list.
"""
def select(self, modalities: Set[str]):
"""
Construct a new `MultiModalDataItems` instance containing only the
selected modalities.
"""
return MultiModalDataItems(
{modality: self[modality] for modality in modalities}
)
def get_count(self, modality: str, *, strict: bool = True) -> int:
"""
Get the number of data items belonging to a modality.
@@ -497,19 +517,11 @@ class MultiModalDataParser:
) -> TypeGuard[torch.Tensor | list[torch.Tensor]]:
if isinstance(data, torch.Tensor):
return data.ndim == 3
if is_list_of(data, torch.Tensor):
if is_list_of(data, torch.Tensor) and len(data) > 0:
return data[0].ndim == 2 # type: ignore[index]
return False
def _is_empty(self, data: object) -> TypeGuard[None]:
if isinstance(data, list):
return len(data) == 0
if isinstance(data, (np.ndarray, torch.Tensor)):
return data.size == 0
return False
def _get_audio_with_sr(
self,
audio: AudioItem,
@@ -545,12 +557,6 @@ class MultiModalDataParser:
data: ModalityData[AudioItem],
) -> ModalityDataItems[Any, Any] | None:
if data is None:
return AudioProcessorItems(None)
# also check single audio item with sampling rate
if self._is_empty(data) or (
isinstance(data, tuple) and self._is_empty(data[0])
):
return None
if self.is_embeddings(data):
@@ -558,9 +564,8 @@ class MultiModalDataParser:
data_items: list[AudioItem]
if (
is_list_of(data, float)
or isinstance(data, (np.ndarray, torch.Tensor))
and data.ndim == 1
(is_list_of(data, float) and len(data) > 0)
or (isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 1)
or isinstance(data, tuple)
):
data_items = [data]
@@ -591,18 +596,13 @@ class MultiModalDataParser:
data: ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if data is None:
return ImageProcessorItems(None)
if self._is_empty(data):
return None
if self.is_embeddings(data):
return ImageEmbeddingItems(data, self.expected_hidden_size)
if (
isinstance(data, (PILImage.Image, MediaWithBytes))
or isinstance(data, (np.ndarray, torch.Tensor))
and data.ndim == 3
if isinstance(data, (PILImage.Image, MediaWithBytes)) or (
isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 3
):
data_items = [data]
elif isinstance(data, (np.ndarray, torch.Tensor)):
@@ -617,19 +617,14 @@ class MultiModalDataParser:
data: ModalityData[VideoItem],
) -> ModalityDataItems[Any, Any] | None:
if data is None:
return VideoProcessorItems(None)
if self._is_empty(data):
return None
if self.is_embeddings(data):
return VideoEmbeddingItems(data, self.expected_hidden_size)
data_items: list[VideoItem]
if (
is_list_of(data, PILImage.Image)
or isinstance(data, (np.ndarray, torch.Tensor))
and data.ndim == 4
if (is_list_of(data, PILImage.Image) and len(data) > 0) or (
isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 4
):
data_items = [data]
elif isinstance(data, (np.ndarray, torch.Tensor)):
@@ -664,12 +659,15 @@ class MultiModalDataParser:
data: ModalityData[Any],
) -> ModalityDataItems[Any, Any] | None:
"""Parse vision chunk data (unified image and video chunks)."""
if data is None or self._is_empty(data):
if data is None:
return None
if self.is_embeddings(data):
raise ValueError("Do not support embedding data for vision_chunk right now")
if isinstance(data, dict):
data = [data]
return VisionChunkProcessorItems(data)
def _get_subparsers(self) -> Mapping[str, ModalityDataParser]:
@@ -693,3 +691,20 @@ class MultiModalDataParser:
mm_items[k] = parsed_data
return mm_items
MultiModalUUIDItems: TypeAlias = dict[str, Sequence[str | None]]
"""
As [`MultiModalUUIDDict`][vllm.multimodal.inputs.MultiModalUUIDDict], but
normalized such that each entry corresponds to a list.
"""
def parse_mm_uuids(mm_uuids: MultiModalUUIDDict | None) -> MultiModalUUIDItems:
if mm_uuids is None:
return {}
return {
modality: [uuids] if isinstance(uuids, str) else uuids
for modality, uuids in mm_uuids.items()
}