[VLM] Move supported limits and max tokens to merged multi-modal processor (#11669)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
@@ -21,10 +21,15 @@ _I = TypeVar("_I")
|
||||
|
||||
class ModalityDataItems(ABC, Generic[_T, _I]):
|
||||
|
||||
def __init__(self, data: _T) -> None:
|
||||
def __init__(self, data: _T, modality: str) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.data = data
|
||||
self.modality = modality
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (f"{type(self).__name__}(modality={self.modality!r}, "
|
||||
f"len={len(self)})")
|
||||
|
||||
def __len__(self) -> int:
|
||||
return self.get_count()
|
||||
@@ -64,14 +69,6 @@ class ModalityDataItems(ABC, Generic[_T, _I]):
|
||||
|
||||
class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
|
||||
|
||||
def __init__(self, data: Sequence[_T], modality: str) -> None:
|
||||
super().__init__(data)
|
||||
|
||||
self.modality = modality
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (f"{type(self).__name__}(modality={self.modality!r})")
|
||||
|
||||
def get_count(self) -> int:
|
||||
return len(self.data)
|
||||
|
||||
@@ -87,14 +84,6 @@ class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
|
||||
|
||||
class EmbeddingItems(ModalityDataItems[NestedTensors, torch.Tensor]):
|
||||
|
||||
def __init__(self, data: NestedTensors, modality: str) -> None:
|
||||
super().__init__(data)
|
||||
|
||||
self.modality = modality
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (f"{type(self).__name__}(modality={self.modality!r})")
|
||||
|
||||
def get_count(self) -> int:
|
||||
return len(self.data)
|
||||
|
||||
@@ -222,22 +211,13 @@ class MultiModalDataParser:
|
||||
Parses :class:`MultiModalDataDict` into :class:`MultiModalDataItems`.
|
||||
|
||||
Args:
|
||||
max_mm_counts (Mapping[str, int]): The maximum allowed number of items
|
||||
belonging to each modality. This effectively sets a hard limit over
|
||||
`--limit-mm-per-prompt`.
|
||||
target_sr (float, optional): Enables automatic resampling of audio
|
||||
items to the model's expected sampling rate.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
max_mm_counts: Mapping[str, int] = {},
|
||||
target_sr: Optional[float] = None,
|
||||
) -> None:
|
||||
def __init__(self, *, target_sr: Optional[float] = None) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.max_mm_counts = max_mm_counts
|
||||
self.target_sr = target_sr
|
||||
|
||||
def _is_embeddings(self, data: object) -> TypeGuard[NestedTensors]:
|
||||
@@ -345,7 +325,6 @@ class MultiModalDataParser:
|
||||
|
||||
def parse_mm_data(self,
|
||||
mm_data: MultiModalDataDict) -> MultiModalDataItems:
|
||||
max_mm_counts = self.max_mm_counts
|
||||
subparsers = self._get_subparsers()
|
||||
|
||||
mm_items = MultiModalDataItems()
|
||||
@@ -353,16 +332,6 @@ class MultiModalDataParser:
|
||||
if k not in subparsers:
|
||||
raise ValueError(f"Unsupported modality: {k}")
|
||||
|
||||
modality_items = subparsers[k](v)
|
||||
|
||||
if k in max_mm_counts:
|
||||
max_count = max_mm_counts[k]
|
||||
if len(modality_items) > max_count:
|
||||
raise ValueError(
|
||||
f"This model supports at most {max_count} {k} items "
|
||||
f"per prompt, but {len(modality_items)} {k} items "
|
||||
"were given or set as its limit_mm_per_prompt.")
|
||||
|
||||
mm_items[k] = modality_items
|
||||
mm_items[k] = subparsers[k](v)
|
||||
|
||||
return mm_items
|
||||
|
||||
Reference in New Issue
Block a user