[VLM] Merged multi-modal processor for InternVL-based models (#12553)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
|
||||
from collections import UserDict, defaultdict
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import accumulate
|
||||
from typing import (TYPE_CHECKING, Any, Literal, Optional, TypedDict, TypeVar,
|
||||
Union, cast, final)
|
||||
|
||||
@@ -258,6 +259,16 @@ class MultiModalFieldConfig:
|
||||
slices=slices,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def flat_from_sizes(modality: str, size_per_item: torch.Tensor):
|
||||
slice_idxs = [0, *accumulate(size_per_item)]
|
||||
slices = [
|
||||
slice(slice_idxs[i], slice_idxs[i + 1])
|
||||
for i in range(len(size_per_item))
|
||||
]
|
||||
|
||||
return MultiModalFieldConfig.flat(modality, slices)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
field_cls: type[BaseMultiModalField],
|
||||
|
||||
@@ -680,7 +680,11 @@ class BaseProcessingInfo:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
|
||||
def get_mm_max_tokens_per_item(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
) -> Mapping[str, int]:
|
||||
"""
|
||||
Get the maximum possible number of tokens per data item
|
||||
for each modality.
|
||||
|
||||
@@ -151,7 +151,8 @@ class MultiModalProfiler(Generic[_I]):
|
||||
mm_counts = self.get_mm_limits()
|
||||
|
||||
info = self.processing_info
|
||||
mm_max_tokens_per_item = info.get_mm_max_tokens_per_item(seq_len)
|
||||
mm_max_tokens_per_item = info.get_mm_max_tokens_per_item(
|
||||
seq_len, mm_counts)
|
||||
|
||||
if mm_counts.keys() != mm_max_tokens_per_item.keys():
|
||||
raise AssertionError(
|
||||
|
||||
@@ -264,7 +264,9 @@ class MultiModalRegistry:
|
||||
)
|
||||
processor = self.create_processor(model_config, tokenizer)
|
||||
seq_len = model_config.max_model_len
|
||||
return processor.info.get_mm_max_tokens_per_item(seq_len)
|
||||
mm_limits = self.get_mm_limits_per_prompt(model_config)
|
||||
return processor.info.get_mm_max_tokens_per_item(
|
||||
seq_len, mm_limits)
|
||||
|
||||
return {
|
||||
key: plugin.get_max_multimodal_tokens(model_config)
|
||||
|
||||
Reference in New Issue
Block a user