[V1] Enable multi-input by default (#15799)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1034,6 +1034,20 @@ class BaseProcessingInfo:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_allowed_mm_limits(self) -> Mapping[str, int]:
|
||||
"""Return the maximum allowed number of items for each modality."""
|
||||
supported_mm_limits = self.get_supported_mm_limits()
|
||||
mm_config = self.ctx.get_mm_config()
|
||||
|
||||
allowed_limits = dict[str, int]()
|
||||
for modality, supported_limit in supported_mm_limits.items():
|
||||
user_limit = mm_config.get_limit_per_prompt(modality)
|
||||
|
||||
allowed_limits[modality] = (user_limit if supported_limit is None
|
||||
else min(user_limit, supported_limit))
|
||||
|
||||
return allowed_limits
|
||||
|
||||
|
||||
_I = TypeVar("_I", bound=BaseProcessingInfo)
|
||||
|
||||
@@ -1087,14 +1101,24 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
before passing them to :meth:`_get_hf_mm_data`.
|
||||
"""
|
||||
mm_items = self.data_parser.parse_mm_data(mm_data)
|
||||
mm_config = self.info.ctx.get_mm_config()
|
||||
supported_mm_limits = self.info.get_supported_mm_limits()
|
||||
allowed_mm_limits = self.info.get_allowed_mm_limits()
|
||||
|
||||
for modality, items in mm_items.items():
|
||||
limit = mm_config.get_limit_per_prompt(modality)
|
||||
if len(items) > limit:
|
||||
supported_limit = supported_mm_limits.get(modality, 0)
|
||||
allowed_limit = allowed_mm_limits.get(modality, 0)
|
||||
num_items = len(items)
|
||||
|
||||
if supported_limit is not None and num_items > supported_limit:
|
||||
raise ValueError(
|
||||
f"You set {modality}={limit} (or defaulted to 1) in "
|
||||
f"`--limit-mm-per-prompt`, but passed {len(items)} "
|
||||
f"The model only supports at most {supported_limit} "
|
||||
f"{modality} items, but you passed {num_items} "
|
||||
f"{modality} items in the same prompt.")
|
||||
|
||||
if num_items > allowed_limit:
|
||||
raise ValueError(
|
||||
f"You set or defaulted to {modality}={allowed_limit} "
|
||||
f"in --limit-mm-per-prompt`, but passed {num_items} "
|
||||
f"{modality} items in the same prompt.")
|
||||
|
||||
return mm_items
|
||||
|
||||
@@ -162,23 +162,7 @@ class MultiModalProfiler(Generic[_I]):
|
||||
return self.processor.dummy_inputs
|
||||
|
||||
def get_mm_limits(self) -> Mapping[str, int]:
|
||||
mm_config = self.processing_info.ctx.get_mm_config()
|
||||
supported_mm_limits = self.processing_info.get_supported_mm_limits()
|
||||
|
||||
mm_limits = {
|
||||
modality: mm_config.get_limit_per_prompt(modality)
|
||||
for modality in supported_mm_limits
|
||||
}
|
||||
|
||||
for modality, supported_limit in supported_mm_limits.items():
|
||||
limit = mm_limits[modality]
|
||||
if supported_limit is not None and supported_limit < limit:
|
||||
raise ValueError(
|
||||
f"You set {modality}={limit} (or defaulted to 1) in "
|
||||
f"`--limit-mm-per-prompt`, but this model only supports "
|
||||
f"at most {supported_limit} {modality} items.")
|
||||
|
||||
return mm_limits
|
||||
return self.processing_info.get_allowed_mm_limits()
|
||||
|
||||
def _get_dummy_mm_inputs(
|
||||
self,
|
||||
|
||||
@@ -265,8 +265,10 @@ class MultiModalRegistry:
|
||||
|
||||
return profiler.get_mm_max_tokens(
|
||||
seq_len,
|
||||
{modality: 1
|
||||
for modality in mm_limits},
|
||||
{
|
||||
modality: 1
|
||||
for modality, limit in mm_limits.items() if limit > 0
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
|
||||
@@ -264,7 +264,7 @@ fetch_video = global_media_connector.fetch_video
|
||||
|
||||
def encode_audio_base64(
|
||||
audio: np.ndarray,
|
||||
sampling_rate: int,
|
||||
sampling_rate: float,
|
||||
) -> str:
|
||||
"""Encode audio as base64."""
|
||||
audio_io = AudioMediaIO()
|
||||
|
||||
Reference in New Issue
Block a user