[VLM] Calculate maximum number of multi-modal tokens by model (#6121)
This commit is contained in:
+40
-35
@@ -7,7 +7,7 @@ from vllm.config import ModelConfig
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalInputs,
|
||||
MultiModalPlugin)
|
||||
MultiModalPlugin, MultiModalTokensCalc)
|
||||
from .image import ImagePlugin
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -48,28 +48,9 @@ class MultiModalRegistry:
|
||||
msg = f"Unknown multi-modal data type: {data_type_key}"
|
||||
raise NotImplementedError(msg)
|
||||
|
||||
def register_image_input_mapper(
|
||||
self,
|
||||
mapper: Optional[MultiModalInputMapper] = None,
|
||||
):
|
||||
"""
|
||||
Register an input mapper for image data to a model class.
|
||||
|
||||
See :meth:`MultiModalPlugin.register_input_mapper` for more details.
|
||||
"""
|
||||
return self.register_input_mapper("image", mapper)
|
||||
|
||||
def _process_input(self, key: str, value: object,
|
||||
model_config: ModelConfig) -> MultiModalInputs:
|
||||
plugin = self._plugins.get(key)
|
||||
if plugin:
|
||||
return plugin.map_input(model_config, value)
|
||||
msg = f"Unknown multi-modal data type: {key}"
|
||||
raise NotImplementedError(msg)
|
||||
|
||||
def register_input_mapper(
|
||||
self,
|
||||
data_type: str,
|
||||
data_type_key: str,
|
||||
mapper: Optional[MultiModalInputMapper] = None,
|
||||
):
|
||||
"""
|
||||
@@ -77,16 +58,14 @@ class MultiModalRegistry:
|
||||
|
||||
See :meth:`MultiModalPlugin.register_input_mapper` for more details.
|
||||
"""
|
||||
plugin = self._plugins.get(data_type)
|
||||
if not plugin:
|
||||
msg = f"Unknown multi-modal data type: {data_type}"
|
||||
raise NotImplementedError(msg)
|
||||
return plugin.register_input_mapper(mapper)
|
||||
return self._get_plugin(data_type_key).register_input_mapper(mapper)
|
||||
|
||||
def register_image_input(self,
|
||||
mapper: Optional[MultiModalInputMapper] = None):
|
||||
def register_image_input_mapper(
|
||||
self,
|
||||
mapper: Optional[MultiModalInputMapper] = None,
|
||||
):
|
||||
"""
|
||||
Register an input mapper for image pixel data to a model class.
|
||||
Register an input mapper for image data to a model class.
|
||||
|
||||
See :meth:`MultiModalPlugin.register_input_mapper` for more details.
|
||||
"""
|
||||
@@ -102,8 +81,8 @@ class MultiModalRegistry:
|
||||
merged_dict: Dict[str, torch.Tensor] = {}
|
||||
|
||||
for data_key, data_value in data.items():
|
||||
input_dict = self._process_input(data_key, data_value,
|
||||
model_config)
|
||||
input_dict = self._get_plugin(data_key) \
|
||||
.map_input(model_config, data_value)
|
||||
|
||||
for input_key, input_tensor in input_dict.items():
|
||||
if input_key in merged_dict:
|
||||
@@ -121,9 +100,35 @@ class MultiModalRegistry:
|
||||
"""
|
||||
return functools.partial(self.map_input, model_config)
|
||||
|
||||
def get_num_input_tokens(self):
|
||||
def register_max_multimodal_tokens(
|
||||
self,
|
||||
data_type_key: str,
|
||||
max_mm_tokens: Optional[MultiModalTokensCalc] = None,
|
||||
):
|
||||
"""
|
||||
Get the number of input tokens for profiling purposes.
|
||||
Register the maximum number of tokens, belonging to a
|
||||
specific modality, input to the language model for a model class.
|
||||
"""
|
||||
# TODO: Provide this number on a per model basis.
|
||||
return 3000
|
||||
return self._get_plugin(data_type_key) \
|
||||
.register_max_multimodal_tokens(max_mm_tokens)
|
||||
|
||||
def register_max_image_tokens(
|
||||
self,
|
||||
max_mm_tokens: Optional[MultiModalTokensCalc] = None,
|
||||
):
|
||||
"""
|
||||
Register the maximum number of image tokens
|
||||
input to the language model for a model class.
|
||||
"""
|
||||
return self.register_max_multimodal_tokens("image", max_mm_tokens)
|
||||
|
||||
def get_max_multimodal_tokens(self, model_config: ModelConfig) -> int:
|
||||
"""
|
||||
Get the maximum number of multi-modal tokens
|
||||
for profiling the memory usage of a model.
|
||||
|
||||
See :meth:`MultiModalPlugin.get_max_multimodal_tokens` for more details.
|
||||
"""
|
||||
return sum(
|
||||
plugin.get_max_multimodal_tokens(model_config)
|
||||
for plugin in self._plugins.values())
|
||||
|
||||
Reference in New Issue
Block a user