[VLM] Calculate maximum number of multi-modal tokens by model (#6121)

This commit is contained in:
Cyrus Leung
2024-07-04 16:37:23 -07:00
committed by GitHub
parent 69ec3ca14c
commit ae96ef8fbd
12 changed files with 265 additions and 95 deletions
+40 -35
View File
@@ -7,7 +7,7 @@ from vllm.config import ModelConfig
from vllm.logger import init_logger
from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalInputs,
MultiModalPlugin)
MultiModalPlugin, MultiModalTokensCalc)
from .image import ImagePlugin
logger = init_logger(__name__)
@@ -48,28 +48,9 @@ class MultiModalRegistry:
msg = f"Unknown multi-modal data type: {data_type_key}"
raise NotImplementedError(msg)
def register_image_input_mapper(
self,
mapper: Optional[MultiModalInputMapper] = None,
):
"""
Register an input mapper for image data to a model class.
See :meth:`MultiModalPlugin.register_input_mapper` for more details.
"""
return self.register_input_mapper("image", mapper)
def _process_input(self, key: str, value: object,
model_config: ModelConfig) -> MultiModalInputs:
plugin = self._plugins.get(key)
if plugin:
return plugin.map_input(model_config, value)
msg = f"Unknown multi-modal data type: {key}"
raise NotImplementedError(msg)
def register_input_mapper(
self,
data_type: str,
data_type_key: str,
mapper: Optional[MultiModalInputMapper] = None,
):
"""
@@ -77,16 +58,14 @@ class MultiModalRegistry:
See :meth:`MultiModalPlugin.register_input_mapper` for more details.
"""
plugin = self._plugins.get(data_type)
if not plugin:
msg = f"Unknown multi-modal data type: {data_type}"
raise NotImplementedError(msg)
return plugin.register_input_mapper(mapper)
return self._get_plugin(data_type_key).register_input_mapper(mapper)
def register_image_input(self,
mapper: Optional[MultiModalInputMapper] = None):
def register_image_input_mapper(
self,
mapper: Optional[MultiModalInputMapper] = None,
):
"""
Register an input mapper for image pixel data to a model class.
Register an input mapper for image data to a model class.
See :meth:`MultiModalPlugin.register_input_mapper` for more details.
"""
@@ -102,8 +81,8 @@ class MultiModalRegistry:
merged_dict: Dict[str, torch.Tensor] = {}
for data_key, data_value in data.items():
input_dict = self._process_input(data_key, data_value,
model_config)
input_dict = self._get_plugin(data_key) \
.map_input(model_config, data_value)
for input_key, input_tensor in input_dict.items():
if input_key in merged_dict:
@@ -121,9 +100,35 @@ class MultiModalRegistry:
"""
return functools.partial(self.map_input, model_config)
def get_num_input_tokens(self):
def register_max_multimodal_tokens(
self,
data_type_key: str,
max_mm_tokens: Optional[MultiModalTokensCalc] = None,
):
"""
Get the number of input tokens for profiling purposes.
Register the maximum number of tokens, belonging to a
specific modality, input to the language model for a model class.
"""
# TODO: Provide this number on a per model basis.
return 3000
return self._get_plugin(data_type_key) \
.register_max_multimodal_tokens(max_mm_tokens)
def register_max_image_tokens(
self,
max_mm_tokens: Optional[MultiModalTokensCalc] = None,
):
"""
Register the maximum number of image tokens
input to the language model for a model class.
"""
return self.register_max_multimodal_tokens("image", max_mm_tokens)
def get_max_multimodal_tokens(self, model_config: ModelConfig) -> int:
"""
Get the maximum number of multi-modal tokens
for profiling the memory usage of a model.
See :meth:`MultiModalPlugin.get_max_multimodal_tokens` for more details.
"""
return sum(
plugin.get_max_multimodal_tokens(model_config)
for plugin in self._plugins.values())