[V1][VLM] Proper memory profiling for image language models (#11210)

Signed-off-by: Roger Wang <ywang@roblox.com>
Co-authored-by: ywang96 <ywang@example.com>
This commit is contained in:
Roger Wang
2024-12-16 22:10:57 -08:00
committed by GitHub
co-authored by ywang96
parent 66d4b16724
commit 59c9b6ebeb
6 changed files with 98 additions and 13 deletions
+20 -3
View File
@@ -200,6 +200,23 @@ class MultiModalRegistry:
"""
return self.register_max_multimodal_tokens("image", max_mm_tokens)
def get_max_tokens_per_item_by_modality(
self,
model_config: "ModelConfig",
) -> Mapping[str, int]:
"""
Get the maximum number of tokens per data item from each modality
for profiling the memory usage of a model.
Note:
This is currently directly used only in V1.
"""
return {
key: plugin.get_max_multimodal_tokens(model_config)
for key, plugin in self._plugins.items()
}
def get_max_tokens_by_modality(
self,
model_config: "ModelConfig",
@@ -216,9 +233,9 @@ class MultiModalRegistry:
limits_per_plugin = self._limits_by_model[model_config]
return {
key: (limits_per_plugin[key] *
plugin.get_max_multimodal_tokens(model_config))
for key, plugin in self._plugins.items()
key: limits_per_plugin[key] * max_tokens_per_mm_item
for key, max_tokens_per_mm_item in
self.get_max_tokens_per_item_by_modality(model_config).items()
}
def get_max_multimodal_tokens(self, model_config: "ModelConfig") -> int: