[V1] Fix multimodal profiling for Molmo (#11325)

Signed-off-by: ywang96 <ywang@example.com>
Co-authored-by: ywang96 <ywang@example.com>
This commit is contained in:
Roger Wang
2024-12-19 16:27:22 +00:00
committed by GitHub
co-authored by ywang96
parent 6c7f881541
commit 7379b3d4b2
4 changed files with 24 additions and 4 deletions
+17 -2
View File
@@ -151,17 +151,31 @@ class MMHasher:
def __init__(self):
pass
def hash_mm_data(
def hash_dummy_mm_data(
self,
mm_data: Optional[MultiModalDataDict]) -> Optional[List[str]]:
"""Hash user-defined dummy multimodal data used for profiling."""
if mm_data is None:
return None
image_inputs = mm_data['image']
# This is a temporary workaround for models (e.g, Molmo) that
# process multimodal data in the input processor (therefore
# image_inputs is MultiModalKwargs instead of raw input format).
# `raw_mm_data` with the original input format is expected
# in this case.
if isinstance(image_inputs, dict):
assert "raw_mm_data" in image_inputs and isinstance(
image_inputs["raw_mm_data"], PIL.Image.Image)
image_inputs = image_inputs.pop("raw_mm_data")
return self.hash_images(image_inputs)
def hash_prompt(self, prompt: PromptType) -> Optional[List[str]]:
def hash_prompt_mm_data(self, prompt: PromptType) -> Optional[List[str]]:
"""Hash multimodal data in the user input prompt if they exist."""
if "multi_modal_data" not in prompt:
return None
@@ -171,6 +185,7 @@ class MMHasher:
return self.hash_images(image_inputs)
def hash_images(self, image_inputs) -> Optional[List[str]]:
"""Hash PIL image objects to strings."""
if not isinstance(image_inputs, list):
image_inputs = [image_inputs]
assert len(image_inputs) > 0
+1 -1
View File
@@ -79,7 +79,7 @@ class Processor:
# Compute MM hashes (if enabled)
mm_hashes = None
if self.use_hash:
mm_hashes = self.mm_hasher.hash_prompt(prompt)
mm_hashes = self.mm_hasher.hash_prompt_mm_data(prompt)
# Process inputs.
preprocessed_inputs = self.input_preprocessor.preprocess(