[MM] Simplify _apply_hf_processor_main (#53275)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-22 01:49:37 -07:00
committed by GitHub
parent 7ca49fbe4b
commit 0b19ebcacd
90 changed files with 1848 additions and 1706 deletions
+41 -32
View File
@@ -1929,30 +1929,33 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
def _apply_hf_processor_tokens_only(
self,
prompt_tokens: list[int],
) -> list[int]:
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
processor = self.info.get_hf_processor()
tokenizer = processor.tokenizer
bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id
if len(prompt_tokens) == 0 or prompt_tokens[0] != bos_token_id:
if len(prompt) == 0 or prompt[0] != bos_token_id:
# Prepend the bos token to the prompt tokens
prompt_tokens = [bos_token_id] + prompt_tokens
prompt = [bos_token_id] + prompt
return prompt_tokens
return prompt
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
mm_counts = mm_items.get_all_counts()
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
mm_data = dict(processor_data)
hf_config = self.info.get_hf_config()
hf_processor = self.info.get_hf_processor(**mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
def patched_call(text=None, images=None, videos=None, **kwargs) -> BatchFeature:
res = hf_processor(text=text, images=images, videos=videos, **kwargs)
@@ -1984,14 +1987,16 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
# NOTE: metadata.frames_indices indicates
# the sampled frames indices of pre-sampled videos, which is
# used to calculate the timestamps. Make sure that
# do_sample_frames in mm_kwargs is false for presampled videos.
# do_sample_frames in hf_processor_mm_kwargs is false for
# presampled videos.
# NOTE: a copy of mm_kwargs is created to update do_sample_frames,
# otherwise mm_hash for the object will be incorrect.
video_mm_kwargs = dict(**mm_kwargs)
# NOTE: a copy of hf_processor_mm_kwargs is created to update
# do_sample_frames, otherwise mm_hash for the object will be
# incorrect.
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
if "do_sample_frames" not in video_mm_kwargs:
# molmo_utils already has "do_sample_frames" in
# mm_kwargs, don't overwrite it.
# hf_processor_mm_kwargs, don't overwrite it.
video_mm_kwargs["do_sample_frames"] = metadata.get(
"do_sample_frames", False
)
@@ -2015,7 +2020,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
input_ids = input_ids[:, 1:]
video_string = tokenizer.batch_decode(input_ids)[0]
prompt = prompt.replace(VIDEO_PROMPT, video_string, 1)
prompt_text = prompt_text.replace(VIDEO_PROMPT, video_string, 1)
video_grids = video_outputs.pop("video_grids")
assert video_grids[:, 0].sum() == len(
@@ -2054,10 +2059,10 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
else:
all_video_outputs = dict()
processed_outputs = self.info.ctx.call_hf_processor(
processed_data = self.info.ctx.call_hf_processor(
patched_call,
dict(text=prompt, **mm_data),
mm_kwargs,
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
if (images := mm_data.get("images")) is not None:
@@ -2077,22 +2082,22 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
for image_size in image_sizes
]
num_crops = torch.tensor(tilings).prod(-1) + 1
assert sum(num_crops) == len(processed_outputs["pixel_values"])
assert sum(num_crops) == processed_outputs["image_num_crops"].sum().item()
assert sum(num_crops) == len(processed_data["pixel_values"])
assert sum(num_crops) == processed_data["image_num_crops"].sum().item()
image_grids = processed_outputs.pop("image_grids")
image_grids = processed_data.pop("image_grids")
image_num_pooled_patches = image_grids[:, :2].prod(dim=1) + image_grids[
:, 2:
].prod(dim=1)
processed_outputs["image_num_pooled_patches"] = image_num_pooled_patches
n_patches = processed_outputs["pixel_values"].shape[1]
processed_outputs["image_num_patches"] = (
processed_outputs["image_num_crops"] * n_patches
processed_data["image_num_pooled_patches"] = image_num_pooled_patches
n_patches = processed_data["pixel_values"].shape[1]
processed_data["image_num_patches"] = (
processed_data["image_num_crops"] * n_patches
)
(
processed_outputs["image_tokens"],
processed_outputs["num_image_tokens"],
processed_data["image_tokens"],
processed_data["num_image_tokens"],
) = build_flat_image_bool_length(
image_grids,
hf_config,
@@ -2101,7 +2106,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
use_single_crop_start_token=hf_processor.use_single_crop_start_token,
)
return BatchFeature({**processed_outputs, **all_video_outputs})
processed_data.update(all_video_outputs)
processed_data.update(passthrough_data)
processed_data.pop("input_ids")
return processed_data
def _get_mm_fields_config(
self,