[MM] Simplify _apply_hf_processor_main (#53275)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1929,30 +1929,33 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
|
||||
|
||||
|
||||
class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
def _apply_hf_processor_tokens_only(
|
||||
self,
|
||||
prompt_tokens: list[int],
|
||||
) -> list[int]:
|
||||
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
|
||||
processor = self.info.get_hf_processor()
|
||||
tokenizer = processor.tokenizer
|
||||
bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id
|
||||
|
||||
if len(prompt_tokens) == 0 or prompt_tokens[0] != bos_token_id:
|
||||
if len(prompt) == 0 or prompt[0] != bos_token_id:
|
||||
# Prepend the bos token to the prompt tokens
|
||||
prompt_tokens = [bos_token_id] + prompt_tokens
|
||||
prompt = [bos_token_id] + prompt
|
||||
|
||||
return prompt_tokens
|
||||
return prompt
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
|
||||
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
mm_data = dict(processor_data)
|
||||
|
||||
hf_config = self.info.get_hf_config()
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
def patched_call(text=None, images=None, videos=None, **kwargs) -> BatchFeature:
|
||||
res = hf_processor(text=text, images=images, videos=videos, **kwargs)
|
||||
@@ -1984,14 +1987,16 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
# NOTE: metadata.frames_indices indicates
|
||||
# the sampled frames indices of pre-sampled videos, which is
|
||||
# used to calculate the timestamps. Make sure that
|
||||
# do_sample_frames in mm_kwargs is false for presampled videos.
|
||||
# do_sample_frames in hf_processor_mm_kwargs is false for
|
||||
# presampled videos.
|
||||
|
||||
# NOTE: a copy of mm_kwargs is created to update do_sample_frames,
|
||||
# otherwise mm_hash for the object will be incorrect.
|
||||
video_mm_kwargs = dict(**mm_kwargs)
|
||||
# NOTE: a copy of hf_processor_mm_kwargs is created to update
|
||||
# do_sample_frames, otherwise mm_hash for the object will be
|
||||
# incorrect.
|
||||
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
|
||||
if "do_sample_frames" not in video_mm_kwargs:
|
||||
# molmo_utils already has "do_sample_frames" in
|
||||
# mm_kwargs, don't overwrite it.
|
||||
# hf_processor_mm_kwargs, don't overwrite it.
|
||||
video_mm_kwargs["do_sample_frames"] = metadata.get(
|
||||
"do_sample_frames", False
|
||||
)
|
||||
@@ -2015,7 +2020,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
input_ids = input_ids[:, 1:]
|
||||
|
||||
video_string = tokenizer.batch_decode(input_ids)[0]
|
||||
prompt = prompt.replace(VIDEO_PROMPT, video_string, 1)
|
||||
prompt_text = prompt_text.replace(VIDEO_PROMPT, video_string, 1)
|
||||
|
||||
video_grids = video_outputs.pop("video_grids")
|
||||
assert video_grids[:, 0].sum() == len(
|
||||
@@ -2054,10 +2059,10 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
else:
|
||||
all_video_outputs = dict()
|
||||
|
||||
processed_outputs = self.info.ctx.call_hf_processor(
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
patched_call,
|
||||
dict(text=prompt, **mm_data),
|
||||
mm_kwargs,
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
if (images := mm_data.get("images")) is not None:
|
||||
@@ -2077,22 +2082,22 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
for image_size in image_sizes
|
||||
]
|
||||
num_crops = torch.tensor(tilings).prod(-1) + 1
|
||||
assert sum(num_crops) == len(processed_outputs["pixel_values"])
|
||||
assert sum(num_crops) == processed_outputs["image_num_crops"].sum().item()
|
||||
assert sum(num_crops) == len(processed_data["pixel_values"])
|
||||
assert sum(num_crops) == processed_data["image_num_crops"].sum().item()
|
||||
|
||||
image_grids = processed_outputs.pop("image_grids")
|
||||
image_grids = processed_data.pop("image_grids")
|
||||
image_num_pooled_patches = image_grids[:, :2].prod(dim=1) + image_grids[
|
||||
:, 2:
|
||||
].prod(dim=1)
|
||||
|
||||
processed_outputs["image_num_pooled_patches"] = image_num_pooled_patches
|
||||
n_patches = processed_outputs["pixel_values"].shape[1]
|
||||
processed_outputs["image_num_patches"] = (
|
||||
processed_outputs["image_num_crops"] * n_patches
|
||||
processed_data["image_num_pooled_patches"] = image_num_pooled_patches
|
||||
n_patches = processed_data["pixel_values"].shape[1]
|
||||
processed_data["image_num_patches"] = (
|
||||
processed_data["image_num_crops"] * n_patches
|
||||
)
|
||||
(
|
||||
processed_outputs["image_tokens"],
|
||||
processed_outputs["num_image_tokens"],
|
||||
processed_data["image_tokens"],
|
||||
processed_data["num_image_tokens"],
|
||||
) = build_flat_image_bool_length(
|
||||
image_grids,
|
||||
hf_config,
|
||||
@@ -2101,7 +2106,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
use_single_crop_start_token=hf_processor.use_single_crop_start_token,
|
||||
)
|
||||
|
||||
return BatchFeature({**processed_outputs, **all_video_outputs})
|
||||
processed_data.update(all_video_outputs)
|
||||
processed_data.update(passthrough_data)
|
||||
processed_data.pop("input_ids")
|
||||
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user