[MM] Simplify _apply_hf_processor_main (#53275)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-22 01:49:37 -07:00
committed by GitHub
parent 7ca49fbe4b
commit 0b19ebcacd
90 changed files with 1848 additions and 1706 deletions
@@ -345,28 +345,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
return mm_fields
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Run the HF processor and unpad inputs."""
try:
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
except ValueError:
if any(mm_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
prompt_ids = self.info.get_tokenizer().encode(prompt)
hf_inputs = transformers.BatchFeature(
dict(input_ids=[prompt_ids]), tensor_type="pt"
)
self._unpad_images(hf_inputs)
self._unpad_audios(hf_inputs, mm_data, mm_kwargs)
return hf_inputs
def _unpad_audios(
self,
hf_inputs: "BatchFeature",
@@ -450,19 +428,6 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
deriving them from updates."""
return []
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Ask which modality owns each token of the expanded prompt, which is the
only marking of the tokens an expansion adds around an item."""
if any(mm_data.values()):
mm_data = {**mm_data, "return_mm_token_type_ids": True}
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_mm_token_ids(self, modality: str) -> list[int]:
"""Token ids marking where `modality` sits in the prompt, which for some
processors differ from the placeholder written into it.
@@ -612,24 +577,55 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
with timing_ctx.record("apply_hf_processor"):
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
prompt = hf_processor.decode(prompt)
prompt_text = hf_processor.decode(prompt)
# Bypass cached processor and always apply to the full set of mm inputs
# NOTE: we can't just set caching=False because base class method
# transforms outputs to `MultiModalKwargs` which is not going to
# work for Transformers. The vision path has logic tied to
# `mm_tokens_per_modality` in _apply_vision()
prompt_ids, processed_data = self._apply_hf_processor_text_mm(
prompt_text=prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={
**hf_processor_mm_kwargs,
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
# (`_call_hf_processor` disables `add_special_tokens`).
"add_special_tokens": False,
},
hf_processor_mm_kwargs = {
**hf_processor_mm_kwargs,
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
# (the HF processor call disables `add_special_tokens`).
"add_special_tokens": False,
}
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
# Ask which modality owns each token of the expanded prompt, which
# is the only marking of the tokens an expansion adds around an item
if any(processor_data.values()):
processor_data = {**processor_data, "return_mm_token_type_ids": True}
try:
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **processor_data),
hf_processor_mm_kwargs,
)
except ValueError:
if any(processor_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
prompt_ids = self.info.get_tokenizer().encode(prompt_text)
processed_data = transformers.BatchFeature(
dict(input_ids=[prompt_ids]), tensor_type="pt"
)
self._unpad_images(processed_data)
self._unpad_audios(processed_data, processor_data, hf_processor_mm_kwargs)
processed_data.update(passthrough_data)
input_ids = processed_data.pop("input_ids")
if not isinstance(input_ids, list):
input_ids = input_ids.tolist()
(prompt_ids,) = input_ids
# Use overrides if provided; fallback to data-dependent hashing.
with timing_ctx.record("get_mm_hashes"):
@@ -773,17 +769,41 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
except (AttributeError, KeyError, TypeError):
return None
def _call_hf_processor(
def _apply_hf_processor_main(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Ask for the replacement each placeholder expands to, and record it as
per-item fields: its token ids, and the tokens or patches behind them."""
mm_counts = mm_items.get_all_counts()
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
# Ask for the replacement each placeholder expands to, and record it as
# per-item fields: its token ids, and the tokens or patches behind them
if has_mm_data := any(mm_data.values()):
mm_data = {**mm_data, "return_text_replacement_offsets": True}
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
try:
hf_inputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
)
except ValueError:
if any(mm_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
tokenizer = self.info.get_tokenizer()
hf_inputs = transformers.BatchFeature(
dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt"
)
self._unpad_images(hf_inputs)
self._unpad_audios(hf_inputs, mm_data, hf_processor_mm_kwargs)
# Drop the inputs the model would reject
hf_inputs.pop("mm_token_type_ids", None)
hf_inputs.pop("token_type_ids", None)
@@ -802,6 +822,8 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
"so it can be fixed, and install `transformers<5.15.0` in the "
"meantime to locate placeholders in the expanded prompt instead."
)
hf_inputs.update(passthrough_data)
hf_inputs.pop("input_ids", None)
return hf_inputs
tokenizer = self.info.get_tokenizer()
@@ -829,6 +851,9 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
hf_inputs["num_audio_tokens"] = torch.tensor(counts)
hf_inputs.update(passthrough_data)
hf_inputs.pop("input_ids")
return hf_inputs