[MM] Further cleanup _apply_hf_processor_main (#53610)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-09-16 10:21:00 -07:00
committed by GitHub
parent 6ca2b23e22
commit 0b9e018fb9
74 changed files with 1048 additions and 1323 deletions
@@ -219,18 +219,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
Subclasses add the strategy for locating placeholders in the prompt.
"""
def _get_hf_mm_data(
self,
mm_items: MultiModalDataItems,
) -> tuple[Mapping[str, object], Mapping[str, object]]:
"""Rename the parser's `audios` key to the `audio` argument HF audio
processors take.
"""
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
if self.info._is_audio_model() and "audios" in processor_data:
processor_data["audio"] = processor_data.pop("audios")
return processor_data, passthrough_data
def _get_modality_field_names(self, modality: str) -> set[str]:
"""Names of the fields the sub-processor for `modality` produces."""
# TODO: use else branch only once huggingface/transformers#44394 lands.
@@ -353,6 +341,9 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
return mm_fields
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _unpad_audios(
self,
hf_inputs: "BatchFeature",
@@ -597,21 +588,31 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
# work for Transformers. The vision path has logic tied to
# `mm_tokens_per_modality` in _apply_vision()
hf_processor_mm_kwargs = {
**hf_processor_mm_kwargs,
# vLLM needs the untruncated sequence to keep placeholder
# tokens aligned. Note that the text inputs are just dummy
# text, not the original prompt. The original prompt is
# already tokenized by the renderer.
"truncation": False,
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
# (the HF processor call disables `add_special_tokens`).
"add_special_tokens": False,
**hf_processor_mm_kwargs,
}
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
processor_data, _, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_processor_mm_kwargs
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
# The real prompt is fed to the processor below and the placeholder
# ranges are read back out of its output, so the dummy text meant for
# media-only callers is discarded here.
processor_data.pop("text", None)
has_mm_data = any(processor_data.values())
# Ask which modality owns each token of the expanded prompt, which
# is the only marking of the tokens an expansion adds around an item
if any(processor_data.values()):
if has_mm_data:
processor_data = {**processor_data, "return_mm_token_type_ids": True}
try:
@@ -621,7 +622,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
hf_processor_mm_kwargs,
)
except ValueError:
if any(processor_data.values()):
if has_mm_data:
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
@@ -792,31 +793,37 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
except (AttributeError, KeyError, TypeError):
return None
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> "BatchFeature":
mm_counts = mm_items.get_all_counts()
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
prompt_text = hf_data.pop("text")
assert isinstance(prompt_text, str)
# Ask for the replacement each placeholder expands to, and record it as
# per-item fields: its token ids, and the tokens or patches behind them
if has_mm_data := any(mm_data.values()):
mm_data = {**mm_data, "return_text_replacement_offsets": True}
if has_mm_data := any(hf_data.values()):
hf_data = {**hf_data, "return_text_replacement_offsets": True}
try:
hf_inputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **mm_data),
hf_processor_mm_kwargs,
self.info.get_hf_processor(**hf_kwargs),
dict(text=prompt_text, **hf_data),
hf_kwargs,
)
except ValueError:
if any(mm_data.values()):
if any(hf_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
@@ -825,7 +832,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt"
)
self._unpad_images(hf_inputs)
self._unpad_audios(hf_inputs, mm_data, hf_processor_mm_kwargs)
self._unpad_audios(hf_inputs, hf_data, hf_kwargs)
# Drop the inputs the model would reject
hf_inputs.pop("mm_token_type_ids", None)
@@ -845,9 +852,10 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
"so it can be fixed, and install `transformers<5.15.0` in the "
"meantime to locate placeholders in the expanded prompt instead."
)
hf_inputs.update(passthrough_data)
hf_inputs.pop("input_ids", None)
return hf_inputs
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, hf_inputs
)
tokenizer = self.info.get_tokenizer()
replacements = defaultdict[str, list[list[int]]](list)
@@ -865,7 +873,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
)
if modality == "image":
hf_inputs["num_image_patches"] = self._get_num_image_patches(
hf_inputs, mm_data, len(seqs)
hf_inputs, hf_data, len(seqs)
)
elif modality == "audio":
counts = []
@@ -874,10 +882,11 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
hf_inputs["num_audio_tokens"] = torch.tensor(counts)
hf_inputs.update(passthrough_data)
hf_inputs.pop("input_ids")
return hf_inputs
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, hf_inputs
)
# From this version on, a processor reporting no offsets is an error rather than a