[MM] Further cleanup _apply_hf_processor_main (#53610)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-09-16 10:21:00 -07:00
committed by GitHub
parent 6ca2b23e22
commit 0b9e018fb9
74 changed files with 1048 additions and 1323 deletions
+116 -84
View File
@@ -1008,7 +1008,20 @@ For an item `MultiModalPromptUpdates[k][i]`,
_I = TypeVar("_I", bound=BaseProcessingInfo)
class MultiModalProcessingInfo(NamedTuple):
class HFMultiModalInputs(NamedTuple):
hf_data: dict[str, object]
"""The data inputs to the HF processor."""
hf_kwargs: Mapping[str, object]
"""The non-data inputs to the HF processor."""
passthrough_data: dict[str, object]
"""The inputs to assign to the output of the HF processor,
instead of passing them to the HF processor."""
class MultiModalProcessingResult(NamedTuple):
prompt_ids: list[int]
kwargs: MultiModalKwargsOptionalItems
hashes: MultiModalHashes
prompt_updates: MultiModalPromptUpdates
@@ -1137,23 +1150,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
return find_mm_placeholders(new_token_ids, mm_prompt_updates)
def _get_hf_mm_data(
self,
mm_items: MultiModalDataItems,
) -> tuple[Mapping[str, object], Mapping[str, object]]:
"""Extract processor and passthrough data from multi-modal items."""
processor_data = dict[str, object]()
passthrough_data = dict[str, object]()
for items in mm_items.values():
processor_data.update(items.get_processor_data())
passthrough_data.update(items.get_passthrough_data())
return processor_data, passthrough_data
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str | None:
"""Get the text to pass to the HF processor alongside the multi-modal
data.
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None:
"""Get the text to pass to the HF processor alongside the multi-modal data.
By default, no text is passed. If the HF processor requires that
text and multi-modal items correspond to each other, you should
@@ -1162,29 +1160,53 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
"""
return None
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
"""Pre-process the multi-modal data and HF processor keyword arguments
before they are passed to the HF processor.
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
"""Extract the multi-modal data and corresponding keyword arguments
for HF processing.
By default, both are returned as-is. If the HF processor expects the
multi-modal data under different keys than those provided by the
multi-modal items (e.g. `audio` instead of `audios`), or requires
additional keyword arguments (e.g. `sampling_rate`), you should
override this method.
If the HF processor expects the multi-modal data under different keys
than those provided by the multi-modal items, or requires additional
keyword arguments, you should override this method.
"""
return mm_data, hf_processor_mm_kwargs
hf_data = dict[str, object]()
passthrough_data = dict[str, object]()
for items in mm_items.values():
if not items:
continue
hf_data.update(items.get_processor_data())
passthrough_data.update(items.get_passthrough_data())
if hf_data:
prompt_text = self._get_hf_mm_text(mm_items.get_all_counts())
if prompt_text is not None:
hf_data = dict(text=prompt_text, **hf_data)
# vLLM needs the untruncated sequence to keep placeholder
# tokens aligned. Note that the text inputs are just dummy
# text, not the original prompt. The original prompt is
# already tokenized by the renderer.
hf_kwargs = {"truncation": False, **hf_kwargs}
# HF processors accept (images, text, videos, audio)
# so we need to remap vLLM's "audios" -> HF "audio"
if "audios" in hf_data:
hf_data["audio"] = hf_data.pop("audios")
return HFMultiModalInputs(hf_data, hf_kwargs, passthrough_data)
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
mm_inputs: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
"""Post-process the output of the HF processor.
"""Post-process the combined processor and passthrough data.
By default, the output is returned as-is. If you need to modify the
output of the HF processor before it is converted into multi-modal
@@ -1192,41 +1214,51 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
"""
return processed_data
def _call_hf_processor(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Call the HF processor on the normalized multi-modal inputs."""
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_kwargs),
hf_data,
hf_kwargs,
)
def _finalize_hf_mm_data(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
passthrough_data: Mapping[str, object],
processed_data: BatchFeature | None = None,
) -> BatchFeature:
"""Merge passthrough fields and apply model-specific postprocessing."""
if processed_data is None:
from transformers.feature_extraction_utils import BatchFeature
processed_data = BatchFeature()
processed_data.update(passthrough_data)
return self._postprocess_hf_mm_data(hf_data, hf_kwargs, processed_data)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Apply the HF processor on the multi-modal data."""
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
if processor_data:
processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
processor_data, hf_processor_mm_kwargs
)
prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
if prompt_text is not None:
processor_data = dict(text=prompt_text, **processor_data)
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
processor_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
if hf_data:
processed_data = self._call_hf_processor(hf_data, hf_kwargs)
else:
from transformers.feature_extraction_utils import BatchFeature
processed_data = None
processed_data = BatchFeature(dict(passthrough_data))
return self._postprocess_hf_mm_data(
processor_data,
hf_processor_mm_kwargs,
processed_data,
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
@@ -1261,6 +1293,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_missing_data = {}
for modality, idxs in mm_missing_idxs.items():
if not idxs:
continue
missing_modality_data = []
for idx in idxs:
data = mm_data_items[modality][idx]
@@ -1271,6 +1306,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
)
else:
missing_modality_data.append(data)
mm_missing_data[modality] = missing_modality_data
mm_missing_items = self.info.parse_mm_data(mm_missing_data, validate=False)
@@ -1342,11 +1378,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
) -> MultiModalProcessingResult:
with timing_ctx.record("apply_hf_processor"):
mm_processed_data = self._apply_hf_processor_main(
mm_items=inputs.mm_data_items,
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
hf_kwargs=inputs.hf_processor_mm_kwargs,
)
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
@@ -1369,26 +1405,28 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_kwargs,
)
mm_info = MultiModalProcessingInfo(
return MultiModalProcessingResult(
prompt_ids=self._postprocess_prompt(inputs.prompt),
kwargs=mm_kwargs,
hashes=mm_hashes,
prompt_updates=mm_prompt_updates,
)
return mm_info
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
) -> MultiModalProcessingResult:
"""Apply the HF processor on the full prompt text,
caching the results and reusing cached results.
"""
cache = self.cache
has_passthrough_data = any(
len(items.get_passthrough_data()) > 0
for items in inputs.mm_data_items.values()
)
_, passthrough_data = self._get_hf_mm_data(inputs.mm_data_items)
if cache is None or passthrough_data:
if cache is None or has_passthrough_data:
return self._apply_hf_processor(inputs, timing_ctx)
with timing_ctx.record("get_mm_hashes"):
@@ -1410,7 +1448,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
with timing_ctx.record("apply_hf_processor"):
mm_missing_processed_data = self._apply_hf_processor_main(
mm_items=mm_missing_data_items,
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
hf_kwargs=inputs.hf_processor_mm_kwargs,
)
mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
@@ -1435,14 +1473,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_missing_prompt_updates=mm_missing_prompt_updates,
)
mm_info = MultiModalProcessingInfo(
return MultiModalProcessingResult(
prompt_ids=self._postprocess_prompt(inputs.prompt),
kwargs=mm_kwargs,
hashes=mm_hashes,
prompt_updates=mm_prompt_updates,
)
return mm_info
def _apply_token_matches(
self,
prompt: list[int],
@@ -1670,17 +1707,15 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsOptionalItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
mm_item_counts = mm_items.get_all_counts()
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)
self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)
prompt_ids, mm_placeholders = self._apply_prompt_updates(
prompt_ids,
mm_prompt_updates,
mm_res.prompt_ids,
mm_res.prompt_updates,
)
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
@@ -1705,15 +1740,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
4. Extract information about the placeholder tokens from the
processed token IDs.
"""
prompt_ids = self._postprocess_prompt(inputs.prompt)
mm_info = self._cached_apply_hf_processor(inputs, timing_ctx)
mm_res = self._cached_apply_hf_processor(inputs, timing_ctx)
with timing_ctx.record("apply_prompt_updates"):
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
mm_items=inputs.mm_data_items,
prompt_ids=prompt_ids,
mm_kwargs=mm_info.kwargs,
mm_prompt_updates=mm_info.prompt_updates,
mm_res=mm_res,
)
mm_placeholder_ranges = {
@@ -1723,8 +1755,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
return mm_input(
prompt_token_ids=prompt_ids,
mm_kwargs=mm_info.kwargs,
mm_hashes=mm_info.hashes,
mm_kwargs=mm_res.kwargs,
mm_hashes=mm_res.hashes,
mm_placeholders=mm_placeholder_ranges,
)