[MM] Simplify _apply_hf_processor_main (#53275)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-22 01:49:37 -07:00
committed by GitHub
parent 7ca49fbe4b
commit 0b19ebcacd
90 changed files with 1848 additions and 1706 deletions
+99 -137
View File
@@ -5,9 +5,10 @@ from collections import defaultdict, deque
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
from dataclasses import dataclass, field, replace
from enum import Enum
from functools import lru_cache, partial
from functools import lru_cache
from typing import (
TYPE_CHECKING,
ClassVar,
Generic,
NamedTuple,
Protocol,
@@ -28,7 +29,6 @@ from vllm.inputs import (
)
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike
from vllm.transformers_utils.processor import call_hf_processor_mm_only
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
from ..inputs import (
@@ -1177,6 +1177,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
Not to be confused with `transformers.ProcessorMixin`.
"""
hf_processor_applies_updates: ClassVar[bool] = False
"""
Only set to True if the tokenizer or chat template
(which are run before MM processing) inserts placeholder tokens.
"""
def __init__(
self,
info: _I,
@@ -1302,131 +1308,100 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
return processor_data, passthrough_data
def _call_hf_processor(
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str | None:
"""
Get the text to pass to the HF processor alongside the multi-modal
data.
By default, no text is passed. If the HF processor requires that
text and multi-modal items correspond to each other, you should
override this method to return dummy text generated by
[`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder].
"""
return None
def _preprocess_hf_mm_data(
self,
prompt: str,
# Not to be confused with `mm_data` in `self.apply`.
# This refers to the data to be passed to HF processor.
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
"""
Pre-process the multi-modal data and HF processor keyword arguments
before they are passed to the HF processor.
By default, both are returned as-is. If the HF processor expects the
multi-modal data under different keys than those provided by the
multi-modal items (e.g. `audio` instead of `audios`), or requires
additional keyword arguments (e.g. `sampling_rate`), you should
override this method.
"""
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
"""
Call the HF processor on the prompt text and
associated multi-modal data.
Post-process the output of the HF processor.
By default, the output is returned as-is. If you need to modify the
output of the HF processor before it is converted into multi-modal
keyword arguments, you should override this method.
"""
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(text=prompt, **mm_data),
mm_kwargs,
)
def _apply_hf_processor_text_mm(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[list[int], BatchFeature]:
"""
Apply the HF processor on the prompt text and multi-modal data
together.
"""
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
processed_data = self._call_hf_processor(
prompt=prompt_text,
mm_data=processor_data,
mm_kwargs=hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
input_ids = processed_data.pop("input_ids")
if not isinstance(input_ids, list):
input_ids = input_ids.tolist()
(prompt_ids,) = input_ids
return prompt_ids, processed_data
def _apply_hf_processor_tokens_only(
self,
prompt_tokens: list[int],
) -> list[int]:
"""
Apply the HF processor on the prompt tokens only.
Most HF processors accept prompt text but not prompt tokens.
If the HF processor adds or removes tokens that are not related to
multi-modal data, you should override this method so it is consistent
with the tokenization of the corresponding text.
"""
return prompt_tokens
def _apply_hf_processor_mm_only(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> BatchFeature:
"""
Apply the HF processor on the multi-modal data only.
Since HF processor requires that text and multi-modal items
correspond to each other, we generate dummy text using
[`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
to go along with the multi-modal data.
"""
# Custom logic based on text inputs
if type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor:
mm_counts = mm_items.get_all_counts()
_, mm_processed_data = self._apply_hf_processor_text_mm(
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
return mm_processed_data
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
processed_data = self.info.ctx.call_hf_processor(
partial(
call_hf_processor_mm_only,
self.info.get_hf_processor(**hf_processor_mm_kwargs),
),
processor_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
return processed_data
def _apply_hf_processor_main(
self,
prompt: list[int],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[list[int], BatchFeature, bool]:
) -> BatchFeature:
"""
Apply the HF processor on the prompt tokens and multi-modal data.
In addition, return whether prompt updates have been applied
(`False` here, since for token prompts the updates are normally
applied separately via `_maybe_apply_prompt_updates`).
Apply the HF processor on the multi-modal data.
"""
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
valid_mm_items = mm_items.select(
{k for k, c in mm_items.get_all_counts().items() if c > 0}
)
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
mm_processed_data = self._apply_hf_processor_mm_only(
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
if processor_data:
processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
processor_data, hf_processor_mm_kwargs
)
prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
if prompt_text is not None:
processor_data = dict(text=prompt_text, **processor_data)
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
processor_data,
hf_processor_mm_kwargs,
)
processed_data.update(passthrough_data)
else:
from transformers.feature_extraction_utils import BatchFeature
processed_data = BatchFeature(dict(passthrough_data))
return self._postprocess_hf_mm_data(
processor_data,
hf_processor_mm_kwargs,
processed_data,
)
return prompt_ids, mm_processed_data, False
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
"""
Post-process the prompt token IDs before locating or applying
multi-modal prompt updates.
By default, the prompt is returned as-is. If the HF processor (or
chat template) applies additional transformations to the prompt
that are not reflected in its multi-modal outputs, you should
override this method to replicate them.
"""
return prompt
def _get_cache_missing_items(
self,
@@ -1531,14 +1506,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
) -> MultiModalProcessingInfo:
with timing_ctx.record("apply_hf_processor"):
(
prompt_ids,
mm_processed_data,
is_update_applied,
) = self._apply_hf_processor_main(
prompt=inputs.prompt,
mm_processed_data = self._apply_hf_processor_main(
mm_items=inputs.mm_data_items,
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
)
@@ -1569,13 +1539,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
prompt_updates=mm_prompt_updates,
)
return prompt_ids, mm_info, is_update_applied
return mm_info
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
) -> MultiModalProcessingInfo:
"""
Apply the HF processor on the full prompt text,
caching the results and reusing cached results.
@@ -1599,16 +1569,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_hashes=mm_hashes,
)
# NOTE: `prompt` does not correspond to `mm_missing_data_items`,
# NOTE: The prompt does not correspond to `mm_missing_data_items`,
# so we can't apply prompt updates until the new multimodal
# items are combined with the cached multimodal items
with timing_ctx.record("apply_hf_processor"):
(
prompt_ids,
mm_missing_processed_data,
is_update_applied,
) = self._apply_hf_processor_main(
prompt=inputs.prompt,
mm_missing_processed_data = self._apply_hf_processor_main(
mm_items=mm_missing_data_items,
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
)
@@ -1641,7 +1606,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
prompt_updates=mm_prompt_updates,
)
return prompt_ids, mm_info, is_update_applied
return mm_info
def _apply_token_matches(
self,
@@ -1770,7 +1735,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
"There is likely a problem with your "
"implementation of merged multi-modal processor for this "
"model (usually arising from an inconsistency between "
"`_call_hf_processor` and `_get_mm_fields_config`)."
"`_apply_hf_processor_main` and `_get_mm_fields_config`)."
)
def _validate_mm_updates(
@@ -1805,8 +1770,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
f"Expected there to be {item_count} prompt placeholders "
f"corresponding to {item_count} {modality} items, but "
f"instead found {len(placeholders)} prompt placeholders! "
"Make sure the implementation of `_call_hf_processor` and "
"`_get_mm_fields_config` are consistent with each other."
"Make sure the implementation of `_apply_hf_processor_main` "
"and `_get_mm_fields_config` are consistent with each other."
)
def _maybe_apply_prompt_updates(
@@ -1854,11 +1819,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
3. Extract information about the placeholder tokens from the
processed token IDs.
"""
(
prompt_ids,
mm_info,
is_update_applied,
) = self._cached_apply_hf_processor(inputs, timing_ctx)
prompt_ids = self._postprocess_prompt(inputs.prompt)
mm_info = self._cached_apply_hf_processor(inputs, timing_ctx)
with timing_ctx.record("apply_prompt_updates"):
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
@@ -1866,7 +1828,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
prompt_ids=prompt_ids,
mm_kwargs=mm_info.kwargs,
mm_prompt_updates=mm_info.prompt_updates,
is_update_applied=is_update_applied,
is_update_applied=self.hf_processor_applies_updates,
)
mm_placeholder_ranges = {