[MM] Simplify _apply_hf_processor_main (#53275)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -5,9 +5,10 @@ from collections import defaultdict, deque
|
||||
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from enum import Enum
|
||||
from functools import lru_cache, partial
|
||||
from functools import lru_cache
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
ClassVar,
|
||||
Generic,
|
||||
NamedTuple,
|
||||
Protocol,
|
||||
@@ -28,7 +29,6 @@ from vllm.inputs import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.transformers_utils.processor import call_hf_processor_mm_only
|
||||
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
|
||||
|
||||
from ..inputs import (
|
||||
@@ -1177,6 +1177,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
Not to be confused with `transformers.ProcessorMixin`.
|
||||
"""
|
||||
|
||||
hf_processor_applies_updates: ClassVar[bool] = False
|
||||
"""
|
||||
Only set to True if the tokenizer or chat template
|
||||
(which are run before MM processing) inserts placeholder tokens.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
info: _I,
|
||||
@@ -1302,131 +1308,100 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
return processor_data, passthrough_data
|
||||
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str | None:
|
||||
"""
|
||||
Get the text to pass to the HF processor alongside the multi-modal
|
||||
data.
|
||||
|
||||
By default, no text is passed. If the HF processor requires that
|
||||
text and multi-modal items correspond to each other, you should
|
||||
override this method to return dummy text generated by
|
||||
[`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder].
|
||||
"""
|
||||
return None
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
# Not to be confused with `mm_data` in `self.apply`.
|
||||
# This refers to the data to be passed to HF processor.
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
"""
|
||||
Pre-process the multi-modal data and HF processor keyword arguments
|
||||
before they are passed to the HF processor.
|
||||
|
||||
By default, both are returned as-is. If the HF processor expects the
|
||||
multi-modal data under different keys than those provided by the
|
||||
multi-modal items (e.g. `audio` instead of `audios`), or requires
|
||||
additional keyword arguments (e.g. `sampling_rate`), you should
|
||||
override this method.
|
||||
"""
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Call the HF processor on the prompt text and
|
||||
associated multi-modal data.
|
||||
Post-process the output of the HF processor.
|
||||
|
||||
By default, the output is returned as-is. If you need to modify the
|
||||
output of the HF processor before it is converted into multi-modal
|
||||
keyword arguments, you should override this method.
|
||||
"""
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
mm_kwargs,
|
||||
)
|
||||
|
||||
def _apply_hf_processor_text_mm(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[list[int], BatchFeature]:
|
||||
"""
|
||||
Apply the HF processor on the prompt text and multi-modal data
|
||||
together.
|
||||
"""
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
processed_data = self._call_hf_processor(
|
||||
prompt=prompt_text,
|
||||
mm_data=processor_data,
|
||||
mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
input_ids = processed_data.pop("input_ids")
|
||||
if not isinstance(input_ids, list):
|
||||
input_ids = input_ids.tolist()
|
||||
|
||||
(prompt_ids,) = input_ids
|
||||
|
||||
return prompt_ids, processed_data
|
||||
|
||||
def _apply_hf_processor_tokens_only(
|
||||
self,
|
||||
prompt_tokens: list[int],
|
||||
) -> list[int]:
|
||||
"""
|
||||
Apply the HF processor on the prompt tokens only.
|
||||
|
||||
Most HF processors accept prompt text but not prompt tokens.
|
||||
If the HF processor adds or removes tokens that are not related to
|
||||
multi-modal data, you should override this method so it is consistent
|
||||
with the tokenization of the corresponding text.
|
||||
"""
|
||||
return prompt_tokens
|
||||
|
||||
def _apply_hf_processor_mm_only(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Apply the HF processor on the multi-modal data only.
|
||||
|
||||
Since HF processor requires that text and multi-modal items
|
||||
correspond to each other, we generate dummy text using
|
||||
[`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
|
||||
to go along with the multi-modal data.
|
||||
"""
|
||||
# Custom logic based on text inputs
|
||||
if type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor:
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
|
||||
_, mm_processed_data = self._apply_hf_processor_text_mm(
|
||||
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
return mm_processed_data
|
||||
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
partial(
|
||||
call_hf_processor_mm_only,
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
),
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[list[int], BatchFeature, bool]:
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Apply the HF processor on the prompt tokens and multi-modal data.
|
||||
|
||||
In addition, return whether prompt updates have been applied
|
||||
(`False` here, since for token prompts the updates are normally
|
||||
applied separately via `_maybe_apply_prompt_updates`).
|
||||
Apply the HF processor on the multi-modal data.
|
||||
"""
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
mm_processed_data = self._apply_hf_processor_mm_only(
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
if processor_data:
|
||||
processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
|
||||
processor_data, hf_processor_mm_kwargs
|
||||
)
|
||||
|
||||
prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
|
||||
if prompt_text is not None:
|
||||
processor_data = dict(text=prompt_text, **processor_data)
|
||||
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
else:
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
processed_data = BatchFeature(dict(passthrough_data))
|
||||
|
||||
return self._postprocess_hf_mm_data(
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
processed_data,
|
||||
)
|
||||
|
||||
return prompt_ids, mm_processed_data, False
|
||||
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
|
||||
"""
|
||||
Post-process the prompt token IDs before locating or applying
|
||||
multi-modal prompt updates.
|
||||
|
||||
By default, the prompt is returned as-is. If the HF processor (or
|
||||
chat template) applies additional transformations to the prompt
|
||||
that are not reflected in its multi-modal outputs, you should
|
||||
override this method to replicate them.
|
||||
"""
|
||||
return prompt
|
||||
|
||||
def _get_cache_missing_items(
|
||||
self,
|
||||
@@ -1531,14 +1506,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
||||
) -> MultiModalProcessingInfo:
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
(
|
||||
prompt_ids,
|
||||
mm_processed_data,
|
||||
is_update_applied,
|
||||
) = self._apply_hf_processor_main(
|
||||
prompt=inputs.prompt,
|
||||
mm_processed_data = self._apply_hf_processor_main(
|
||||
mm_items=inputs.mm_data_items,
|
||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
)
|
||||
@@ -1569,13 +1539,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
prompt_updates=mm_prompt_updates,
|
||||
)
|
||||
|
||||
return prompt_ids, mm_info, is_update_applied
|
||||
return mm_info
|
||||
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
||||
) -> MultiModalProcessingInfo:
|
||||
"""
|
||||
Apply the HF processor on the full prompt text,
|
||||
caching the results and reusing cached results.
|
||||
@@ -1599,16 +1569,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_hashes=mm_hashes,
|
||||
)
|
||||
|
||||
# NOTE: `prompt` does not correspond to `mm_missing_data_items`,
|
||||
# NOTE: The prompt does not correspond to `mm_missing_data_items`,
|
||||
# so we can't apply prompt updates until the new multimodal
|
||||
# items are combined with the cached multimodal items
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
(
|
||||
prompt_ids,
|
||||
mm_missing_processed_data,
|
||||
is_update_applied,
|
||||
) = self._apply_hf_processor_main(
|
||||
prompt=inputs.prompt,
|
||||
mm_missing_processed_data = self._apply_hf_processor_main(
|
||||
mm_items=mm_missing_data_items,
|
||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
)
|
||||
@@ -1641,7 +1606,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
prompt_updates=mm_prompt_updates,
|
||||
)
|
||||
|
||||
return prompt_ids, mm_info, is_update_applied
|
||||
return mm_info
|
||||
|
||||
def _apply_token_matches(
|
||||
self,
|
||||
@@ -1770,7 +1735,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"There is likely a problem with your "
|
||||
"implementation of merged multi-modal processor for this "
|
||||
"model (usually arising from an inconsistency between "
|
||||
"`_call_hf_processor` and `_get_mm_fields_config`)."
|
||||
"`_apply_hf_processor_main` and `_get_mm_fields_config`)."
|
||||
)
|
||||
|
||||
def _validate_mm_updates(
|
||||
@@ -1805,8 +1770,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
f"Expected there to be {item_count} prompt placeholders "
|
||||
f"corresponding to {item_count} {modality} items, but "
|
||||
f"instead found {len(placeholders)} prompt placeholders! "
|
||||
"Make sure the implementation of `_call_hf_processor` and "
|
||||
"`_get_mm_fields_config` are consistent with each other."
|
||||
"Make sure the implementation of `_apply_hf_processor_main` "
|
||||
"and `_get_mm_fields_config` are consistent with each other."
|
||||
)
|
||||
|
||||
def _maybe_apply_prompt_updates(
|
||||
@@ -1854,11 +1819,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
3. Extract information about the placeholder tokens from the
|
||||
processed token IDs.
|
||||
"""
|
||||
(
|
||||
prompt_ids,
|
||||
mm_info,
|
||||
is_update_applied,
|
||||
) = self._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
prompt_ids = self._postprocess_prompt(inputs.prompt)
|
||||
mm_info = self._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
with timing_ctx.record("apply_prompt_updates"):
|
||||
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
|
||||
@@ -1866,7 +1828,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
prompt_ids=prompt_ids,
|
||||
mm_kwargs=mm_info.kwargs,
|
||||
mm_prompt_updates=mm_info.prompt_updates,
|
||||
is_update_applied=is_update_applied,
|
||||
is_update_applied=self.hf_processor_applies_updates,
|
||||
)
|
||||
|
||||
mm_placeholder_ranges = {
|
||||
|
||||
Reference in New Issue
Block a user